/*       Copyright (c) Eric Ledoux.  All rights reserved.       */
/* See http://www.dwell.net/terms for code sharing information. */

// CSharpSourceFile.cs
//
// Implements C# source code parsing and formatting.
//

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using DwellNet.CodeDoc;
using CodeDocApi.Properties;

namespace DwellNet.CodeDoc
{

public class CSharpSourceFile : SourceFile
{
    //////////////////////////////////////////////////////////////////////////
    // Private Statics Used For Parsing XML Comments
    //

    static readonly Regex s_tripleSlashRegex = new Regex(@"^ *\/\/\/");

    static readonly Regex s_spacesPrefixRegex = new Regex(@"^ *");

    static readonly Regex s_allBlankRegex = new Regex(@"^ *$");

    static readonly Regex s_lineOfSlashRegex = new Regex(@"^ */{5,} *$");

    static readonly Regex s_lastNewline = new Regex(@"\n$");

    static readonly Regex s_slashStarStarRegex = new Regex(@"^ */\*\* *");

    static readonly Regex s_starSlashRegex = new Regex(@" *\*/ *$");

    static readonly Regex s_spaceStarSpaceRegex = new Regex(@"^ *\*? *");

    static readonly Regex s_lineOfStarRegex = new Regex(@"^ *\*{5,} *$");

    static readonly Regex s_preprocSymbolRegex = new Regex(@"^[a-zA-Z0-9_]+$");

    static readonly char[] s_operatorSymbolCharacters = new char[]
    {
        '+', '-', '!', '~', '*', '/', '%', '&', '|', '^', '<', '>', '='
    };

    //////////////////////////////////////////////////////////////////////////
    // Internal Methods
    //

    internal CSharpSourceFile(DocumentationSet docSet,
        string sourceFileAbsolutePath, string sourceFileRelativePath)
    {
        m_docSet = docSet;
        m_sourceFileAbsolutePath = sourceFileAbsolutePath;
        m_sourceFileRelativePath = sourceFileRelativePath;
    }

    internal override void Load(ICollection<string> initialSymbols)
    {
        // load the C# file into the linked list, between <m_headToken> and
        // <m_tailToken>
        using (StreamReader reader = new StreamReader(m_sourceFileAbsolutePath,
                Encoding.UTF8))
            Load(reader);

        // perform preprocessing of conditional compilation constructs like #if
        Preprocess(initialSymbols);

        // locate documentation topics within the linked list of tokens
        LocateTopics();
    }

    internal void Load(TextReader reader)
    {
        CSharpTokenParser tokenParser =
            new CSharpTokenParser(m_sourceFileAbsolutePath, reader);
        while (true)
        {
            int lineNumber = tokenParser.CurrentLineNumber;
            Token token = tokenParser.ParseToken();
            if (token == null)
                break;
            if (token.IsAnXmlComment)
                AddXmlComment(token);
            else
                Append(token);
        }
    }

/// <summary>
/// My Foo class.
/// </summary>
class Foo
#if SOME_SYMBOL
    : Bar
#endif
{
}

/// <summary>
/// My Foo class.
/// </summary>
class Foo
    : Bar
{
}

/// <summary>
/// My Foo class.
/// </summary>
class Foo
{
}

    internal void Preprocess(ICollection<string> initialSymbols)
    {
        // <symbols> will hold a dictionary containing the names of
        // preprocessor symbols we know about, mapped to true (if currently
        // defined) or false (if not); initialize <symbols> from
        // <initialSymbols> (if not null)
        Dictionary<string, bool> symbols = new Dictionary<string, bool>(20);
        if (initialSymbols != null)
        {
            foreach (string symbol in initialSymbols)
                symbols[symbol] = true;
        }

        // preprocess the outer run of tokens
        Token token = HeadToken.Next;
        PreprocessRun(ref token, false, symbols);

#if false
        // for debugging purposes, display the results of preprocessing
        foreach (Token t in this)
            Console.WriteLine(t.Dump());
#endif

        // error if we're not at the end of the file
        if (!token.IsTail)
            throw Unexpected(token, Resources.UnexpectedEndif);
    }

    internal void PreprocessRun(ref Token token, bool exclude,
        Dictionary<string, bool> symbols)
    {
        // loop once per source file line
        while (!token.IsTail)
        {
            // set <startOfLine> to the token that begins this line; set
            // <firstSignificant> to the first token in the line that isn't
            // white space, a comment, etc.; set <endOfLine> to the
            // TokenType.Newline, TokenType.XmlComment, or TokenType.Tail
            // that ends this line
            Token startOfLine = token;
            ParseInsignificantTokens(ref token, null);
            Token firstSignificant = token;
            Token endOfLine = token.FindEndOfLineOrTripleSlash();

            // set <token> to the significant token on the line
            token = firstSignificant;

            // see if this line begins an "#if...#elif...#else...#endif" block;
            // note that even in "exclude" mode we need to match an "#if" with
            // the corresponding "#endif" so we know when enclosing run ends
            if (token.IsReservedWord("#if"))
            {
                // loop once for each "#if" and "#elif" block; set
                // <innerExclude> to true when following "#if" and "#elif"
                // blocks should be excluded (i.e. after processing an "#if"
                // or "#elif" that had a preprocessor condition expression that
                // evaluated to false, or if <exclude> is true to begin with);
                // quit after "#endif" has been preprocessed
                bool innerExclude = exclude;
                while (true)
                {
                    // keep track of the "#if" or "#elif" token for error
                    // messages
                    Token ifOrElifToken = token;

                    // collect the preprocessor directive into <directiveToken>
                    DirectiveToken directiveToken =
                        ConvertTokensToDirective(ref token);

                    // parse the preprocessor expression; set <condition> to
                    // true if it evaluates to true, false if not
                    bool condition = ParsePreprocessorExpression(
                        directiveToken.ContainedTokens.HeadToken.Next.Next,
                        symbols, directiveToken);

                    // skip to the start of the next line
                    token = token.FindStartOfNextLine();

                    // recursively preprocess the run of tokens until a
                    // "#elif", "#else", or "#endif" that matches this "#if"
                    // or "#elif"; exclude the run (i.e. treat it as you would
                    // code within "#if false ... #endif") if <condition> is
                    // false or <innerExclude> is true
                    PreprocessRun(ref token, innerExclude || !condition,
                        symbols);
                    Debug.Assert(token.IsTail || token.IsAReservedWord);

                    // if this <condition> was true for thi "#if" or "#elif",
                    // following "#elif" blocks will be excluded
                    if (condition)
                        innerExclude = true;

                    // process the next token
                    if (token.IsTail)
                    {
                        // hit end of file without finding "#endif"
                        throw NewParsingException(ifOrElifToken,
                            Resources.IfOrElifWithoutEndif,
                            ifOrElifToken.Text, ifOrElifToken.GetLineNumber());
                    }
                    else
                    if (token.Text == "#else")
                    {
                        // preprocess the "#else" block
                        ConvertTokensToDirective(ref token);
                        token = token.FindStartOfNextLine();
                        PreprocessRun(ref token, innerExclude, symbols);

                        // we should now be at "#endif" -- convert it to a
                        // TokenType.Directive and break
                        if (!token.IsReservedWord("#endif"))
                        {
                            throw NewParsingException(ifOrElifToken,
                                Resources.IfOrElifWithoutEndif,
                                ifOrElifToken.Text,
                                ifOrElifToken.GetLineNumber());
                        }
                        ConvertTokensToDirective(ref token);
                        break;
                    }
                    else
                    if (token.Text == "#endif")
                    {
                        ConvertTokensToDirective(ref token);
                        break;
                    }

                    // this is a "#elif"
                    Debug.Assert(token.Text == "#elif");
                    if (token.Text != "#elif")
                    {
                        throw NewParsingException(token,
                            Resources.InternalError, 101);
                    }
                }

                // skip to the start of the next line (if any)
                token = token.FindStartOfNextLine();
                continue;
            }

            // stop at "#elif", "#else", and "#endif"
            if (token.IsAReservedWord)
            {
                if ((token.Text == "#elif") || (token.Text == "#else") ||
                    (token.Text == "#endif"))
                    break;
            }

            // if we're in "exclude" mode, then since this isn't the beginning
            // of an "#if...#elif...#else...#endif" block, this line (except
            // the terminating TokenType.Newline token) is converted into a
            // single TokenType.Excluded token); note that if this line ends
            // with the beginning of an XML comment, that XML comment token is
            // included in the TokenType.Excluded token)
            if (exclude)
            {
                // move tokens between <startOfLine> and the token before
                // <endOfLine>, inclusive, to a new list, <containedTokens>;
                // set <beforeStartOfLine> to the token before the extracted
                // tokens; set <startOfNextLine> to the beginning of the
                // next line
                Token beforeStartOfLine = startOfLine.Previous;
                TokenList containedTokens;
                Token startOfNextLine = endOfLine.Next;
                if (endOfLine.TokenType == TokenType.XmlComment)
                {
                    containedTokens = TokenList.ExtractTokens(
                        startOfLine, endOfLine);
                }
                else
                {
                    containedTokens = TokenList.ExtractTokens(
                        startOfLine, endOfLine.Previous);
                }

                // if <containedTokens> isn't empty, create a new
                // TokenType.Excluded token to contain <containTokns> and
                // insert it in place of the tokens that were just extracted
                if (!containedTokens.IsEmpty)
                {
                    ExcludedToken excludedToken = new ExcludedToken(
                        containedTokens,
                        containedTokens.ToString(int.MaxValue, true, true));
                    excludedToken.InsertAfter(beforeStartOfLine);
                }

                // skip to the start of the next line (if any) and continue
                token = startOfNextLine;
                continue;
            }

            // skip everything except reserved word tokens starting with '#'
            if ((token.TokenType != TokenType.ReservedWord) ||
                !token.Text.StartsWith("#"))
            {
                token = endOfLine.FindStartOfNextLine();
                continue;
            }

            // see if this is a "#define" or "#undef" directive
            bool isDefine = (token.Text == "#define");
            if (isDefine || (token.Text == "#undef"))
            {
                // collect the preprocessor directive into <directiveToken>
                DirectiveToken directiveToken =
                    ConvertTokensToDirective(ref token);

                // point <tokenInDirective> to the first token after "#define"
                Token tokenInDirective =
                    directiveToken.ContainedTokens.HeadToken.Next;
                Debug.Assert(tokenInDirective.IsReservedWord("#define"));
                tokenInDirective = tokenInDirective.Next;

                // the next token should be the symbol being defined; set
                // <symbol> to its name
                ParseInsignificantTokens(ref tokenInDirective);
                Token symbolToken = tokenInDirective;
                if (!tokenInDirective.IsAWordOrReservedWord ||
                    !s_preprocSymbolRegex.Match(symbolToken.Text).Success)
                {
                    throw NewParsingException(directiveToken,
                        Resources.WantedPreprocSymbol,
                        symbolToken.Description);
                }
                tokenInDirective = tokenInDirective.Next;

                // there should be no more tokens in the directive
                ParseInsignificantTokens(ref tokenInDirective);
                if (!tokenInDirective.IsTail)
                {
                    throw NewParsingException(directiveToken,
                        Resources.UnexpectedTextAfter,
                        symbolToken.Description);
                }

                // define or undefine the symbol
                symbols[symbolToken.Text] = isDefine;

                // skip to the start of the next line (if any)
                token = endOfLine.FindStartOfNextLine();
            }

            // see if this is an "#error" directive -- it's used for unit tests
            if (token.Text == "#error")
            {
                DirectiveToken directiveToken =
                    ConvertTokensToDirective(ref token);
                throw NewParsingException(directiveToken,
                    Resources.ErrorDirective, directiveToken.Text);
            }

            // see if this is any other directive -- if so, just convert to a
            // TokenType.Directive token
            if ((token.Text == "#warning") ||
                (token.Text == "#line") ||
                (token.Text == "#region") ||
                (token.Text == "#endregion") ||
                (token.Text == "#pragma"))
            {
                ConvertTokensToDirective(ref token);
                token = token.FindStartOfNextLine();
                continue;
            }

            // ignore this line
            token = endOfLine.FindStartOfNextLine();
        }
    }

    internal void LocateTopics()
    {
        // locate documentation topics within the linked list of tokens
        Token token2 = HeadToken.Next;
        UsingContext usingContext = new UsingContext();
        LocateTopics(ref token2, null, null, null, usingContext);
    }

    //////////////////////////////////////////////////////////////////////////
    // Private Methods
    //

    DirectiveToken ConvertTokensToDirective(ref Token token)
    {
        // <token> should be a reserved token that starts a preprocessor
        // directive
        Debug.Assert(token.IsAReservedWord);
        Debug.Assert(token.Text.StartsWith("#"));

        // set <firstToken> to the first token of the directive; set
        // <lastToken> to the last token before the end of the line that's not
        // a white space token
        Token firstToken = token;
        Token lastToken = token;
        while (!token.IsNewline)
        {
            if (!token.IsWhiteSpace)
                lastToken = token;
            token = token.Next;
        }

        // move tokens between <firstToken> and <lastToken>, inclusive, to a
        // new list, <containedTokens>; set <beforeFirstToken> to the token that
        // precedes the extracted token
        Token beforeFirstToken = firstToken.Previous;
        TokenList containedTokens = TokenList.ExtractTokens(firstToken,
            lastToken);

        // set <directiveToken> to the new TokenType.Directive token that will
        // contain <containedTokens>
        DirectiveToken directiveToken = new DirectiveToken(containedTokens,
            containedTokens.ToString());

        // insert <directiveToken> in place of the tokens that were just
        // removed from the list
        directiveToken.InsertAfter(beforeFirstToken);

        // done
        return directiveToken;
    }

    bool ParsePreprocessorExpression(Token tokenInDirective,
        Dictionary<string, bool> symbols, Token directiveToken)
    {
        // From the C# Language Specification 1.2 (section 2.5.2):
        //
        // pp-expr:
        //   -- [whitespace] pp-or-expr [whitespace]
        //
        // pp-or-expr:
        //   -- pp-and-expr
        //   -- pp-or-expr [whitespace] "||" [whitespace] pp-and-expr
        //
        // pp-and-expr:
        //   -- pp-equality-expr
        //   -- pp-and-expr [whitespace] "&&" [whitespace] pp-equality-expr
        //
        // pp-equality-expr:
        //   -- pp-unary-expr
        //   -- pp-equality-expr [whitespace] "==" [whitespace] pp-unary-expr
        //   -- pp-equality-expr [whitespace] "!=" [whitespace] pp-unary-expr
        //
        // pp-unary-expr:
        //   -- pp-primary-expr
        //   -- "!" [whitespace] pp-unary-expr
        //
        // pp-primary-expr:
        //   -- "true"
        //   -- "false"
        //   -- conditional-symbol
        //   -- "(" [whitespace] pp-expr [whitespace] ")"
        // 
        // Operator order of precedence:
        //   !       <- highest
        //   == !=
        //   &&
        //   ||      <- lowest
        //

        // parse a top-level expression ending with TokenType.Tail
        bool unused;
        bool value = ParsePpAndOrExpression(ref tokenInDirective, symbols,
            directiveToken, out unused);
        if (!tokenInDirective.IsTail)
        {
            throw NewParsingException(directiveToken,
                Resources.BadPreprocessorExpressionSyntax);
        }
        else
            return value;
    }

    bool ParsePpAndOrExpression(ref Token tokenInDirective,
        Dictionary<string, bool> symbols, Token directiveToken, out bool isOr)
    {
        // skip white space, comments, etc.
        ParseInsignificantTokens(ref tokenInDirective, null);

        // parse an "equality expression"
        bool leftValue = ParsePpEqualityExpression(ref tokenInDirective,
            symbols, directiveToken);

        // skip white space, comments, etc.
        ParseInsignificantTokens(ref tokenInDirective, null);

        // check for "&&" or "||" operator
        if (tokenInDirective.IsCharacter('&') &&
            tokenInDirective.Next.IsCharacter('&'))
        {
            isOr = false;
        }
        else
        if (tokenInDirective.IsCharacter('|') &&
            tokenInDirective.Next.IsCharacter('|'))
        {
            isOr = true;
        }
        else
        {
            isOr = false;
            return leftValue;
        }

        // skip the "&&" or "||" operator
        tokenInDirective = tokenInDirective.Next.Next;

        // recursively parse the remainder of this "and/or expression"; this
        // results in the expression being evaluated right-to-left, which is
        // one way of forcing "&&" operators to be evaluated before "||"
        // operators (when you consider that all "||" operators must appear
        // before all "&&" operators)
        bool nextIsOr;
        bool rightValue = ParsePpAndOrExpression(ref tokenInDirective,
            symbols, directiveToken, out nextIsOr);

        // all "||" operators must come before all "&&" operators
        if (!isOr && nextIsOr)
        {
            throw NewParsingException(directiveToken,
                Resources.PreprocessorExprOrAfterAnd);
        }

        // calculate the result
        if (isOr)
            return leftValue || rightValue;
        else
            return leftValue && rightValue;
    }

    bool ParsePpEqualityExpression(ref Token tokenInDirective,
        Dictionary<string, bool> symbols, Token directiveToken)
    {
        // skip white space, comments, etc.
        ParseInsignificantTokens(ref tokenInDirective, null);

        // parse a "unary expression"; use the result as the initial value
        // of <totalValue>, which tracks the current value of the "equality
        // expression" as we parse it
        bool totalValue = ParsePpUnaryExpression(ref tokenInDirective, symbols,
            directiveToken);

        // loop once for each operator/expression pair
        while (true)
        {
            // check for "==" or "!=" operator
            bool isEquals;
            if (tokenInDirective.IsCharacter('=') &&
                tokenInDirective.Next.IsCharacter('='))
            {
                isEquals = true;
            }
            else
            if (tokenInDirective.IsCharacter('!') &&
                tokenInDirective.Next.IsCharacter('='))
            {
                isEquals = false;
            }
            else
                break;

            // skip the "==" or "!=" operator
            tokenInDirective = tokenInDirective.Next.Next;

            // skip white space, comments, etc.
            ParseInsignificantTokens(ref tokenInDirective, null);

            // parse another "unary expression"
            bool value = ParsePpUnaryExpression(ref tokenInDirective, symbols,
                directiveToken);

            // update <totalValue>
            if (isEquals)
                totalValue = (totalValue == value);
            else
                totalValue = (totalValue != value);
        }

        // done
        return totalValue;
    }

    bool ParsePpUnaryExpression(ref Token tokenInDirective,
        Dictionary<string, bool> symbols, Token directiveToken)
    {
        // parse "!" operators
        bool isNot = false;
        while (true)
        {
            // skip white space, comments, etc.
            ParseInsignificantTokens(ref tokenInDirective, null);

            // check for "!" operator
            if (tokenInDirective.IsCharacter('!') &&
                !tokenInDirective.Next.IsCharacter('='))
            {
                isNot = !isNot;
                tokenInDirective = tokenInDirective.Next;
            }
            else
                break;
        }

        // parse the "primary expression"
        bool value = ParsePpPrimaryExpression(ref tokenInDirective, symbols,
            directiveToken);

        // apply the "!" operator(s), if any
        if (isNot)
            value = !value;

        // done
        return value;
    }

    bool ParsePpPrimaryExpression(ref Token tokenInDirective,
        Dictionary<string, bool> symbols, Token directiveToken)
    {
        // skip white space, comments, etc.
        ParseInsignificantTokens(ref tokenInDirective, null);

        // look for the constant "true"
        if (tokenInDirective.IsReservedWord("true"))
        {
            tokenInDirective = tokenInDirective.Next;
            return true;
        }

        // look for the constant "false"
        if (tokenInDirective.IsReservedWord("false"))
        {
            tokenInDirective = tokenInDirective.Next;
            return false;
        }

        // look for a preprocessor symbol
        if (tokenInDirective.IsAWordOrReservedWord)
        {
            bool value;
            string symbol = tokenInDirective.Text;
            tokenInDirective = tokenInDirective.Next;
            if (!symbols.TryGetValue(symbol, out value))
                return false;
            else
                return value;
        }

        // look for a parenthesized expression
        if (tokenInDirective.IsCharacter(LParen))
        {
            // skip the <LParen>
            tokenInDirective = tokenInDirective.Next;

            // parse an "and/or expression"
            bool unused;
            bool value = ParsePpAndOrExpression(ref tokenInDirective, symbols,
                directiveToken, out unused);

            // skip white space, comments, etc.
            ParseInsignificantTokens(ref tokenInDirective, null);

            // we should now be at a <RParen>
            if (!tokenInDirective.IsCharacter(RParen))
            {
                throw NewParsingException(directiveToken,
                    Resources.BadPreprocessorExpressionSyntax);
            }

            // skip the <RParen>
            tokenInDirective = tokenInDirective.Next;

            // done
            return value;
        }

        // this isn't a valid "primary expression"
        throw NewParsingException(directiveToken,
            Resources.BadPreprocessorExpressionSyntax);
    }

    void LocateTopics(ref Token token, TokenList namespaceName,
        TokenList parentTypeName, Topic parentTopic, UsingContext usingContext)
    {
        while (token.TokenType != TokenType.Tail)
        {
            if (token.IsAReservedWord)
            {
                if (token.Text == "namespace")
                {
                    // remember this "namespace" token
                    Token namespaceToken = token;

                    // parse the namespace name into <childNamespaceName>
                    string childNamespaceName;
                    token = token.Next.SkipWhiteSpace();
                    if (token.TokenType != TokenType.Word)
                    {
                        throw NewParsingException(token,
                            Resources.BadNamespaceSyntax);
                    }
                    token = token.ReadDotDelimitedId(out childNamespaceName);
                    Debug.Assert((childNamespaceName == null) ||
                        (childNamespaceName.Length != 0));

                    // find and skip the left brace
                    token = token.SkipWhiteSpace();
                    if (!token.IsCharacter(LBrace))
                    {
                        throw NewParsingException(token,
                            Resources.ExpectLBraceAfterNamespace);
                    }
                    token = token.Next;

                    // error if we're within a class, struct or enum -- can't
                    // have a namespace within a type declaration
                    if (parentTypeName != null)
                    {
                        throw NewParsingException(token,
                            Resources.NamespaceWithinType, childNamespaceName,
                            parentTypeName);
                    }

                    // locate topics recursively within the namespace
                    TokenList qualifiedChildNamespaceName =
                        TokenList.Join(namespaceName, '.',
                            CSharpTokenParser.NewWordToken(childNamespaceName));
                    LocateTopics(ref token, qualifiedChildNamespaceName,
                        null, null, usingContext.Clone());
                }
                else
                if (token.Text == "using")
                {
#if true
                    // see if the next token is the beginning of a namespace
                    // name (e.g. "System.IO") or is a type or namespace alias
                    // (e.g. "RE" in "using RE = System.Text.RegularExpressions"
                    token = token.Next; // skip "using"
                    ParseInsignificantTokens(ref token, null);
                    if (!token.IsAWordOrReservedWord)
                    {
                        throw Expected(token,
                            Resources.ExpectedUsingAliasOrNamespaceOrType);
                    }
                    Token token2 = token.Next;
                    ParseInsignificantTokens(ref token2, null);
                    if (token2.IsCharacter('='))
                    {
                        // this is a type or namespace alias (e.g.
                        // (e.g. "using RE = System.Text.RegularExpressions;")
                        string alias = token.Text;
                        token = token2.Next;
                        ParseInsignificantTokens(ref token, null);
                        TokenList namespaceOrTypeName =
                            ParseType(ref token, null);
                        ParseInsignificantTokens(ref token, null);
                        if (!token.IsCharacter(';'))
                        {
                            throw Expected(token,
                                Resources.ExpectedSemicolonEndingUsing);
                        }
                        else
                            token = token.Next;
                        usingContext.AddAlias(alias, namespaceOrTypeName);
                    }
                    else
                    {
                        // this is an unaliased "using" namespace declaration
                        // (e.g. "using System.IO;")
                        TokenList namespaceOrTypeName =
                            ParseType(ref token, null);
                        ParseInsignificantTokens(ref token, null);
                        if (!token.IsCharacter(';'))
                        {
                            throw Expected(token,
                                Resources.ExpectedSemicolonEndingUsing);
                        }
                        else
                            token = token.Next;
                        usingContext.AddUnaliased(namespaceOrTypeName);
                    }
#else
                    // read a namespace name (e.g. "System.IO") or alias (e.g.
                    // "RE" in "using RE = System.Text.RegularExpressions"
                    token = token.Next; // skip "using"
                    ParseInsignificantTokens(ref token, null);
                    string id;
                    Token nextToken = token.ReadDotDelimitedId(out id);
                    if (nextToken == null)
                    {
                        throw Expected(token,
                            Resources.ExpectedUsingAliasOrNamespaceOrClass);
                    }
                    else
                        token = nextToken;

                    // the next character should be "=" (if <id> is an alias
                    // and the following token(s) contain the namespace or
                    // class name) or ";" (if <id> was a namespace or class
                    // name)
                    ParseInsignificantTokens(ref token, null);
                    if (token.IsCharacter('='))
                    {
                        string alias = id;
                        token = token.Next;
                        ParseInsignificantTokens(ref token, null);
                        string namespaceOrTypeName;
                        if ((nextToken = token.ReadDotDelimitedId(
                            out namespaceOrTypeName)) == null)
                        {
                            throw Expected(token,
                                Resources.ExpectedUsingNamespaceOrClass);
                        }
                        else
                            token = nextToken;
                        ParseInsignificantTokens(ref token, null);
                        if (!token.IsCharacter(';'))
                        {
                            throw Expected(token,
                                Resources.ExpectedSemicolonEndingUsing);
                        }
                        else
                            token = token.Next;
                        usingContext.AddAlias(alias, namespaceOrTypeName);
                    }
                    else
                    if (token.IsCharacter(';'))
                    {
                        usingContext.AddUnaliased(id);
                        token = token.Next;
                    }
#endif
                }
                else
                    token = token.Next;
            }
            else
            if (token.IsACharacter)
            {
                Debug.Assert(token.Text.Length == 1);
                if (token.Text[0] == LBrace)
                {
                    // this is a "{ ... }" section within the source code,
                    // such as the body of a method -- skip it
                    ParseGroup(ref token, null);
                }
                else
                if (token.Text[0] == RBrace)
                {
                    // skip <RBrace>, then end parsing
                    token = token.Next;
                    return;
                }
                else
                    token = token.Next;
            }
            else
            if (token.IsAnXmlComment)
            {
                // this is an XML comment, which may or may not be a topic --
                // if it is, extract the XML from the token and parse the C#
                // declaration that follows it; if it's a type that can contain
                // other types (e.g. class, struct, interface, enum) parse
                // the tokens within it
                if (!ParseTopicFromTokens(ref token, namespaceName,
                        parentTypeName, parentTopic, usingContext))
                    token = token.Next;
            }
            else
            if (token.IsTail)
            {
                // end of file
                return;
            }
            else
            {
                // some other token -- skip it
                token = token.Next;
            }
        }

        // hit the end of the file
        Debug.Assert(token.IsTail);
    }

    bool ParseTopicFromTokens(ref Token token, TokenList namespaceName,
        TokenList parentTypeName, Topic parentTopic, UsingContext usingContext)
    {
        // save the original value of <token> in case we need to return false
        Token origToken = token;

        // this should be an XML comment token -- specifically, the first of
        // one or more XML comments that precede the C# member declaration
        Debug.Assert(token.IsAnXmlComment);
        XmlCommentToken firstXmlComment = (XmlCommentToken) token;

        // create <topic> to hold information about the potential topic;
        // note that we're not yet certain this is a real topic
        Topic topic = new Topic(this);
        topic.DocumentationSet = DocumentationSet;
        topic.ParentTopic = parentTopic;
        topic.FirstXmlComment = firstXmlComment;
        topic.NamespaceName = namespaceName;
#if false
        topic.ParentTypeName = parentTypeName;
#endif
        topic.UsingContext = usingContext;

        // point <topic> to its parent NamespaceTopic; note that this may
        // cause a NamespaceTopic to be created that will never be used,
        // in the case where this isn't a valid topic
        topic.NamespaceTopic = DocumentationSet.FindOrAddNamespaceTopic(
            TokenList.ToString(namespaceName));

        // if there's no parent topic, the NamespaceTopic is the parent
        if (parentTopic == null)
            topic.ParentTopic = parentTopic = topic.NamespaceTopic;

        // extract the XML markup from this XML comment; advance <token> to the
        // start of the C# declaration
        topic.Xml = ExtractXmlFromComments(ref token);
        if (topic.Xml == null)
        {
            token = origToken;
            return false; // not a documentation topic
        }

        // do a preprocessing pass through the XML to make sure it's valid
        // XML documentation -- note that this doesn't do a thorough analysis,
        // just a few simple checks
        try
        {
            topic.PrevalidateXmlComment();
        }
        catch (XmlException ex)
        {
            topic.Xml = Resources.InvalidXmlCommentPlaceholder;
            ParsingException ex2 =
                NewParsingException(firstXmlComment, "{0}", ex.Message);
            FireInvalidXmlComment(ex2);
            if (ExceptionOnInvalidXmlComment)
                throw ex2;
        }
        catch (ParsingException ex)
        {
            topic.Xml = Resources.InvalidXmlCommentPlaceholder;
            FireInvalidXmlComment(ex);
            if (ExceptionOnInvalidXmlComment)
                throw;
        }

        // parse the C# code following this XML comment to figure out what
        // kind of topic this is (class, field, property, method, etc.)
        topic.StartOfDeclaration = token;
        if (!ParseDeclaration(ref token, topic))
        {
            token = origToken;
            Warning(origToken, Resources.NoDeclarationForXmlComment);
            return false; // not a documentation topic
        }

        // at this point we know this XML comment is a documentation topic...

        // add a reference from <firstXmlComment> to this topic
        firstXmlComment.Topic = topic;

        // add a reference from the parent topic to this topic
        parentTopic.AddChildTopic(topic);

        // add this topic to the list of topics in this source file
        TopicsList.Add(topic);

        return true;
    }

    string ExtractXmlFromComments(ref Token token)
    {
        /// <xmlBuffer> will contain the extracted XML
        StringBuilder xmlBuffer = new StringBuilder(token.Text.Length);

        // extract XML from <token> and all following XmlCommentToken objects
        while (true)
        {
            // skip white space, comments, etc., including DirectiveToken
            // and ExcludedToken objects but not including XmlCommentToken
            // objects
            ParseInsignificantTokens(ref token, null, null,
                ParseFlags.NoAttributes);

            // if this is not an XmlCommentToken, we're done
            if (!token.IsAnXmlComment)
                break;

            // extract the XML from <token>
            if (!ExtractXmlFromComments((XmlCommentToken) token, xmlBuffer))
            {
                // since this method's contract states that, on exit, <token>
                // will refer to the first token of the member declaration,
                // skip white space etc. here
                ParseInsignificantTokens(ref token, null, null,
                    ParseFlags.NoAttributes);
                break;
            }

            // advance to the next token
            token = token.Next;
        }

        // done
        return (xmlBuffer.Length == 0) ? null : xmlBuffer.ToString();
    }

    bool ExtractXmlFromComments(XmlCommentToken token, StringBuilder xmlBuffer)
    {
        // keep track of the number of characters stored in <xmlBuffer> on
        // entry so we know if we've written anything
        int initialBufferLength = xmlBuffer.Length;

        // split <token.Text> into lines, stored in <lines>
        string[] lines = s_lastNewline.Replace(token.Text, "").Split('\n');
        if (lines.Length == 0)
            return false;

        // parse the XML text from <lines> (removing the XML comment
        // delimiters) and store the result in <xmlBuffer>, taking care to
        // preserve the correct spacing at the beginning of each line
        if (token.IsTripleSlash)
        {
            // this is a "///"-style XML comment...

            // remove the leading "///" (and preceding white space, if any)
            // from each line in <lines>; set <prefix> to the prefix string
            // that's in common to all resulting lines; blank lines are ignored
            string prefix = null;
            for (int iLine = 0; iLine < lines.Length; iLine++)
            {
                string line = lines[iLine];
                lines[iLine] = line = s_tripleSlashRegex.Replace(line, "");
                if (prefix == null)
                    prefix = line;
                else
                if (!s_allBlankRegex.Match(line).Success)
                    prefix = FindCommonPrefix(line, prefix);
            }

            // set <prefixLength> to the length of the portion of <prefix>
            // containing spaces
            int prefixLength;
            if (prefix == null)
                prefixLength = 0;
            else
                prefixLength = s_spacesPrefixRegex.Match(prefix).Length;

            // copy <lines> to <xmlBuffer>, and remove <prefix> from each
            foreach (string line in lines)
            {
                if (s_lineOfSlashRegex.Match(line).Success)
                {
                    // line is a line of "/" characters, e.g. a separator --
                    // ignore it
                    continue;
                }
                else
                if (s_allBlankRegex.Match(line).Success)
                    xmlBuffer.Append(line);
                else
                    xmlBuffer.Append(line.Substring(prefixLength));
                xmlBuffer.Append("\r\n");
            }
        }
        else
        {
            // this is a "/**"-style XML comment...

            // <lineIndex> will refer to the first non-ignored line in <lines>;
            // <lineEndINdex> will refer to the line after the last non-ignored
            // line in <lines>
            int lineIndex = 0;
            int lineEndIndex = lines.Length;
            
            // remove the leading "/**"; if <lines> starts with "/**" without
            // any other text, ignore it
            lines[0] = s_slashStarStarRegex.Replace(lines[0], "");
            if (lines[0].Length == 0)
                lineIndex++;
            if (lineIndex >= lineEndIndex)
                return false; // empty XML comment

            // remove the trailing "*/"; if <lines> ends with "*/" without any
            // other text, ignore it
            lines[lineEndIndex - 1] =
                s_starSlashRegex.Replace(lines[lineEndIndex - 1], "");
            if (lines[lineEndIndex - 1].Length == 0)
                lineEndIndex--;

            // set <prefix> to the prefix string that's in common to all lines
            // starting from the second line; blank lines are ignored
            string prefix = null;
            for (int iLine = 1; iLine < lineEndIndex; iLine++)
            {
                string line = lines[iLine];
                if (prefix == null)
                    prefix = line;
                else
                if (!s_allBlankRegex.Match(line).Success)
                    prefix = FindCommonPrefix(line, prefix);
            }

            // set <prefixLength> to the length of the space-star-space prefix
            // at the beginning of <prefix>
            int prefixLength;
            if (prefix == null)
                prefixLength = 0;
            else
                prefixLength = s_spaceStarSpaceRegex.Match(prefix).Length;

            // copy <lines> to <xmlBuffer>, and remove <prefix> from each
            // starting at the second line
            for (int iLine = lineIndex; iLine < lineEndIndex; iLine++)
            {
                string line = lines[iLine];
                if (s_lineOfStarRegex.Match(line).Success)
                {
                    // line is a line of "*" characters, e.g. a separator --
                    // ignore it
                    continue;
                }
                else
                if ((iLine == 0) || s_allBlankRegex.Match(line).Success)
                    xmlBuffer.Append(line);
                else
                    xmlBuffer.Append(line.Substring(prefixLength));
                xmlBuffer.Append("\r\n");
            }
        }

        // return true if we extracted XML
        return (xmlBuffer.Length != initialBufferLength);
    }

    bool ParseDeclaration(ref Token token, Topic topic)
    {
        // Examples:
        //
        // In the examples below, "private" is a placeholder for any set of
        // modifiers, and "int" or "bar<T>" is a placeholder for any type.
        // Note that attributes, like [Description("whatever")] may precede the
        // declaration and need to be skipped.
        //
        // The following C# declaration types have keywords which help
        // identify them:
        //
        //   -- class:
        //          private class Foo { ... }
        //          private class Foo : ... { ... }
        //          private class Foo<T,U> where ... : ... { ... }
        //          private class Foo<T,U> : List<T> where T : IDisposable
        //              where U : struct { ... }
        //
        //   -- struct:
        //          Same as class, but with "struct" keyword.
        //
        //   -- interface:
        //          Same as class, but with "interface" keyword.
        //
        //   -- event:
        //          private event int Foo;
        //          private event bar<T> Foo;
        //
        //   -- delegate:
        //          private delegate bar<T> Foo(...);
        //          private delegate bar<T> Foo<T>(...) where ...;
        //
        //   -- enum:
        //          private enum Foo { ... }
        //          private enum Foo : int { ... }
        //
        // The following C# declaration types don't have clearly identifying
        // keywords and so are a bit trickier to parse:
        //
        //   -- enum values (within an enum):
        //          Foo, // comma is optional
        //          Foo = some value, // comma is optional
        //
        //   -- fields (within a type):
        //          private bar<T> Foo;
        //          const int Foo = 1;
        //          private bar<T> Foo = ..., Bar = ...;
        //              // not supported for XML comments
        //          private mydel Foo = delegate() { }, Bar = null;
        //
        //   -- properties (within a type):
        //          private bar<T> Foo { get { ... } } // (or set)
        //          public abstract bar<T> Foo { get; ... } // (or extern)
        //          private int this[...] { get { ... } ... } // indexer
        //
        //   -- methods (within a type):
        //          public abstract bar<T> Foo(); // (or extern)
        //          private bar<T> Foo(...) { ... }
        //          private bar<T> Foo<T> where ... (...) { ... }
        //          private bar<T> operator + (...) { ... }
        //          private Foo(...) { ... } // constructor
        //          private Foo(...) : base(...) { ... } // constructor
        //          private ~Foo() { ... }; // destructor
        //
        //  

        // Rules about all declaration parsing:
        //   -- White space, newlines, and comments are ignored.
        //   -- Attributes, like [Description("whatever")], are skipped if
        //      they precede <modifier-words> but incorporated as part of the
        //      declaration if they appear elsewhere (e.g. within parameters).
        //

        // parse attributes such as "[Foo(123), Bar(X=45, Y=67)]" and modifiers
        // such as "public static"
        topic.AttributesList = new List<TokenList>(10);
        topic.Modifiers = ParseModifiers(ref token, topic.AttributesList);

        // if this is a "class", "struct", or "interface" declaration, parse it
        if (token.IsAReservedWord)
        {
            if (token.Text == "class")
            {
                topic.TopicKind = TopicKind.ClassTopic;
                ParseClassStructInterface(ref token, topic);
                return true;
            }
            else
            if (token.Text == "struct")
            {
                topic.TopicKind = TopicKind.StructTopic;
                ParseClassStructInterface(ref token, topic);
                return true;
            }
            else
            if (token.Text == "interface")
            {
                topic.TopicKind = TopicKind.InterfaceTopic;
                ParseClassStructInterface(ref token, topic);
                return true;
            }
            else
            if (token.Text == "event")
            {
                topic.TopicKind = TopicKind.EventTopic;
                ParseEvent(token, topic);
                return true;
            }
            else
            if (token.Text == "delegate")
            {
                topic.TopicKind = TopicKind.DelegateTopic;
                ParseDelegate(token, topic);
                return true;
            }
            else
            if (token.Text == "enum")
            {
                topic.TopicKind = TopicKind.EnumTopic;
                ParseEnum(ref token, topic);
                return true;
            }
        }

        // see if this is an enum value (e.g. "Abc" within "enum Foo { Abc }"),
        // field, property, or method declaration
        return ParseOtherDeclaration(ref token, topic);
    }

    void ParseClassStructInterface(ref Token token, Topic topic)
    {
        // Examples:
        //   -- private class Foo { ... }
        //   -- private class Foo : ... { ... }
        //   -- private class Foo<T,U> where ... : ... { ... }
        //   -- private class Foo<T,U> : List<T> where T : IDisposable
        //      where U : struct { ... }

        // skip the "class", "struct", or "interface" token
        Debug.Assert(topic.TopicKindCSharpKeyword == token.Text);
        token = token.Next;

        // parse the member name (e.g. class name)
        if ((topic.MemberName = ParseWord(ref token)) == null)
        {
            throw Expected(token, Resources.KeywordName,
                topic.TopicKindCSharpKeyword);
        }

        // parse the type parameters, if any
        topic.TypeParameters = ParseTypeParameters(ref token, topic);

        // parse the base types, if any
        topic.BaseTypes = ParseBaseTypes(ref token, topic);

        // parse the "where" clause list, if any
        topic.Constraints = ParseConstraints(ref token, topic);

        // parse the <LBrace>
        ParseInsignificantTokens(ref token);
        if (!token.IsCharacter(LBrace))
        {
            throw Expected(token, Resources.ExpectedLBrace,
                topic.TopicKindCSharpKeyword,
                topic.GetDeclarationLineNumber());
        }
        token = token.Next;

        // add this topic to the list of type topics associated with the
        // namespace
        topic.NamespaceTopic.AddTypeTopic(topic);

        // locate topics recursively within this parent type
        TokenList qualifiedNewParentTypeName = Join(topic.ParentTypeName, '.',
            CSharpTokenParser.NewWordToken(topic.MemberName));
        LocateTopics(ref token, topic.NamespaceName,
            qualifiedNewParentTypeName, topic, topic.UsingContext);
    }

    void ParseEvent(Token token, Topic topic)
    {
        // Examples:
        //   -- private event int Foo;
        //   -- private event bar<T> Foo;

        // skip the "event" keyword
        Debug.Assert(topic.TopicKindCSharpKeyword == "event");
        Debug.Assert(token.IsReservedWord("event"));
        token = token.Next;

        // parse the member type
        if ((topic.MemberType = ParseType(ref token, topic)) == null)
        {
            throw Expected(token, Resources.ExpectedTypeInEventDecl,
                topic.GetDeclarationLineNumber());
        }

        // parse the member name (i.e. the name of the event)
        if ((topic.MemberName = ParseWord(ref token)) == null)
            throw Expected(token, Resources.EventName);
    }

    void ParseDelegate(Token token, Topic topic)
    {
        // Examples:
        //   -- private delegate bar<T> Foo(...);
        //   -- private delegate bar<T> Foo<T>(...) where ...;

        // skip the "delegate" keyword
        Debug.Assert(topic.TopicKindCSharpKeyword == "delegate");
        Debug.Assert(token.IsReservedWord("delegate"));
        token = token.Next;

        // parse the type
        if ((topic.MemberType = ParseType(ref token, topic)) == null)
        {
            throw Expected(token, Resources.ExpectedTypeInDelegateDecl,
                topic.GetDeclarationLineNumber());
        }

        // parse the member name (i.e. the name of the delegate)
        if ((topic.MemberName = ParseWord(ref token)) == null)
            throw Expected(token, "delegate name");

        // parse the type parameters, if any
        topic.TypeParameters = ParseTypeParameters(ref token, topic);

        // next should be a <LParen> that begins parameter list
        ParseInsignificantTokens(ref token);
        if (!token.IsCharacter(LParen))
        {
            throw Expected(token, Resources.ExpectedLParenInDelegateDecl,
                topic.GetDeclarationLineNumber());
        }

        // parse the parameter list
        topic.ParametersList = ParseParameters(ref token, topic);

        // parse the "where" clause list, if any
        topic.Constraints = ParseConstraints(ref token, topic);

        // next should be ";"
        ParseInsignificantTokens(ref token);
        if (!token.IsCharacter(';'))
        {
            throw Expected(token,
                Resources.ExpectedSemicolonEndingDelegateDecl,
                topic.GetDeclarationLineNumber());
        }

        // add this topic to the list of type topics associated with the
        // namespace
        topic.NamespaceTopic.AddTypeTopic(topic);
    }

    void ParseEnum(ref Token token, Topic topic)
    {
        // Example:
        //   -- private enum Foo { ... }
        //   -- private enum Foo : int { ... }

        // skip the "enum" keyword
        Debug.Assert(topic.TopicKindCSharpKeyword == "enum");
        Debug.Assert(token.IsReservedWord("enum"));
        token = token.Next;

        // parse the member name (i.e. the name of the enum)
        if ((topic.MemberName = ParseWord(ref token)) == null)
            throw Expected(token, Resources.EnumName);

        // parse the type, if any
        ParseInsignificantTokens(ref token);
        if (token.IsCharacter(':'))
        {
            token = token.Next;
            if ((topic.MemberType = ParseType(ref token, topic)) == null)
            {
                throw Expected(token, Resources.ExpectedTypeInEnumDecl,
                    topic.GetDeclarationLineNumber());
            }
        }

        // parse the <LBrace>
        ParseInsignificantTokens(ref token);
        if (!token.IsCharacter(LBrace))
        {
            throw Expected(token, Resources.ExpectedLBraceInEnumDecl,
                topic.GetDeclarationLineNumber());
        }
        token = token.Next;

        // add this topic to the list of type topics associated with the
        // namespace
        topic.NamespaceTopic.AddTypeTopic(topic);

        // parse the contents of the "enum { ... }", to get any documentation
        // topics within it; initialize <topic.EnumValueNamesList> to contain
        // enum value names
        topic.EnumValueNamesList = new List<string>();
        TokenList qualifiedNewParentTypeName = Join(topic.ParentTypeName, '.',
            CSharpTokenParser.NewWordToken(topic.MemberName));
        LocateTopics(ref token, topic.NamespaceName,
            qualifiedNewParentTypeName, topic, topic.UsingContext);
    }

    bool ParseOtherDeclaration(ref Token token, Topic topic)
    {
        // Examples:
        //
        //   -- enum values (within an enum):
        //          Foo, // comma is optional
        //          Foo = some value, // comma is optional
        //
        //   -- fields (within a type):
        //          <modifiers> <type> <name> [= <value>] , ... ;
        //      e.g.:
        //          private bar<T> Foo;
        //          const int Foo = 1;
        //          private bar<T> Foo = ..., Bar = ...;
        //              // not supported for XML comments
        //          private mydel Foo = delegate() { }, Bar = null;
        //
        //   -- properties (within a type):
        //          <modifiers> <type> <name> <square-bracket-parameters>
        //              { <modifiers> <get/set> <brace-group-or-semicolon>
        //                <modifiers> <get/set> <brace-group-or-semicolon> }
        //      e.g.:
        //          private bar<T> Foo { get { ... } }
        //          private bar<T> Foo { internal set { ... } }
        //          public abstract bar<T> Foo { get; ... } // (or extern)
        //          private int this[...] { get { ... } ... } // indexer
        //
        //   -- methods (within a type):
        //          <modifiers> <type> <name> <type-parameters> <where-clauses>
        //              <paren-argument-group-or-semicolon>
        //      special cases:
        //          -- operator: <name> is "operator" plus character(s)
        //          -- constructor: <name> is the same as the parent class name
        //          -- destructor: <name> is preceded by ~
        //      e.g.:
        //          public abstract bar<T> Foo(); // (or extern)
        //          private bar<T> Foo(...) { ... }
        //          private bar<T> Foo<T> where ... (...) { ... }
        //          private bar<T> operator + (...) { ... }
        //          private Foo(...) { ... } // constructor
        //          private Foo(...) : base(...) { ... } // constructor
        //          private ~Foo() { ... }; // destructor
        //
        // Parsing approach (after determining it's not an enum value);
        // note that <modifiers> is already parsed:
        //   -- parse <type>; special case:
        //        -- <type> is missing for constructors and destructors
        //           (see special cases below)
        //   -- parse <name>; special cases:
        //        -- operator: <name> is "operator" plus character(s)
        //        -- constructor: <name> is the same as the parent class name
        //        -- destructor: <name> is preceded by ~
        //   -- then, attempt to parse as a field list, property, or method;
        //      whichever succeeds, wins; if multiple succeed, throw an error
        //      ("ambiguous syntax -- could be any of the following: ...")
        //

        // see if this is an enum value (e.g. "Abc" within "enum Foo { Abc }")
        if ((topic.ParentTopic != null) &&
            (topic.ParentTopic.TopicKind == TopicKind.EnumTopic))
        {
            if (ParseEnumValueDeclaration(ref token, topic))
                return true;
        }

        // skip white space, comments, etc.
        ParseInsignificantTokens(ref token, null);

        // keep track of the first non-white-space token
        Token firstNonWhiteSpace = token;

        // see if this is a constructor declaration
        if ((topic.ParentTopic != null) &&
            (topic.ParentTopic.MemberName != null) &&
            token.IsWordOrReservedWord(topic.ParentTopic.MemberName))
        {
            // this declaration starts with a word that's the same as the
            // name of the class -- if it's followed by an <LParen> it's
            // presumably a constructor, otherwise it's probably just e.g.
            // a method that returns the same type as the parent class
            Token foundToken = FindSignificantToken(token.Next);
            if (foundToken.IsCharacter(LParen))
            {
                // this is a constructor declaration
                topic.MethodKind = MethodKind.Constructor;
                topic.MemberName = token.Text;
                token = foundToken;

                // parse the rest of the declaration
                return ParseMethodDeclaration(ref token, topic);
            }
        }

        // see if this is a destructor declaration
        if (token.IsCharacter('~'))
        {
            // this is a destructor declaration
            topic.MethodKind = MethodKind.Destructor;

            // expecting the next word to be the same as the class name
            token = token.Next;
            ParseInsignificantTokens(ref token, null);
            if ((topic.ParentTopic == null) ||
                (topic.ParentTopic.MemberName == null) ||
                !token.IsWordOrReservedWord(topic.ParentTopic.MemberName))
            {
                throw Unexpected(token, Resources.UnexpectedDestructorDecl);
            }
            topic.MemberName = "~" + token.Text;
            token = token.Next;

            // parse the rest of the declaration
            return ParseMethodDeclaration(ref token, topic);
        }

//xxx
        // parse the member type, unless this is e.g. 
        // "public static implicit operator Type1(Type2 value)"
        bool isOperator;
        if (token.IsReservedWord("operator"))
        {
            // this is an operator declaration (the kind that doesn't have a
            // member type)
            isOperator = true;
        }
        else
        {
            // parse the member type
            if ((topic.MemberType = ParseType(ref token, topic)) == null)
                return false; // we didn't find a C# declaration

            // skip white space, comments, etc.
            ParseInsignificantTokens(ref token, null);

            // see if this is an operator declaration (the kind that *does*
            // have a member type)
            isOperator = token.IsReservedWord("operator");
        }

        // see if this is an operator declaration; if so, set
        // <memberNameAlreadyFound> to it
        string memberNameAlreadyFound = null;
        if (isOperator)
        {
            // this is an operator declaration
            topic.MethodKind = MethodKind.Operator;

            // skip the token "operator" and following white space etc.
            token = token.Next;
            ParseInsignificantTokens(ref token, null);

            if (token.IsCharacter(s_operatorSymbolCharacters))
            {
                StringBuilder memberNameBuilder = new StringBuilder(10);
                while (true)
                {
                    memberNameBuilder.Append(token.Text);
                    token = token.Next;
                    if (!token.IsCharacter(s_operatorSymbolCharacters))
                        break;
                }
                memberNameAlreadyFound = "operator " +
                    memberNameBuilder.ToString();
            }

#if false
            // parse the operator name
#if false
            if (token.IsReservedWord("true"))
            {
                topic.MemberName = "operator true";
                token = token.Next;
            }
            else
            if (token.IsReservedWord("false"))
            {
                topic.MemberName = "operator false";
                token = token.Next;
            }
            else
#endif
            if (token.IsAWordOrReservedWord)
            {
                // e.g. "public static implicit operator Type1(Type2 value)" --
                // <token> is "Type1" in this example
                topic.MemberName = "operator " + token.Text;
                token = token.Next;
            }
            else
            {
                StringBuilder memberNameBuilder = new StringBuilder(10);
                while (token.IsCharacter('+', '-', '!', '~', '*', '/', '%',
                        '&', '|', '^', '<', '>', '='))
                {
                    memberNameBuilder.Append(token.Text);
                    token = token.Next;
                }
                if (memberNameBuilder.Length == 0)
                {
                    throw Expected(token, Resources.ExpectedOperatorName,
                        topic.GetDeclarationLineNumber());
                }
                topic.MemberName = "operator " + memberNameBuilder.ToString();
            }

            // parse the rest of the declaration
            return ParseMethodDeclaration(ref token, topic);
#endif
        }
        else
        if (token.IsReservedWord("this"))
        {
            // this is an indexer declaration
            topic.PropertyKind = PropertyKind.Indexer;
            topic.MemberName = "Item";

            // parse the rest of the declaration
            token = token.Next;
            return ParsePropertyDeclaration(ref token, topic);
        }
        else
        {
            // this is a "normal" method, property, etc. declaration
        }

        // set <topic.MemberName> to the member name
        if (memberNameAlreadyFound != null)
        {
            // we already found the member name
            topic.MemberName = memberNameAlreadyFound;
        }
        else
        {
            // read the member name (which may be multiple words, e.g. method
            // "IDisposable.Dispose" or "global::IMyDictionary<int,T>.Add"
            if (!token.IsAWordOrReservedWord)
                return false; // we didn't find a C# declaration
            TokenList memberName = new TokenList();
            memberName.Append(token.Clone());
            token = token.Next;
            while (true)
            {
                // skip white space, comments, etc.
                ParseInsignificantTokens(ref token, null);

                // If we hit a "<...>" group, parse type parameters and then
                // break.  BUT, consider the following two examples
                //   IMyList<T>.Foo() // method name "IMyList<T>.Foo"
                //   Bar<T>()         // method name "Bar", type params "<T>"
                // In the first case, "<T>" is treated as part of the member
                // name; in the second case, "<T>" is treated as type
                // parameters following the member name.  To distinguish
                // between these two cases, we look for a '.' following the
                // type parameters
                //
                if (token.IsCharacter(LAngle))
                {
                    Token token2 = token;
                    TokenList[] typeParameters =
                        ParseTypeParameters(ref token2, topic);
                    ParseInsignificantTokens(ref token2, null);
                    if (!token2.IsCharacter('.'))
                        break; // second case above
                    token = token2;
                    if (typeParameters != null)
                    {
                        memberName.Append(new Token(LAngle));
                        int count = 0;
                        foreach (TokenList typeParameter in typeParameters)
                        {
                            if (count++ > 0)
                                memberName.Append(new Token(','));
                            memberName.Append(typeParameter);
                        }
                        memberName.Append(new Token(RAngle));
                    }
                }

                // stop if the this token isn't "." or "::" (the latter being
                // two tokens, actually
                if (token.IsCharacter('.'))
                {
                    memberName.Append(token.Clone());
                    token = token.Next;
                }
                else
                if (token.IsCharacter(':') && token.Next.IsCharacter(':'))
                {
                    memberName.Append(token.Clone());
                    memberName.Append(token.Next.Clone());
                    token = token.Next.Next;
                }
                else
                    break;

                // skip white space, comments, etc.
                ParseInsignificantTokens(ref token, null);

                // see if this is an indexer declaration
                if (token.IsReservedWord("this"))
                {
                    // this is an indexer declaration
                    topic.PropertyKind = PropertyKind.Indexer;
                    memberName.Append(new Token(TokenType.ReservedWord,
                        "Item"));
                    token = token.Next;
                    break;
                }

                // if the next token is something other than a word or a
                // reserved word, this isn't a member name
                if (!token.IsAWordOrReservedWord)
                    return false; // we didn't find a C# declaration

                // assume this word is part of the type
                memberName.Append(token.Clone());
                token = token.Next;
            }

            // divide <memberName> into Topic.ExplicitPrefix and
            // Topic.MemberName, unless this is an operator (since that would
            // result in a member name like "Abc.Def.operator Ghi" instead of
            // "operator Abc.Def.Ghi")
            if (isOperator)
                topic.MemberName = "operator " + memberName;
            else
            {
                topic.MemberName = memberName.TailToken.Previous.ToString();
                topic.MemberName = memberName.TailToken.Previous.ToString();
                memberName.TailToken.Previous.RemoveFromList();
                if (!memberName.IsEmpty)
                    topic.ExplicitPrefix = memberName;
            }
        }

        // if this is an operator, we know we're dealing with a method
        if (isOperator)
            return ParseMethodDeclaration(ref token, topic);

        // at this point we're not sure if the declaration is a field,
        // property, or method; so, we attempt to parse it as each, and see
        // which succeeds (if multiple succeed, we'll throw an error indicating
        // ambiguous syntax); in each case, we make a shallow copy of <topic>
        // so that changes to the shallow state of the Topic within
        // Parse*Declaration() don't "pollute" the others

        // try parsing as a method
        Token methodToken = token;
        Topic methodTopic = topic.MemberwiseClone();
        methodTopic.MethodKind =
            (topic.MethodKind == MethodKind.NotAMethod) ?
                MethodKind.Normal : topic.MethodKind;
        bool matchedMethod = ParseMethodDeclaration(ref methodToken,
            methodTopic);

        // try parsing as a property
        Token propertyToken = token;
        Topic propertyTopic = topic.MemberwiseClone();
        propertyTopic.PropertyKind =
            (topic.PropertyKind == PropertyKind.NotAProperty) ?
                PropertyKind.Normal : topic.PropertyKind;
        bool matchedProperty = ParsePropertyDeclaration(ref propertyToken,
            propertyTopic);

        // try parsing as a field
        Token fieldToken = token;
        Topic fieldTopic = topic.MemberwiseClone();
        bool matchedField = ParseFieldDeclaration(ref fieldToken, fieldTopic);

        // see what succeeded
        int matches = (matchedMethod ? 1 : 0) + (matchedProperty ? 1 : 0) +
            (matchedField ? 1 : 0);
        if (matches == 0)
            return false; // can't parse declaration
        else
        if (matches > 1)
        {
            List<string> candidates = new List<string>(3);
            if (matchedMethod)
                candidates.Add("method");
            if (matchedProperty)
                candidates.Add("property");
            if (matchedField)
                candidates.Add("field");
            Debug.Assert(candidates.Count > 1);
            Warning(topic.StartOfDeclaration, Resources.AmbiguousDecl,
                String.Join(Resources.CommaSpace, candidates.ToArray()));
            return false; // can't parse declaration
        }

        // set <topic> to the victor, and <token> to the Token reference
        // returned by the victor
        bool copied = false; // double-check that there's only one victor
        if (matchedMethod)
        {
            Debug.Assert(!copied);
            PartialShallowTopicCopy(methodTopic, topic);
            token = methodToken;
            copied = true;
        }
        if (matchedProperty)
        {
            Debug.Assert(!copied);
            PartialShallowTopicCopy(propertyTopic, topic);
            token = propertyToken;
            copied = true;
        }
        if (matchedField)
        {
            Debug.Assert(!copied);
            PartialShallowTopicCopy(fieldTopic, topic);
            token = fieldToken;
            copied = true;
        }

        return true;
    }

    void PartialShallowTopicCopy(Topic fromTopic, Topic toTopic)
    {
        // copy shallow state
        toTopic.TopicKind       = fromTopic.TopicKind;
        toTopic.MethodKind      = fromTopic.MethodKind;
        toTopic.PropertyKind    = fromTopic.PropertyKind;
        toTopic.TypeParameters  = fromTopic.TypeParameters;
        toTopic.Constraints     = fromTopic.Constraints;
        toTopic.ParametersList  = fromTopic.ParametersList;
        toTopic.GetAccessor     = fromTopic.GetAccessor;
        toTopic.SetAccessor     = fromTopic.SetAccessor;

        // verify the copy -- if the following assertion fails, it's likely
        // due to the assignment statements above being incomplete, or
        // a Parse*Declaration() method call disobeying the part of its
        // contract that says "This method may only modify state of <topic>
        // copied by PartialShallowTopicCopy"
#if DEBUG
        string fromDump = fromTopic.Dump(int.MaxValue);
        string toDump = toTopic.Dump(int.MaxValue);
        if (fromDump != toDump)
        {
#if false
            Console.WriteLine("fromDump ---\n{0}\n---", fromDump);
            Console.WriteLine("toDump ---\n{0}\n---", toDump);
#endif
            Debug.Assert(false);
        }
#endif
    }

    bool ParseMethodDeclaration(ref Token token, Topic topic)
    {
        // Method syntax (valid only within a type declaration):
        //
        //      <modifiers> <type> <name> <type-parameters> <where-clauses>
        //          <paren-argument-group-or-semicolon>
        //      special cases:
        //          -- operator: <name> is "operator" plus character(s)
        //          -- constructor: <name> is the same as the parent class name
        //          -- destructor: <name> is preceded by ~
        //
        // Examples:
        //      public abstract bar<T> Foo(); // (or extern)
        //      private bar<T> Foo(...) { ... }
        //      private bar<T> Foo<T> where ... (...) { ... }
        //      private bar<T> operator + (...) { ... }
        //      private Foo(...) { ... } // constructor
        //      private Foo(...) : base(...) { ... } // constructor
        //      private ~Foo() { ... }; // destructor
        //

        // skip white space, comments, etc.
        ParseInsignificantTokens(ref token, null);

        // parse the type parameters, if any
        topic.TypeParameters = ParseTypeParameters(ref token, topic);

        // if the next token isn't an <LParen> (i.e. start of the parameter
        // list), this isn't a method declaration
        ParseInsignificantTokens(ref token, null);
        if (!token.IsCharacter(LParen))
            return false; // not a method declaration

        // parse the parameter list
        topic.ParametersList = ParseParameters(ref token, topic);

        // parse the "where" clause list, if any
        topic.Constraints = ParseConstraints(ref token, topic);

        // if the next token isn't an <LBrace> (i.e. start of the method code
        // block) or semicolon (in the case of an abstract method), or a colon
        // (i.e. a constructor base, e.g. "MyClass() : base(123) {...}"), this
        // isn't a method declaration
        ParseInsignificantTokens(ref token, null);
        if (!token.IsCharacter(LBrace, ';', ':'))
            return false; // not a method declaration

        // this declaration matches the syntax of a method declaration
        topic.TopicKind = TopicKind.MethodTopic;
        return true;
    }

    bool ParsePropertyDeclaration(ref Token token, Topic topic)
    {
        // Property Syntax (valid only within a type declaration):
        //
        //      <modifiers> <type> <name> <square-bracket-parameters>
        //          { <modifiers> <get/set> <brace-group-or-semicolon>
        //            <modifiers> <get/set> <brace-group-or-semicolon> }
        //
        // Examples:
        //      private bar<T> Foo { get { ... } }
        //      private bar<T> Foo { internal set { ... } }
        //      public abstract bar<T> Foo { get; ... } // (or extern)
        //      private int this[...] { get { ... } ... } // indexer
        //

        // skip white space, comments, etc.
        ParseInsignificantTokens(ref token, null, null,
            ParseFlags.NoAttributes);

        // do special processing required for indexers
        if (topic.PropertyKind == PropertyKind.Indexer)
        {
            // if the next token isn't an <LSquare> (i.e. start of the parameter
            // list), this isn't a property declaration
            if (!token.IsCharacter(LSquare))
                return false; // not a property declaration

            // parse the parameter list
            topic.ParametersList = ParseParameters(ref token, topic);

            // skip white space, comments, etc.
            ParseInsignificantTokens(ref token, null);
        }

        // if the next token isn't an <LBrace> (i.e. start of the property
        // accessor block), this isn't a property declaration
        ParseInsignificantTokens(ref token, null);
        if (!token.IsCharacter(LBrace))
            return false; // not a property declaration
        token = token.Next;

        // loop once for each accessor ("get" or "set") within the property
        // declaration
        while (true)
        {
            // create an Accessor object to hold information about this
            // accessor
            Accessor accessor = new Accessor();

            // parse modifiers such as "internal"
            accessor.AccessorModifiers = ParseModifiers(ref token, null);

            // the next token determines if this is a "get" or "set" accessor;
            // if the token isn't "get" or "set" then this isn't a property
            // declaration
            ParseInsignificantTokens(ref token, null);
            if (token.IsReservedWord("get"))
            {
                accessor.AccessorKind = AccessorKind.Get;
                topic.GetAccessor = accessor;
            }
            else
            if (token.IsReservedWord("set"))
            {
                accessor.AccessorKind = AccessorKind.Set;
                topic.SetAccessor = accessor;
            }
            else
                return false; // not a property declaration
            token = token.Next;

            // skip the code block "{...}" or semicolon ";" following the "get"
            // or "set" keyword
            ParseInsignificantTokens(ref token, null);
            if (token.IsCharacter(';'))
                token = token.Next;
            else
            if (token.IsCharacter(LBrace))
                ParseGroup(ref token, null);
            else
                return false; // not a property declaration

            // if we reached an <RBrace>, we're done with the accessor block
            ParseInsignificantTokens(ref token, null);
            if (token.IsCharacter(RBrace))
            {
                token = token.Next;
                break;
            }
        }

        // this declaration matches the syntax of a property declaration
        topic.TopicKind = TopicKind.PropertyTopic;
        return true;
    }

    bool ParseFieldDeclaration(ref Token token, Topic topic)
    {
        // Field syntax (valid only within a type declaration):
        //
        //      <modifiers> <type> <name> [= <value>] , ... ;
        //
        // Examples:
        //
        //      private bar<T> Foo;
        //      const int Foo = 1;
        //      private bar<T> Foo = ..., Bar = ...;
        //          // not supported for XML comments
        //      private mydel Foo = delegate() { }, Bar = null;
        //

        // skip white space, comments, etc.
        ParseInsignificantTokens(ref token, null);

        // if the next token is ';', this is a field declaration that has no
        // initializer
        if (token.IsCharacter(';'))
        {
            topic.TopicKind = TopicKind.FieldTopic;
            token = token.Next;
            return true;
        }

        // if the next token is not '=', this is not a field declaration
        if (!token.IsCharacter('='))
            return false; // not a field declaration

        // this declaration matches the syntax of a field declaration that
        // has an initializer
        topic.TopicKind = TopicKind.FieldTopic;

        // skip the rest of the field declaration -- everything until a ';'
        // (but ignoring semicolons in groups like "delegate() { return; }")
        while (true)
        {
            if (token.IsTail)
                break;
            else
            if (token.IsCharacter(';'))
            {
                token = token.Next;
                break;
            }
            else
            if (token.IsCharacter(LBrace, LParen, LSquare, LAngle))
                ParseGroup(ref token, null);
            else
                token = token.Next;
        }

        return true;
    }

    bool ParseEnumValueDeclaration(ref Token token, Topic topic)
    {
        // parse the member name (i.e. the name of the enum value)
        if (!token.IsAWord)
            return false; // not an enum value declaration
        if ((topic.MemberName = ParseWord(ref token)) == null)
            return false; // not an enum value declaration

        // at this point we're going to assume we have a valid enum value...

        // update <topic>
        topic.TopicKind = TopicKind.EnumValueTopic;
        topic.NamespaceName = topic.ParentTopic.NamespaceName;
        topic.MemberType = topic.ParentTopic.QualifiedMemberName;

        // add the value name to <topic.EnumValueNamesList>
        topic.ParentTopic.EnumValueNamesList.Add(topic.MemberName);

        return true;
    }

    TokenList ParseModifiers(ref Token token, List<TokenList> attributesList)
    {
        TokenList buffer = new TokenList();
        while (true)
        {
            ParseInsignificantTokens(ref token, buffer, attributesList,
                (ParseFlags) 0);
            if (!token.IsAWordOrReservedWord)
                break;
            bool unused;
            if (!CSharpTokenParser.Modifiers.TryGetValue(
                    token.Text, out unused))
                break;
            buffer.Append(token.Clone());
            token = token.Next;
        }
        return buffer.TrimBlankTokens();
    }

    TokenList ParseType(ref Token token, Topic topic)
    {
        // Examples:
        //   -- int // reserved word
        //   -- AppDomain // not a reserved word
        //   -- List<int>
        //   -- int?
        //   -- string[]
        //   -- string[,]
        //   -- string[][]
        //   -- System.Collections.Generic.List<int>
        //   -- global::System.String

        // skip insignificant tokens, unless this isn't a type declaration
        Token foundToken = FindSignificantToken(token);
        if (!IsTypeWord(foundToken))
            return null; // no type declaration
        token = foundToken;

        // this is the first word of the type
        TokenList buffer = new TokenList();
        buffer.Append(token.Clone());
        token = token.Next;

        // parse the rest of the type parameter declaration, of the form
        // ".<word>.<word>..." or "::<word>.<word>..." stopping at type
        // parameters (which are included in the type) or at something other
        // than "." or "::"
        while (true)
        {
            // skip white space, comments, etc. -- note that "[]" needs to be
            // treated as an array declaration, not an attribute
            ParseInsignificantTokens(ref token, buffer, null,
                ParseFlags.NoAttributes);

            // if we hit a "<...>" group, parse type parameters and then break
            if (token.IsCharacter(LAngle))
            {
                TokenList[] typeParameters =
                    ParseTypeParameters(ref token, topic);
                if (typeParameters != null)
                {
                    buffer.Append(new Token(LAngle));
                    int count = 0;
                    foreach (TokenList typeParameter in typeParameters)
                    {
                        if (count++ > 0)
                            buffer.Append(new Token(','));
                        buffer.Append(typeParameter);
                    }
                    buffer.Append(new Token(RAngle));
                }
            }

            // stop if the this token isn't "." or "::" (the latter being two
            // tokens, actually
            if (token.IsCharacter('.'))
            {
                buffer.Append(token.Clone());
                token = token.Next;
            }
            else
            if (token.IsCharacter(':') && token.Next.IsCharacter(':'))
            {
                buffer.Append(token.Clone());
                buffer.Append(token.Next.Clone());
                token = token.Next.Next;
            }
            else
                break;

            // skip white space, comments, etc.
            ParseInsignificantTokens(ref token, buffer, null,
                ParseFlags.NoAttributes);

            // error if the next token is something other than a word or a
            // reserved word
            if (!IsTypeWord(token))
            {
                if (topic == null)
                    throw Expected(token, Resources.ExpectedType);
                else
                {
                    throw Expected(token, Resources.ExpectedTypeOfDecl,
                        topic.TopicKindCSharpKeyword,
                        topic.GetDeclarationLineNumber());
                }
            }

            // assume this word is part of the type
            buffer.Append(token.Clone());
            token = token.Next;
        }

        // if "?" appears at this point, it's a nullable type
        ParseInsignificantTokens(ref token, buffer, null,
            ParseFlags.NoAttributes);
        if (token.IsCharacter('?'))
        {
            buffer.Append(token.Clone());
            token = token.Next;
        }

        // if an array square bracket construct (e.g. "[]" or "[,]" or
        // "[][,,]" etc.) appears at this point, it's an array declaration,
        // i.e.  part of the type
        Token token2 = token;
        TokenList squareBrackets = ParseArraySquareBrackets(ref token2);
        if (squareBrackets != null)
        {
            buffer.Append(squareBrackets);
            token = token2;
        }

        // return the result, trimming leading and trailing white space tokens
        return buffer.TrimBlankTokens();
    }

    bool IsTypeWord(Token token)
    {
        // check if <token> is a non-reserved word (e.g. "abc")
        if (token.IsAWord)
            return true;

        // check if <token> is a word that's a C# type (e.g. "int")
        if (token.IsAReservedWord)
        {
            bool unused;
            if (CSharpTokenParser.ReservedTypes.TryGetValue(token.Text,
                    out unused))
                return true;
        }

        // <token> is not a C# type word
        return false;
    }

    TokenList ParseArraySquareBrackets(ref Token token)
    {
        TokenList buffer = new TokenList();
        while (true)
        {
            // skip white space, comments, etc. -- but don't treat square
            // brackets as attributes
            ParseInsignificantTokens(ref token, buffer, null,
                ParseFlags.NoAttributes);

            // parse a square brackets construct (e.g. "[]" or "[,,]"), if any
            if (!token.IsCharacter(LSquare))
                break;
            ParseGroup(ref token, buffer);
            if (!token.Previous.IsCharacter(RSquare))
                return null;
        }

        return buffer.TrimBlankTokens();
    }

    TokenList[] ParseTypeParameters(ref Token token, Topic topic)
    {
        // skip insignificant tokens, unless this isn't a type parameter
        // declaration
        Token startToken = FindSignificantToken(token);
        if (!startToken.IsCharacter(LAngle))
            return null; // no type parameters
        token = startToken.Next;

        // parse the type parameter declaration
        List<TokenList> typeParameters = new List<TokenList>(10);
        while (true)
        {
            // the next token should be a type parameter or argument --
            // parse it as a type
            TokenList typeParameter = ParseType(ref token, topic);
            if (typeParameter == null)
            {
                throw Expected(token, Resources.ExpectedTypeParam,
                    startToken.GetLineNumber());
            }
            else
                typeParameters.Add(typeParameter);

            // skip white space, comments, etc.
            ParseInsignificantTokens(ref token, null, null,
                ParseFlags.NoAttributes);

            // see if we hit the end of the type parameter list
            if (token.IsCharacter(RAngle))
                break;

            // error if the next token is not a comma
            if (!token.IsCharacter(','))
            {
                throw Expected(token, Resources.ExpectedCommaInTypeParamDecl,
                    ((topic == null) ? token.GetLineNumber()
                        : topic.GetDeclarationLineNumber()));
            }
            else
                token = token.Next;
        }

        // skip the closing <RAngle>
        Debug.Assert(token.IsCharacter(RAngle));
        token = token.Next;

        // done
        return typeParameters.ToArray();
    }

    TokenList ParseConstraints(ref Token token, Topic topic)
    {
        // skip insignificant tokens, unless this isn't a type parameter
        // declaration
        Token foundToken = FindSignificantToken(token);
        if (!foundToken.IsReservedWord("where"))
            return null; // no constraints
        token = foundToken;

        // parse the where clause list -- it's everything until an <LBrace>
        TokenList buffer = new TokenList();
        while (true)
        {
            // skip white space, comments, etc.
            ParseInsignificantTokens(ref token, buffer);

            // we're done if the next token is <LBrace> or ';'
            if (token.IsCharacter(LBrace, ';'))
                break;

            // error if the next token is the end of the source file or
            // <RBrace>
            if (token.IsTail || token.IsCharacter(RBrace))
            {
                throw Expected(token, Resources.ExpectedLBraceInConstraint,
                    topic.TopicKindCSharpKeyword,
                    topic.GetDeclarationLineNumber());
            }

            // add this token to the base type token list
            buffer.Append(token.Clone());

            // continue parsing
            token = token.Next;
        }

        // done
        return buffer.TrimBlankTokens();
    }

    TokenList[] ParseBaseTypes(ref Token token, Topic topic)
    {
        // skip insignificant tokens, unless this isn't a type parameter
        // declaration
        Token foundToken = FindSignificantToken(token);
        if (!foundToken.IsCharacter(':'))
            return null; // no base type
        token = foundToken.Next;

        // parse the list of base types
        List<TokenList> result = new List<TokenList>(10);
        while (true)
        {
            // parse the base type
            TokenList baseType;
            if ((baseType = ParseType(ref token, topic)) == null)
            {
                throw Expected(token, Resources.ExpectedBaseType,
                    topic.GetDeclarationLineNumber());
            }
            result.Add(baseType);

            // skip white space, comments, etc.
            ParseInsignificantTokens(ref token, null, null,
                ParseFlags.NoAttributes);

            // we're done if the next token is "where" or <LBrace>
            if (token.IsReservedWord("where") || token.IsCharacter(LBrace))
                break;

            // error if the next token isn't a comma
            if (!token.IsCharacter(','))
            {
                throw Expected(token, Resources.ExpectedNextBaseType,
                    topic.GetDeclarationLineNumber());
            }

            // skip the comma and continue parsing
            token = token.Next;
        }

        return result.ToArray();
    }

    List<Parameter> ParseParameters(ref Token token, Topic topic)
    {
        // remember what kind of group we're in: "(...)" or "[...]"
        if (token.TokenType != TokenType.Character)
            throw new ArgumentException("token");
        Debug.Assert(token.Text.Length == 1);
        char startTokenChar = token.Text[0], endTokenChar;
        if (token.TextIs(LParen))
            endTokenChar = RParen;
        else
        if (token.TextIs(LSquare))
            endTokenChar = RSquare;
        else
            throw new ArgumentException("token");

        // advance past the start-of-group token without using
        // ParseInsignificantTokens(),
        // because this token may be <LSquare> and ParseInsignificantTokens()
        // considers that to be "insignificant"; remember the location of
        // the parameter list for error messages
        Token parameterListStart = token;
        token = token.Next;

        // parse parameters until we hit <endTokenChar>
        List<Parameter> parameters = new List<Parameter>(10);
        while (true)
        {
            // skip white space, comments, etc.
            ParseInsignificantTokens(ref token, null);

            // if we hit <endTokenChar> we're at the end of the parameter list
            if (token.IsCharacter(endTokenChar))
                break;

            // <parameter> will hold informatin about the current parameter
            Parameter parameter = new Parameter();

            // error if we hit <LBrace>, <RBrace>, or the end of the source
            // file
            if (token.IsCharacter(LBrace, RBrace) || token.IsTail)
            {
                throw Expected(token, Resources.ExpectedEndOfParamList,
                    parameterListStart.GetLineNumber());
            }

            // check for the keyword "out" or "ref"
            if (token.IsReservedWord("out"))
            {
                parameter.ParameterKind = ParameterKind.Out;
                token = token.Next;
            }
            else
            if (token.IsReservedWord("ref"))
            {
                parameter.ParameterKind = ParameterKind.Ref;
                token = token.Next;
            }
            else
            if (token.IsReservedWord("params"))
            {
                parameter.ParameterKind = ParameterKind.Params;
                token = token.Next;
            }
            else
                parameter.ParameterKind = ParameterKind.Normal;

            // parse the parameter type
            if ((parameter.ParameterType = ParseType(ref token, topic))
                == null)
            {
                throw Expected(token, Resources.ExpectedTypeOfParamInParamList,
                    parameterListStart.GetLineNumber());
            }

            // parse the parameter name
            ParseInsignificantTokens(ref token, null);
            if ((parameter.ParameterName =
                ParseWordOrReservedWord(ref token)) == null)
            {
                throw Expected(token, Resources.ExpectedNameOfParamInParamList,
                    parameterListStart.GetLineNumber());
            }

            // the next token should be either ',' or <endTokenChar>; if it's
            // a comma, advance past it
            ParseInsignificantTokens(ref token, null);
            if (!token.IsCharacter(',', endTokenChar))
            {
                throw Expected(token,
                    Resources.ExpectedCommaOrDelimInParamList,
                    endTokenChar, parameterListStart.GetLineNumber());
            }
            if (token.IsCharacter(','))
                token = token.Next;

            // add this parameter to the list
            parameters.Add(parameter);
        }

        // skip past <endTokenChar>
        Debug.Assert(token.IsCharacter(endTokenChar));
        token = token.Next;

        return parameters;
    }

    void ParseGroup(ref Token token, TokenList buffer)
    {
        // remember what kind of group we're in (e.g. "<...>", "(...)", etc.)
        if (token.TokenType != TokenType.Character)
            throw new ArgumentException("token");
        Debug.Assert(token.Text.Length == 1);
        char startTokenChar = token.Text[0], endTokenChar;
        if (token.TextIs(LBrace))
            endTokenChar = RBrace;
        else
        if (token.TextIs(LParen))
            endTokenChar = RParen;
        else
        if (token.TextIs(LSquare))
            endTokenChar = RSquare;
        else
        if (token.TextIs(LAngle))
            endTokenChar = RAngle;
        else
            throw new ArgumentException("token");

        // advance past the start-of-group token without using
        // ParseInsignificantTokens(),
        // because this token may be <LSquare> and ParseInsignificantTokens()
        // considers that to be "insignificant"
        if (buffer != null)
            buffer.Append(token.Clone());
        token = token.Next;

        // parse the group
        while (true)
        {
            // skip white space, comments, attributes, etc.
            ParseInsignificantTokens(ref token, buffer);
            if (token.IsTail)
                return;
            else
            if (token.IsCharacter(startTokenChar))
                ParseGroup(ref token, buffer); // recurse
            else
            {
                bool atEndOfGroup = token.IsCharacter(endTokenChar);
                if (buffer != null)
                    buffer.Append(token.Clone());
                token = token.Next;
                if (atEndOfGroup)
                    return;
            }
        }
    }

    string ParseWord(ref Token token)
    {
        // find the first significant token after <token>
        Token foundToken = FindSignificantToken(token);
        if (!foundToken.IsAWord)
            return null; // non-reserved word not found
        token = foundToken;

        // return the word
        string word = token.Text;
        token = token.Next;
        return word;
    }

    string ParseWordOrReservedWord(ref Token token)
    {
        // find the first significant token after <token>
        Token foundToken = FindSignificantToken(token);
        if (!foundToken.IsAWordOrReservedWord)
            return null; // word not found
        token = foundToken;

        // return the word
        string word = token.Text;
        token = token.Next;
        return word;
    }

    Token FindSignificantToken(Token token)
    {
        ParseInsignificantTokens(ref token);
        return token;
    }

    bool ParseInsignificantTokens(ref Token token)
    {
        return ParseInsignificantTokens(ref token, null, null, (ParseFlags) 0);
    }

    bool ParseInsignificantTokens(ref Token token, TokenList buffer)
    {
        return ParseInsignificantTokens(ref token, buffer, null,
            (ParseFlags) 0);
    }

    bool ParseInsignificantTokens(ref Token token, TokenList buffer,
        List<TokenList> attributesList, ParseFlags flags)
    {
        // skip "insignificant" tokens
        bool foundInsignificant = false;
        while (true)
        {
            if (token.IsWhiteSpace || token.IsNewline || token.IsAComment ||
                token.IsADirective || token.IsExcluded)
            {
                // an insignificant token
                foundInsignificant = true;
                token = token.Next;
            }
            else
            if (token.IsCharacter(LSquare) &&
                ((flags & ParseFlags.NoAttributes) == 0))
            {
                // this is presumably an attribute declaration, e.g.
                // "[Foo(123), Bar(X=45, Y=67)]" -- also insignificant, but
                // track attributes if <attributesList> is not null
                foundInsignificant = true;
                if (attributesList != null)
                    ParseAttributeDeclaration(ref token, attributesList);
                else
                    ParseGroup(ref token, null);
            }
            else
                break;
        }

        // if we found insignificant tokens, add a single space token to
        // <buffer>
        if (foundInsignificant && (buffer != null))
            buffer.Append(new Token(TokenType.WhiteSpace, " "));

        return foundInsignificant;
    }

    void ParseAttributeDeclaration(ref Token token,
        List<TokenList> attributesList)
    {
        // skip the <LSquare> token, but remember its location for error
        // messages
        Token lsquareToken = token;
        Debug.Assert(token.IsCharacter(LSquare));
        token = token.Next;

        // loop once for each individual attributes within the attribute
        // declaration; for example, loop twice for each of the the two
        // attributes in "[Foo(123), Bar(X=45, Y=67)]"
        while (true)
        {
            // loop once for each token within this individual attribute, e.g.
            // "Bar(X=45, Y=67)"
            TokenList buffer = new TokenList();
            while (true)
            {
                // skip white space, comments, etc.
                ParseInsignificantTokens(ref token, buffer);

                // if we hit a comma or <RSquare>, we're at the end of this
                // individual attribute
                if (token.IsCharacter(',', RSquare))
                    break;

                // error if we hit <LBrace>, <RBrace>, <RParen>, or the end of
                // the source file
                if (token.IsCharacter(LBrace, RBrace, RParen) || token.IsTail)
                {
                    throw Expected(token,
                        Resources.ExpectedEndOfAttrDeclaration,
                        lsquareToken.GetLineNumber());
                }

                // if we hit a <LParen>, parse up to and including the matching
                // <RParen>
                if (token.IsCharacter(LParen))
                {
                    ParseGroup(ref token, buffer);
                    continue;
                }

                // add this token to the buffer and continue parsing
                buffer.Append(token.Clone());
                token = token.Next;
            }

            // add this individual attribute to <attributesList>, unless it's
            // an empty declaration
            if (buffer.TrimBlankTokens() != null)
                attributesList.Add(buffer);

            // if we hit the <RSquare>, we're at the end of this attribute
            // declaration
            if (token.IsCharacter(RSquare))
                break;

            // skip the comma
            Debug.Assert(token.IsCharacter(','));
            token = token.Next;
        }

        // skip the <RSquare> token
        Debug.Assert(token.IsCharacter(RSquare));
        token = token.Next;
    }

    bool ParseInsignificantTokensAndXmlComments(ref Token token,
        TokenList buffer)
    {
        bool result = false;
        while (true)
        {
            result |= ParseInsignificantTokens(ref token, buffer);
            if (!token.IsAnXmlComment)
                return result;
            result = true;
            token = token.Next;
        }
    }

    [Flags]
    enum ParseFlags
    {

        NoAttributes = 1
    }
}

internal class CSharpTokenParser : TokenParser
{
    //////////////////////////////////////////////////////////////////////////
    // Private Static Fields
    //

    public static readonly Dictionary<string, bool> ReservedWords =
        InitializeReservedWords();

    public static readonly Dictionary<string, bool> ReservedTypes =
        InitializeReservedTypes();

    public static readonly Dictionary<string, bool> Modifiers =
        InitializeModifiers();

    //////////////////////////////////////////////////////////////////////////
    // Public Methods
    //

    public CSharpTokenParser(string sourceFileLabel,
            TextReader sourceFileReader) :
        base(sourceFileLabel, sourceFileReader)
    {
    }

    public static TokenList ParseString(string label, string sourceCode)
    {
        using (StringReader reader = new StringReader(sourceCode))
        {
            CSharpTokenParser parse = new CSharpTokenParser(label, reader);
            TokenList tokenList = new TokenList();
            while (true)
            {
                Token token = parse.ParseToken();
                if (token == null)
                    break;
                tokenList.Append(token.Clone());
            }
            return tokenList;
        }
    }

    //////////////////////////////////////////////////////////////////////////
    // Internal Methods
    //

    internal override Token ParseToken()
    {
        Token token;

        // if there's a token in <m_tokenQueue>, return it
        if (m_tokenQueue.Count > 0)
            return m_tokenQueue.Dequeue();

        // clear the token buffer
        m_tokenBuffer.Length = 0;

        // check for a white space token and end of stream
        int blanks = ReadWhiteSpace();
        if (blanks > 0)
        {
            return new Token(TokenType.WhiteSpace,
                new String(' ', blanks));
        }
        if (m_char1 < 0)
            return null;

        // check for a newline token
        if ((token = ParseNewline(true)) != null)
            return token;

        // check for a string literal
        if ((token = ParseStringLiteral()) != null)
            return token;

        // check for a character literal
        if ((token = ParseCharacterLiteral()) != null)
            return token;

        // check for a "//"-style comment
        if ((token = ParseSlashSlashComment()) != null)
            return token;

        // check for a "/*"-style comment
        if ((token = ParseSlashStarComment()) != null)
            return token;

        // check for a word, i.e. something that follows the lexical rules of
        // an identifier, reserved word, numeric literal, etc.
        if ((token = ParseWord()) != null)
            return token;

        // token is some other character
        char ch = (char) m_char1;
        Advance(false);
        return new Token(ch);
    }

    internal override Token ParseWord()
    {
        // parse the word (if any) into <m_tokenBuffer>
        if (IsDigit(m_char1) ||
            (IsSign(m_char1) && IsDigit(m_char2)) ||
            ((m_char1 == '.') && IsDigit(m_char2)) ||
            (IsSign(m_char1) && (m_char2 == '.') && IsDigit(m_char3)))
        {
            // the token is a numeric literal, e.g. "1", +2", "3.14", ".5",
            // "-.5", "1e5", etc.
            while (IsIdChar(m_char1) || IsSign(m_char1) || (m_char1 == '.'))
                Advance(true);
        }
        else
        if (IsIdChar(m_char1))
        {
            // the token is an identifier, or a reserved word, etc.
            while (IsIdChar(m_char1))
                Advance(true);
        }
        else
            return null; // not a "word"

        // return a new Token -- Word or ReservedWord
        return NewWordToken(m_tokenBuffer.ToString());
    }

    internal static Token NewWordToken(string text)
    {
        bool unused;
        if (ReservedWords.TryGetValue(text, out unused))
            return new Token(TokenType.ReservedWord, text);
        else
            return new Token(TokenType.Word, text);
    }

    //////////////////////////////////////////////////////////////////////////
    // Private Methods
    //

    Token ParseStringLiteral()
    {
        // check if this is a string literal; also, set <atPrefix> to true if
        // this string literal has an "@" prefix, and advance to just past the
        // starting double quote (")
        bool atPrefix;
        if ((m_char1 == '@') && (m_char2 == '"'))
        {
            atPrefix = true;
            Advance2(true);
        }
        else
            if (m_char1 == '"')
            {
                atPrefix = false;
                Advance(true);
            }
            else
                return null; // not a string literal

        // parse the string literal
        while (true)
        {
            if (atPrefix && (m_char1 == '"') && (m_char2 == '"'))
            {
                // double double quotes (""), e.g. @"abc""def"
                Advance2(true);
            }
            else
            if (!atPrefix && (m_char1 == '\\'))
            {
                // backslash-escaped character
                if ((m_char2 == -1) || (m_char2 == '\r') || (m_char2 == '\n'))
                    throw NewParsingException(Resources.NewlineInConstant);
                Advance2(true);
            }
            else
            if (m_char1 == '"')
                break;
            else
            if ((m_char1 == '\r') || (m_char1 == '\n'))
                throw NewParsingException(Resources.NewlineInConstant);
            else
                Advance(true);
        }

        // skip the closing double quote (")
        Advance(true);

        // done
        return new Token(TokenType.StringLiteral,
            m_tokenBuffer.ToString());
    }

    Token ParseCharacterLiteral()
    {
        // check if this is a character literal; advance to just past the
        // starting single quote (')
        if (m_char1 == '\'')
            Advance(true);
        else
            return null; // not a string literal

        // parse the character literal
        while (true)
        {
            if (m_char1 == '\\')
            {
                // backslash-escaped character
                if ((m_char2 == -1) || (m_char2 == '\r') || (m_char2 == '\n'))
                    throw NewParsingException(Resources.NewlineInConstant);
                Advance2(true);
            }
            else
            if (m_char1 == '\'')
                break;
            else
            if ((m_char1 == '\r') || (m_char1 == '\n'))
                throw NewParsingException(Resources.NewlineInConstant);
            else
                Advance(true);
        }

        // skip the closing single quote (')
        Advance(true);

        // done
        return new Token(TokenType.StringLiteral,
            m_tokenBuffer.ToString());
    }

    Token ParseSlashSlashComment()
    {
        // check if this is a "//"-style comment
        if ((m_char1 != '/') || (m_char2 != '/'))
            return null;

        // skip "//"; set <isXmlComment> to true iff this is a "///"-style
        // XML comment
        Advance2(true);
        bool isXmlComment = (m_char1 == '/');

        // parse to the end of the line
        while ((m_char1 != '\r') && (m_char1 != '\n'))
            Advance(true);

        // if this is not an XML comment we're done (we don't want to include
        // the newline within a single-line comment because, for example, that
        // would make preprocessor directives like "#if FOO // comment"
        // fail because Preprocess() looks for a newline to end the directive);
        // if this is an XML comment then continue parsing subsequent lines
        // of comments (since we need to collect all XML comments into a single
        // block of XML)
        if (!isXmlComment)
            return new Token(TokenType.Comment, m_tokenBuffer.ToString());
        while (true)
        {
            // parse the newline and add it to <m_tokenBuffer>
            ParseNewline(false);

            // read leading white space on the next line
            int blanks = ReadWhiteSpace();

            // see if this line is a continuation of the comment; if so,
            // add the leading blanks and the "//" or "///" to <m_tokenBuffer>
            // and set <continuation> to true, otherwise set <continuation> to
            // false
            bool continuation;
            if ((m_char1 == '/') && (m_char2 == '/') && (m_char3 == '/'))
            {
                m_tokenBuffer.Append(new String(' ', blanks));
                Advance2(true);
                Advance(true);
                continuation = true;
            }
            else
                continuation = false;

            // if this is not a continuation of the comment
            if (!continuation)
            {
                // this was not a continuation of the comment -- "push" the
                // leading white space, if any, back into the output stream of
                // tokens (so it's the next token returned by ParseToken())
                if (blanks > 0)
                {
                    m_tokenQueue.Enqueue(new Token(TokenType.WhiteSpace,
                        new String(' ', blanks)));
                }

                // return the token
                return new XmlCommentToken(true, m_tokenBuffer.ToString());
            }

            // advance to the end of this comment line
            while ((m_char1 != '\r') && (m_char1 != '\n'))
                Advance(true);
        }
    }

    Token ParseSlashStarComment()
    {
        // check if this is a "/*"-style comment
        if ((m_char1 != '/') || (m_char2 != '*'))
            return null;

        // skip "/*"; set <isXmlComment> to true iff this is a "/**"-style
        // XML comment
        Advance2(true);
        bool isXmlComment = (m_char1 == '*');

        // parse to the end of the comment
        while (true)
        {
            if (m_char1 == -1) // end of stream
                break;
            if ((m_char1 == '*') && (m_char2 == '/'))
                break;
            Advance(true);
        }

        // skip "*/"
        Advance2(true);

        // return the token
        string text = m_tokenBuffer.ToString();
        if (isXmlComment)
            return new XmlCommentToken(false, text);
        else
            return new Token(TokenType.Comment, text);
    }

    static bool IsIdChar(int iChar)
    {
        return (((iChar >= 'a') && (iChar <= 'z')) ||
            ((iChar >= 'A') && (iChar <= 'Z')) ||
            ((iChar >= '0') && (iChar <= '9')) ||
            (iChar == '_') || (iChar == '#'));
    }

    static Dictionary<string, bool> InitializeReservedWords()
    {
        string[] reservedWordsArray = new string[]
        {
            // from C# documentation
            "abstract", "as", "base", "bool", "break", "byte", "case", "catch",
            "char", "checked", "class", "const", "continue", "decimal",
            "default", "delegate", "do", "double", "else", "enum", "event",
            "explicit", "extern", "false", "finally", "fixed", "float", "for",
            "foreach", "get", "goto", "if", "implicit", "in", "int",
            "interface", "internal", "is", "lock", "long", "namespace", "new",
            "null", "object", "operator", "out", "override", "params",
            "partial", "private", "protected", "public", "readonly", "ref",
            "return", "sbyte", "sealed", "set", "short", "sizeof",
            "stackalloc", "static", "string", "struct", "switch", "this",
            "throw", "true", "try", "typeof", "uint", "ulong", "unchecked",
            "unsafe", "ushort", "using", "value", "virtual", "void",
            "volatile", "where", "while", "yield",

            // additions
            "#if", "#elif", "#else", "#endif", "#define", "#undef",
            "#warning", "#error", "#line", "#region", "#endregion", "#pragma"
        };
        Dictionary<string, bool> reservedWords =
            new Dictionary<string, bool>(reservedWordsArray.Length * 2);
        foreach (string word in reservedWordsArray)
            reservedWords.Add(word, true);
        return reservedWords;
    }

    static Dictionary<string, bool> InitializeReservedTypes()
    {
        string[] reservedTypesArray = new string[]
        {
            "bool",  "byte", "char", "decimal", "double", "fixed", "float",
            "int", "long", "object", "sbyte", "short", "string", "uint",
            "ulong", "ushort", "void"
        };
        Dictionary<string, bool> reservedTypes =
            new Dictionary<string, bool>(reservedTypesArray.Length * 2);
        foreach (string word in reservedTypesArray)
            reservedTypes.Add(word, true);
        return reservedTypes;
    }

    static Dictionary<string, bool> InitializeModifiers()
    {
        string[] reservedModifiersArray = new string[]
        {
            "abstract", "const", "explicit", "extern", "implicit", "internal",
            "new", "override", "partial", "private", "protected", "public",
            "readonly", "sealed", "static", "unsafe", "virtual", "volatile",
        };
        Dictionary<string, bool> reservedModifiers =
            new Dictionary<string, bool>(reservedModifiersArray.Length * 2);
        foreach (string word in reservedModifiersArray)
            reservedModifiers.Add(word, true);
        return reservedModifiers;
    }
}

public class UsingContext
{
    //////////////////////////////////////////////////////////////////////////
    // Private Fields
    //

    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
    List<TokenList> m_unaliased = new List<TokenList>(20);

    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
    Dictionary<string, TokenList> m_aliases =
        new Dictionary<string, TokenList>(20);

    //////////////////////////////////////////////////////////////////////////
    // Public Properties
    //

    public ICollection<TokenList> Unaliased
    {
        [DebuggerStepThrough]
        get
        {
            return new ReadOnlyCollection<TokenList>(m_unaliased);
        }
    }

    public ICollection<KeyValuePair<string,TokenList>> Aliases
    {
        [DebuggerStepThrough]
        get
        {
            List<KeyValuePair<string, TokenList>> list =
                new List<KeyValuePair<string, TokenList>>(m_aliases.Count);
            foreach (KeyValuePair<string, TokenList> kvp in m_aliases)
                list.Add(kvp);
            return new ReadOnlyCollection<
                KeyValuePair<string,TokenList>>(list);
        }
    }

    //////////////////////////////////////////////////////////////////////////
    // Public Methods
    //

    public UsingContext()
    {
    }

    public bool ContainsUnaliased(TokenList namespaceOrTypeName)
    {
        foreach (TokenList name in m_unaliased)
        {
            if (name == namespaceOrTypeName)
                return true;
        }
        return false;
    }

    public TokenList LookupAlias(string alias)
    {
        TokenList namespaceOrTypeName;
        if (m_aliases.TryGetValue(alias, out namespaceOrTypeName))
            return namespaceOrTypeName;
        else
            return null;
    }

    //////////////////////////////////////////////////////////////////////////
    // Internal Methods
    //

    public UsingContext(UsingContext otherContext)
    {
        m_unaliased.InsertRange(0, otherContext.m_unaliased);
        foreach (KeyValuePair<string, TokenList> kvp in otherContext.m_aliases)
            m_aliases.Add(kvp.Key, kvp.Value);
    }

    internal void AddUnaliased(TokenList namespaceOrTypeName)
    {
        if (!ContainsUnaliased(namespaceOrTypeName))
            m_unaliased.Add(namespaceOrTypeName);
    }

    internal void AddAlias(string alias, TokenList namespaceOrTypeName)
    {
        m_aliases[alias] = namespaceOrTypeName;
    }

    internal UsingContext Clone()
    {
        return new UsingContext(this);
    }

#if DEBUG
    /// <summary>
    /// Returns debugging information about this object.
    /// </summary>
    ///
    internal string Dump()
    {
        StringBuilder result = new StringBuilder(500);
        foreach (TokenList name in m_unaliased)
            result.AppendFormat("using {0};\r\n", name);
        foreach (KeyValuePair<string, TokenList> kvp in m_aliases)
            result.AppendFormat("using {0} = {1};\r\n", kvp.Key, kvp.Value);
        return result.ToString();
    }
#endif
}

}