/* Copyright (c) Eric Ledoux. All rights reserved. */ /* See http://www.dwell.net/terms for code sharing information. */ // Token.cs // // Implements class Token and related types. // using System; using System.Collections.Generic; using System.Diagnostics; using System.Text; using System.Text.RegularExpressions; using DwellNet.CodeDoc; using CodeDocApi.Properties; namespace DwellNet.CodeDoc {
Token Class
One small piece of a source code file. |
[DebuggerDisplay("{DebuggerDisplay}")] public class Token : IComparable<Token> { ////////////////////////////////////////////////////////////////////////// // Internal Fields //
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
internal TokenType m_tokenType;
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
internal string m_text;
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
internal Token m_previous;
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
internal Token m_next;
Token.s_newlineRegex Field
Matches a newline character. |
static readonly Regex s_newlineRegex = new Regex("\n"); ////////////////////////////////////////////////////////////////////////// // Private Properties // #if DEBUG /// <summary> /// Gets a string displayed in the debugger. /// </summary> public string DebuggerDisplay { get { const int maxLength = 50; string text = ((Text.Length < maxLength) ? Text : (Text.Substring(0, maxLength) + "...")); return String.Format("{0}: {1}", TokenType, text); } } #endif #if DEBUG /// <summary> /// Gets a string representation of the contents of the portion of /// the linked list containing this token and following tokens, up to /// 100 characters. /// </summary> public string DebuggerListContext { get { return TokenListToString(100, false, false); } } #endif ////////////////////////////////////////////////////////////////////////// // Public Properties //
public TokenType TokenType { [DebuggerStepThrough] get { return m_tokenType; } }
Token.Text Property
Gets the text of the token. In some cases, the text is transformed from the input stream text; for example, tabs are expanded. |
public string Text { [DebuggerStepThrough] get { return m_text; } [DebuggerStepThrough] internal set { m_text = value; } }
Token.Html Property
Gets the text of the token, converted to HTML, without any syntax coloring. |
public string Html { get { return HtmlEncode(m_text); } }
Token.Description Property
Gets a short string describing the token. If the token text representation isn't blank, the description is the text of the token, truncated with ellipsis (...) if it's long. Otherwise the description is a word or phrase in parentheses. |
public string Description { get { switch (TokenType) { case TokenType.Head: return Resources.TokenTypeHead; case TokenType.Tail: return Resources.TokenTypeTail; case TokenType.WhiteSpace: return Resources.TokenTypeWhiteSpace; case TokenType.Newline: return Resources.TokenTypeNewline; default: return SourceFile.Ellipsis(m_text, 20); } } }
Token.Previous Property
Gets the previous token in the source file, or null if this is the first token or the token isn't in a linked list. |
public Token Previous { [DebuggerStepThrough] get { return m_previous; } }
Token.Next Property
Gets the next token in the source file, or null if this is the last token or the token isn't in a linked list. |
public Token Next { [DebuggerStepThrough] get { return m_next; } }
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsHead
{
get
{
return (TokenType == TokenType.Head);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsTail
{
get
{
return (TokenType == TokenType.Tail);
}
}
Token.IsWhiteSpace Property
Returns true if this token is type TokenType.WhiteSpace, false otherwise. |
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsWhiteSpace
{
get
{
return (TokenType == TokenType.WhiteSpace);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsNewline
{
get
{
return (TokenType == TokenType.Newline);
}
}
Token.IsAStringLiteral Property
Returns true if this token is type TokenType.StringLiteral, false otherwise. |
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsAStringLiteral
{
get
{
return (TokenType == TokenType.StringLiteral);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsAComment
{
get
{
return (TokenType == TokenType.Comment);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsAnXmlComment
{
get
{
return (TokenType == TokenType.XmlComment);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsAWord
{
get
{
return (TokenType == TokenType.Word);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsAReservedWord
{
get
{
return (TokenType == TokenType.ReservedWord);
}
}
Token.IsAWordOrReservedWord Property
Returns true if this token is type TokenType.Word or TokenType.ReservedWord. |
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsAWordOrReservedWord
{
get
{
return (TokenType == TokenType.Word) ||
(TokenType == TokenType.ReservedWord);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsACharacter
{
get
{
return (TokenType == TokenType.Character);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsADirective
{
get
{
return (TokenType == TokenType.Directive);
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public bool IsExcluded
{
get
{
return (TokenType == TokenType.Excluded);
}
}
//////////////////////////////////////////////////////////////////////////
// Public Methods
//
Token.ToString Method
Returns the text of the token. |
override public string ToString() { return Text; }
Token.GetLineNumber Method
Returns the 1-based line number that this token starts on.
Remarks
A few tokens, such as TokenType.Comment, cross line boundaries. For these tokens, the return value is the line number of the line on which the token begins. |
public int GetLineNumber() { // if this token is a newline, return its number since a newline is // considered to be part of the previous line if (TokenType == TokenType.Newline) return ((NewlineToken)this).LineNumber; // start walking backward to find the previous newline Token token = Previous; int embeddedNewlines = 0; // number of '\n' in walked-over tokens while (true) { // if we found the previous newline, <this> is on the next line if (token.IsNewline) { return ((NewlineToken)token).LineNumber + 1 + embeddedNewlines; } // if we found the beginning of the list, <this> is on the first // line if (token.IsHead) return 1; // comments (including XML comments) can include embedded // newlines -- if this is one of them, count the newlines to // adjust the line number count if (token.IsAComment || token.IsAnXmlComment) embeddedNewlines += s_newlineRegex.Matches(token.Text).Count; // walk backward token = token.Previous; } }
Token.SkipWhiteSpace Method
Returns the first token at or after this token that is not whitespace. Newlines and comments (excluding XML documentation comments) are also skipped. |
public Token SkipWhiteSpace() { Token token = this; while (token.IsWhiteSpace || token.IsNewline || token.IsAComment) token = token.Next; return token; }
Token.IsBlank Method
Returns true if this token is a TokenType.WhiteSpace or TokenType.Newline token. |
public bool IsBlank() { return IsWhiteSpace || IsNewline; }
Token.TextIs Method
Returns true if the text of this token is specific character.
Parameters
ch The text value to compare with. |
public bool TextIs(char ch) { return ((Text.Length == 1) && (Text[0] == ch)); }
Token.IsCharacter Method (char)
Returns true if this token is a TokenType.Character token with a specific text value.
Parameters
ch The text value to compare with. |
public bool IsCharacter(char ch) { if (TokenType != TokenType.Character) return false; Debug.Assert(Text.Length == 1); return (Text[0] == ch); }
Token.IsCharacter Method (params char[])
Returns true if this token is a TokenType.Character token with one of a set of specific text value.
Parameters
chars The text values to compare with. |
public bool IsCharacter(params char[] chars) { if (TokenType != TokenType.Character) return false; Debug.Assert(Text.Length == 1); foreach (char ch in chars) { if (Text[0] == ch) return true; } return false; }
Token.IsReservedWord Method
Returns true if this token is a TokenType.ReservedWord token with a specific text value.
Parameters
word The text value to compare with. |
public bool IsReservedWord(string word) { if (TokenType != TokenType.ReservedWord) return false; return (Text == word); }
Token.IsWordOrReservedWord Method
Returns true if this token is a TokenType.Word or TokenType.ReservedWord token with a specific text value.
Parameters
word The text value to compare with. |
public bool IsWordOrReservedWord(string word) { if ((TokenType != TokenType.Word) && (TokenType != TokenType.ReservedWord)) return false; return (Text == word); }
Token.TokenListToString Method
Returns a string representation of the contents of the portion of the linked list containing this token and following tokens.
Parameters
maxLength The maximum number of characters to return, not including the ellipsis (...) which is added if the returned value would exceed this length. multiLine true if the return value should include newlines, false if newlines should be converted to spaces. preserveSpaces true to preserve spaces, false to collapse multiple adjacent spaces into a single space. |
public string TokenListToString(int maxLength, bool multiLine, bool preserveSpaces) { StringBuilder result = new StringBuilder(1000); Token token = this; while (!token.IsTail) { if ((token.TokenType == TokenType.WhiteSpace) && !preserveSpaces) result.Append(' '); else if (token.TokenType == TokenType.Newline) { if (multiLine) result.Append("\r\n"); else result.Append(' '); } else { if (multiLine) result.Append(token.Text); else result.Append(token.Text.Replace("\n", " ")); } if (result.Length > maxLength) return result.ToString().Substring(0, maxLength) + "..."; token = token.Next; } return result.ToString(); }
Token.Equals Method (object)
Determines whether this Token equals another.
Parameters
obj The other object to compare to. |
public override bool Equals(object obj) { if (obj == null) return false; Token other = obj as Token; if (other == null) return false; else return Equals(other); }
Token.Equals Method (Token)
Determines whether this Token equals another.
Parameters
other The Token to compare to.
Remarks
Tokens are compared only on their text, not their type. So, for example, TokenType.Word "Foo" and TokenType.ReservedWord "Foo" are considered equal. |
public bool Equals(Token other) { return (Text == other.Text); }
public override int GetHashCode() { return Text.GetHashCode(); } ////////////////////////////////////////////////////////////////////////// // Internal Methods //
Token Constructor (TokenType, string)
Initializes an instance of this class.
Parameters
tokenType What kind of token this is. text The plain text (not HTML) corresponding to this token. This is text from the source file, possibly transformed somewhat (e.g. tabs expanded). |
internal Token(TokenType tokenType, string text) { m_tokenType = tokenType; m_text = text; }
Token Constructor (char)
Initializes an instance of this class. Creates a TokenType.Character token containing a given character.
Parameters
text The character. This should not be white space or a newline character. |
internal Token(char text) { m_tokenType = TokenType.Character; m_text = new String(text, 1); }
Token.Clone Method
Returns a copy of this token. The copy is not in any linked list. |
internal Token Clone() { return new Token(TokenType, Text); }
Token.InsertBefore Method
Inserts this Token before given token in a linked list.
Parameters
nextToken The token to insert before.
Exceptions
|
internal void InsertBefore(Token nextToken) { if ((m_previous != null) || (m_next != null)) throw new InvalidOperationException(); m_previous = nextToken.m_previous; nextToken.m_previous = this; if (m_previous != null) m_previous.m_next = this; m_next = nextToken; }
Token.InsertAfter Method
Inserts this Token after given token in a linked list.
Parameters
previousToken The token to insert after.
Exceptions
|
internal void InsertAfter(Token previousToken) { if ((m_previous != null) || (m_next != null)) throw new InvalidOperationException(); m_next = previousToken.m_next; previousToken.m_next = this; if (m_next != null) m_next.m_previous = this; m_previous = previousToken; }
internal void RemoveFromList() { if (m_previous != null) m_previous.m_next = m_next; if (m_next != null) m_next.m_previous = m_previous; m_previous = null; m_next = null; }
internal Token FindEndOfLine() { Token token = this; while (!token.IsNewline && !token.IsTail) token = token.Next; return token; }
Token.FindEndOfLineOrTripleSlash Method
Returns the next TokenType.Newline or TokenType.Tail token, or the next "///"-style |
internal Token FindEndOfLineOrTripleSlash() { Token token = this; while (!token.IsNewline && !token.IsTail) { if (token.IsAnXmlComment && ((XmlCommentToken)token).IsTripleSlash) break; token = token.Next; } return token; }
Token.FindStartOfNextLine Method
Returns the first token on the next line, or or TokenType.Tail if encountered first.
Remarks
Note that if the current line ends with a "///"-style XML comment, this method will return the token after the entire XML comment. |
internal Token FindStartOfNextLine() { Token token = FindEndOfLineOrTripleSlash(); if (token.IsTail) return token; else return token.Next; }
Token.FindCharacterToken Method
Searches for a TokenType.Character token, starting at this token. Returns the first one found, or null if none.
Parameters
ch The character to find. |
internal Token FindCharacterToken(char ch) { Token token = this; while (!token.IsTail) { if (token.IsCharacter(ch)) return token; return token.Next; } return null; }
Token.ReadDotDelimitedId Method
Assuming this token is the beginning of a sequence of one or more tokens of the form "word.word. ... .word", where each word is a TokenType.Word or TokenType.ReservedWord token, read that sequence and return the first token after that sequence, or null on error.
Parameters
id On exit, contains the parsed identifier; for example, "Abc" or "Abc.Def.Ghi".
Remarks
For example this method can be used to parse a namespace identifier such as "System.Collections.Generic". |
internal Token ReadDotDelimitedId(out string id) { // parse the identifier StringBuilder idBuffer = new StringBuilder(100); Token token = this; while (true) { // skip white space (if any) token = token.SkipWhiteSpace(); // the next token should be a word or reserved word; if we don't // start with a word or reserved word, or if the identifier ends // with a trailing ".", it's an error if (!token.IsAWord && !token.IsAReservedWord) { id = null; return null; } idBuffer.Append(token.Text); token = token.Next; // skip white space (if any) token = token.SkipWhiteSpace(); // look for a "." if (!token.IsCharacter('.')) break; idBuffer.Append('.'); token = token.Next; } // return the identifier id = idBuffer.ToString(); return token; } ////////////////////////////////////////////////////////////////////////// // Internal Methods // #if DEBUG /// <summary> /// Returns debugging information about this token. /// </summary> /// internal string Dump() { switch (m_tokenType) { case TokenType.WhiteSpace: Debug.Assert(m_text.Trim().Length == 0); return String.Format("WhiteSpace: {0} blank(s)", m_text.Length); case TokenType.Newline: Debug.Assert(m_text == "\n"); return String.Format("Newline"); default: return String.Format("{0}: [{1}]", m_tokenType, m_text); } } #endif ////////////////////////////////////////////////////////////////////////// // Private Methods //
Token.HtmlEncode Method
Converts plain text to HTML.
Parameters
text The plain text to convert to HTML. |
internal static string HtmlEncode(string text) { return text.Replace("&", "&").Replace("<", "<") .Replace(">", ">"); } ////////////////////////////////////////////////////////////////////////// // IComparable Implementation //
Token.CompareTo Method
Compares this instance to a specified Token object and returns an indication of their relative values.
Parameters
other The Token to compare to.
Return Value
A signed integer indicating the relative values of this instance and other: a value less than zero if this instance is less than other; zero if this instance is equal to other; a value greater than zero if this instance is greater than other.
Remarks
Tokens are compared only on their text, not their type. So, for example, TokenType.Word "Foo" and TokenType.ReservedWord "Foo" are considered equal. |
public int CompareTo(Token other) { return String.Compare(Text, other.Text); } }
[DebuggerDisplay("{DebuggerDisplay}")] public class NewlineToken : Token { ////////////////////////////////////////////////////////////////////////// // Private Fields //
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
int m_lineNumber;
//////////////////////////////////////////////////////////////////////////
// Public Properties
//
NewlineToken.LineNumber Property
The 1-based line number of the line that is ended by this newline. |
public int LineNumber { [DebuggerStepThrough] get { return m_lineNumber; } } ////////////////////////////////////////////////////////////////////////// // Internal Methods //
NewlineToken Constructor
Initializes an instance of this class.
Parameters
lineNumber The 1-based line number of the line that is ended by this newline. |
internal NewlineToken(int lineNumber) : base(TokenType.Newline, "\n") { m_lineNumber = lineNumber; } }
XmlCommentToken Class
A token of type TokenType.XmlComment. Currently XML comments are only supported for C# source code. |
[DebuggerDisplay("{DebuggerDisplay}")] public class XmlCommentToken : Token { ////////////////////////////////////////////////////////////////////////// // Private Fields //
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
bool m_isTripleSlash;
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
Topic m_topic;
//////////////////////////////////////////////////////////////////////////
// Public Properties
//
XmlCommentToken.IsTripleSlash Property
true if this is a "///"-style C# XML comment, false if this is a "/**"-style C# XML comment. |
public bool IsTripleSlash { [DebuggerStepThrough] get { return m_isTripleSlash; } }
XmlCommentToken.Topic Property
The documentation topic associated with this comment; null if none. |
public Topic Topic { [DebuggerStepThrough] get { return m_topic; } [DebuggerStepThrough] internal set { m_topic = value; } } ////////////////////////////////////////////////////////////////////////// // Internal Methods //
XmlCommentToken Constructor
Initializes an instance of this class.
Parameters
isTripleSlash true if this is a "///"-style XML comment, false if this is a "/**"-style XML comment. text The plain text (not HTML) corresponding to this token. This is text from the source file, possibly transformed somewhat (e.g. tabs expanded). |
internal XmlCommentToken(bool isTripleSlash, string text) : base(TokenType.XmlComment, text) { m_isTripleSlash = isTripleSlash; } }
DirectiveToken Class
A token of type TokenType.Directive. Currently directive are only supported for C# source code. |
[DebuggerDisplay("{DebuggerDisplay}")] public class DirectiveToken : Token { ////////////////////////////////////////////////////////////////////////// // Private Fields //
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
TokenList m_containedTokens;
//////////////////////////////////////////////////////////////////////////
// Private Fields
//
DirectiveToken.ContainedTokens Property
The tokens that were collected into this directive, not including leading and trailing white space and comments or the trailing newline. For example, for the directive "#if FOO && !BAR", ContainedTokens contains all those tokens. |
public TokenList ContainedTokens { [DebuggerStepThrough] get { return m_containedTokens; } } ////////////////////////////////////////////////////////////////////////// // Internal Methods //
DirectiveToken Constructor
Initializes an instance of this class.
Parameters
containedTokens The value of the ContainedTokens property. text The plain text (not HTML) corresponding to this token. This is text from the source file, possibly transformed somewhat (e.g. tabs expanded). |
internal DirectiveToken(TokenList containedTokens, string text) : base(TokenType.Directive, text) { m_containedTokens = containedTokens; } }
ExcludedToken Class
A token of type TokenType.Excluded, representing a line of code excluded by conditional compilation. |
[DebuggerDisplay("{DebuggerDisplay}")] public class ExcludedToken : Token { ////////////////////////////////////////////////////////////////////////// // Private Fields //
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
TokenList m_containedTokens;
//////////////////////////////////////////////////////////////////////////
// Private Fields
//
ExcludedToken.ContainedTokens Property
The tokens that were collected into this directive, not including leading and trailing white space and comments or the trailing newline. |
public TokenList ContainedTokens { [DebuggerStepThrough] get { return m_containedTokens; } } ////////////////////////////////////////////////////////////////////////// // Internal Methods //
ExcludedToken Constructor
Initializes an instance of this class.
Parameters
containedTokens The value of the ContainedTokens property. text The plain text (not HTML) corresponding to this token. This is text from the source file, possibly transformed somewhat (e.g. tabs expanded). |
internal ExcludedToken(TokenList containedTokens, string text) : base(TokenType.Excluded, text) { m_containedTokens = containedTokens; } }
TokenType Enumeration
A kind of Token.
Members
|
public enum TokenType {
TokenType.Head Enumeration Value
The first token in the linked list. This Token is a placeholder and contains no information from the source file. Token.Text is empty. |
Head,
TokenType.Tail Enumeration Value
The last token in the linked list. This Token is a placeholder and contains no information from the source file. Token.Text is empty. |
Tail,
TokenType.WhiteSpace Enumeration Value
White space (outside a string literal or comment). Token.Text contains space characters equivalent to the white space -- tabs are expanded. |
WhiteSpace,
TokenType.Newline Enumeration Value
A newline (CRLF). Token.Text is always "\n". The type of this token is NewlineToken. |
Newline,
TokenType.StringLiteral Enumeration Value
A string literal such as "Abc" or @"Abc", or a character literal such as 'A', including the quotes. |
StringLiteral,
TokenType.Comment Enumeration Value
A "//"-style or "/*"-style comment, but not an XML comment. |
Comment,
TokenType.XmlComment Enumeration Value
A "///"-style or "/**"-style XML comment. Currently XML comments are only supported for C# source code. The type of this token is XmlCommentToken. |
XmlComment,
TokenType.Word Enumeration Value
A numeric literal, identifier, etc., but not a reserved word. |
Word,
TokenType.ReservedWord Enumeration Value
A reserved word (e.g. "while"). |
ReservedWord,
TokenType.Character Enumeration Value
Any other character. |
Character,
TokenType.Directive Enumeration Value
Represent a preprocessor directive such as "#if false". Includes only the directive itself, not leading or trailing comments or white space on that line, or the newline that ends the line. The type of this token is DirectiveToken.
Remarks
See CSharpSourceFile.Preprocess for more information. |
Directive,
TokenType.Excluded Enumeration Value
Represents a line of code excluded by conditional compilation preprocessor directives (
Remarks
See CSharpSourceFile.Preprocess for more information. |
Excluded, }
[DebuggerDisplay("{DebuggerDisplay}")] public class TokenList : IEnumerable<Token>, IComparable<TokenList> { ////////////////////////////////////////////////////////////////////////// // Private Fields //
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
Token m_headToken;
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
Token m_tailToken;
//////////////////////////////////////////////////////////////////////////
// Private Fields
//
TokenList.HeadToken Property
The head of the linked list of tokens. This Token is a placeholder and contains no real information. |
public Token HeadToken { [DebuggerStepThrough] get { return m_headToken; } }
TokenList.TailToken Property
The tail of the linked list of tokens. This Token is a placeholder and contains no real information. |
public Token TailToken { [DebuggerStepThrough] get { return m_tailToken; } } ////////////////////////////////////////////////////////////////////////// // Private Properties // #if DEBUG /// <summary> /// Gets a string displayed in the debugger when examining an instance /// of this class.. /// </summary> [DebuggerBrowsable(DebuggerBrowsableState.Never)] string DebuggerDisplay { get { return ToString(100, false, false); } } #endif #if DEBUG /// <summary> /// Gets a string displayed in the debugger. /// </summary> string _DebugValue { get { return ToString(); } } #endif ////////////////////////////////////////////////////////////////////////// // Public Properties //
TokenList.IsEmpty Property
Returns true if the list is empty (except for TokenType.Head and TokenType.Tail tokens), false if not. |
public bool IsEmpty { get { return Object.ReferenceEquals(HeadToken.Next, TailToken); } } ////////////////////////////////////////////////////////////////////////// // Private Properties //
TokenList.Tokens Property
Enumerates all tokens in the list. |
private IEnumerable<Token> Tokens { get { for (Token token = HeadToken.Next; token != TailToken; token = token.Next) yield return token; } } ////////////////////////////////////////////////////////////////////////// // Public Methods //
TokenList Constructor
Initializes an instance of this class. |
public TokenList() { m_headToken = new Token(TokenType.Head, ""); m_tailToken = new Token(TokenType.Tail, ""); m_tailToken.InsertAfter(m_headToken); }
TokenList.Append Method (Token)
Appends a given token to the end of the token list.
Parameters
token The token to append. |
public void Append(Token token) { token.InsertBefore(TailToken); }
TokenList.Append Method (TokenList)
Moves the tokens of a given token list to the end of this token list.
Parameters
tokenList Tokens in tokenList are are removed from the tokenList and added to the end of the token list of this object. |
public void Append(TokenList tokenList) { // nothing to do if <tokenList> is empty if (tokenList.IsEmpty) return; // if <this> is empty, just transfer the tokens from <tokenList> to // here if (IsEmpty) { // move tokens from <tokenList> to <this> m_headToken.m_next = tokenList.m_headToken.m_next; m_tailToken.m_previous = tokenList.m_tailToken.m_previous; m_headToken.m_next.m_previous = m_headToken; m_tailToken.m_previous.m_next = m_tailToken; m_headToken.m_next = tokenList.m_headToken.m_next; // mark <tokenList> as empty tokenList.m_headToken.m_next = tokenList.m_tailToken; tokenList.m_tailToken.m_previous = tokenList.m_headToken; return; } // neither list is empty -- splice <tokenList> to the end of <this>... // move tokens from <tokenList> to <this> m_tailToken.m_previous.m_next = tokenList.m_headToken.m_next; tokenList.m_headToken.m_next.m_previous = m_tailToken.m_previous; tokenList.m_tailToken.m_previous.m_next = m_tailToken; m_tailToken.m_previous = tokenList.m_tailToken.m_previous; // mark <tokenList> as empty tokenList.m_headToken.m_next = tokenList.m_tailToken; tokenList.m_tailToken.m_previous = tokenList.m_headToken; // check internal consistency -- warning: this is O(n) tokenList.AssertConsistency(); AssertConsistency(); }
TokenList.AppendCopy Method
Copies the tokens of a given token list to the end of this token list.
Parameters
tokenList Tokens in tokenList are are copied from the tokenList. The copies are added to the end of the token list of this object. |
public void AppendCopy(TokenList tokenList) { foreach (Token token in tokenList) Append(token.Clone()); }
TokenList.ExtractTokens Method
Parameters
firstToken The first token to move lastToken The last token to move
Remarks
Assuming firstToken and lastToken are in a linked list of tokens, and firstToken precedes lastToken, move all tokens between firstToken and lastToken, inclusive, into a new TokenList, which is returned. |
public static TokenList ExtractTokens(Token firstToken, Token lastToken) { TokenList tokenList = new TokenList(); Token afterLastToken = lastToken.Next; Token token = firstToken; while (token != afterLastToken) { Token nextToken = token.Next; token.RemoveFromList(); tokenList.Append(token); token = nextToken; } return tokenList; }
TokenList.TrimBlankTokens Method
Removes leading and trailing white space and newline tokens from the list.
Return Value
If the resulting list is empty, null is returned. Otherwise the list is returned. Note that, in either case, trimming of white space tokens is done in-place. |
public TokenList TrimBlankTokens() { while (HeadToken.Next.IsBlank()) HeadToken.Next.RemoveFromList(); while (TailToken.Previous.IsBlank()) TailToken.Previous.RemoveFromList(); if (IsEmpty) return null; else return this; }
TokenList.ToString Method (int, bool, bool)
Returns a string representation of the contents of the list.
Parameters
maxLength The maximum number of characters to return, not including the ellipsis (...) which is added if the returned value would exceed this length. multiLine true if the return value should include newlines, false if newlines should be converted to spaces. preserveSpaces true to preserve spaces, false to collapse multiple adjacent spaces into a single space. |
public string ToString(int maxLength, bool multiLine, bool preserveSpaces) { return HeadToken.Next.TokenListToString(maxLength, multiLine, preserveSpaces); }
TokenList.ToString Method ()
Returns a string representation of the contents of the list. |
public override string ToString() { return ToString(int.MaxValue, true, false); }
TokenList.ToString Method (TokenList)
Returns a string representation of the contents of the given list, or "" if the list is null.
Parameters
list The list to return a string representative of. |
public static string ToString(TokenList list) { if (list == null) return ""; else return list.ToString(); }
TokenList.ToString Method (TokenList, int, bool)
Returns a string representation of the contents of the given list, or "" if the list is null.
Parameters
list The list to return a string representative of. maxLength The maximum number of characters to return, not including the ellipsis (...) which is added if the returned value would exceed this length. multiLine true if the return value should include newlines, false if newlines should be converted to spaces. |
public static string ToString(TokenList list, int maxLength, bool multiLine) { if (list == null) return ""; else return list.ToString(maxLength, multiLine, false); }
public static TokenList Join(char delimiter, params TokenList[] tokenLists) { TokenList result = new TokenList(); Token delimiterToken = new Token(delimiter); foreach (TokenList tokenList in tokenLists) { if (tokenList != null) { if (!result.IsEmpty) result.Append(delimiterToken); foreach (Token token in tokenList) result.Append(token.Clone()); } } return result; }
public static TokenList Join(params TokenList[] tokenLists) { TokenList result = new TokenList(); foreach (TokenList tokenList in tokenLists) { if (tokenList != null) { foreach (Token token in tokenList) result.Append(token.Clone()); } } return result; }
public static TokenList JoinWithSeparator(TokenList separator, params TokenList[] tokenLists) { TokenList result = new TokenList(); bool first = true; foreach (TokenList tokenList in tokenLists) { if (first) first = false; else { foreach (Token token in separator) result.Append(token.Clone()); } if (tokenList != null) { foreach (Token token in tokenList) result.Append(token.Clone()); } } return result; }
TokenList.Join Method (TokenList, char, TokenList)
Creates a new TokenList containing the concatenation of two other TokenList values, with a delimiter token in between.
Parameters
tokenList1 The first TokenList value. If this value is null, it is not used. delimiter The delimiter to use between the first and second TokenList values. If either tokenList1 or tokenList2 are null, delimiter is not used. tokenList2 The second TokenList value. If this value is null, it is not used.
Return Value
If both tokenList1 and tokenList2 are null, null is returned. If one or the other of tokenList1 and tokenList2 are null, but not both, the non-null TokenList is returned -- not a clone of that TokenList. Otherwise, both lists are cloned, a new token containing delimiter inserted in between, and the result is returned. |
public static TokenList Join(TokenList tokenList1, char delimiter, TokenList tokenList2) { if (tokenList1 == null) { if (tokenList2 == null) return null; else return tokenList2; } else { if (tokenList2 == null) return tokenList1; else { TokenList result = new TokenList(); foreach (Token token in tokenList1) result.Append(token.Clone()); result.Append(new Token(delimiter)); foreach (Token token in tokenList2) result.Append(token.Clone()); return result; } } }
TokenList.Join Method (TokenList, char, Token)
Creates a new TokenList containing the concatenation of a given TokenList value and a Token value, with a delimiter token in between.
Parameters
tokenList1 The first value to concatenate. If this value is null, it is not used. delimiter The delimiter to use between tokenList1 and token2. If either tokenList1 or token2 are null, delimiter is not used. token2 The second value to concatenate. If this value is null, it is not used.
Return Value
If both tokenList1 and token2 are null, null is returned. If tokenList1 is null but token2 is not, a new list containing only a clone of token2 is returned. If tokenList1 is not null but token2 is null, tokenList1 itself is returned -- not a clone of that TokenList. Otherwise, both lists are cloned, a new token containing delimiter inserted in between, and the result is returned. |
public static TokenList Join(TokenList tokenList1, char delimiter, Token token2) { if (tokenList1 == null) { if (token2 == null) return null; else { TokenList result = new TokenList(); result.Append(token2.Clone()); return result; } } else { if (token2 == null) return tokenList1; else { TokenList result = new TokenList(); foreach (Token token in tokenList1) result.Append(token.Clone()); result.Append(new Token(delimiter)); result.Append(token2.Clone()); return result; } } }
TokenList.Compare Method
Compares two TokenList objects and returns an indication of their relative values.
Parameters
Return Value
A signed integer indicating the relative values of list1 and list2: a value less than zero if list1 is less than list2; zero if list1 is equal to list2; a value greater than zero if list1 is greater than list2.
Remarks
|
public static int Compare(TokenList list1, TokenList list2) { if (list1 == null) { if (list2 != null) return -1; // <list1> is lesser else return 0; // equal } else { if (list2 == null) return 1; // <list1> is greater else return list1.CompareTo(list2); } }
TokenList.operator == Method
Returns true if two TokenList objects contain tokens for which Token.Equals returns true. |
static public bool operator == (TokenList list1, TokenList list2) { if ((object) list1 == null) return ((object) list2 == null); else if ((object) list2 == null) return false; else return (Compare(list1, list2) == 0); }
TokenList.operator != Method
Returns false if two TokenList objects contain tokens for which Token.Equals returns true. |
static public bool operator != (TokenList list1, TokenList list2) { if ((object) list1 == null) return ((object) list2 != null); else if ((object) list2 == null) return true; else return (Compare(list1, list2) != 0); }
TokenList.Equals Method
Returns true if Token.Equals returns true when comparing each Token in this TokenList with the corresponding Token in another TokenList.
Parameters
obj The object to compare with. |
public override bool Equals(object obj) { if (obj == null) return false; TokenList other = obj as TokenList; if (other == null) return false; else return (Compare(this, other) == 0); }
public override int GetHashCode() { int hashCode = 0; foreach (Token token in this) hashCode = hashCode ^ token.GetHashCode(); return hashCode; } ////////////////////////////////////////////////////////////////////////// // Internal Methods // #if DEBUG /// <summary> /// Dumps the token list to the console. /// </summary> /// internal void DumpToConsole() { foreach (Token token in this) Console.WriteLine(token.Dump()); } #endif
TokenList.AssertConsistency Method
Checks the internal consistency of the object. Warning: this is an O(n) operation. |
[Conditional("DEBUG")] internal void AssertConsistency() { // check the head and tail tokens Debug.Assert(m_headToken.m_previous == null); Debug.Assert(m_tailToken.m_next == null); if (IsEmpty) { Debug.Assert(m_headToken.m_next == m_tailToken); Debug.Assert(m_tailToken.m_previous == m_headToken); } else { Debug.Assert(m_headToken.m_next.m_previous == m_headToken); Debug.Assert(m_tailToken.m_previous.m_next == m_tailToken); } // check each token in the list foreach (Token token in Tokens) { Debug.Assert(token.m_previous.m_next == token); Debug.Assert(token.m_next.m_previous == token); } } ////////////////////////////////////////////////////////////////////////// // IEnumerable<Token> Implementation //
TokenList.IEnumerable<Token>.GetEnumerator Method ()
Returns an enumerator that iterates over all tokens in this token list. |
IEnumerator<Token> IEnumerable<Token>.GetEnumerator()
{
return Tokens.GetEnumerator();
}
//////////////////////////////////////////////////////////////////////////
// IEnumerable Implementation
//
TokenList.System.Collections.IEnumerable.GetEnumerator Method ()
Returns an enumerator that iterates over all tokens in this token list. |
System.Collections.IEnumerator
System.Collections.IEnumerable.GetEnumerator()
{
return Tokens.GetEnumerator();
}
//////////////////////////////////////////////////////////////////////////
// IComparable Implementation
//
TokenList.CompareTo Method
Compares this instance to a specified TokenList object and returns an indication of their relative values.
Parameters
other The TokenList to compare to.
Return Value
A signed integer indicating the relative values of this instance and other: a value less than zero if this instance is less than other; zero if this instance is equal to other; a value greater than zero if this instance is greater than other. |
public int CompareTo(TokenList other) { Token thisToken = HeadToken.Next; Token otherToken = other.HeadToken.Next; while (true) { // see if we're at the end of either token list if (thisToken.IsTail) { if (otherToken.IsTail) return 0; // equal else return -1; // this is less } else if (otherToken.IsTail) return 1; // this is greater // compare current token of each list int compare = thisToken.CompareTo(otherToken); if (compare != 0) return compare; // current tokens are equal -- go to next tokens thisToken = thisToken.Next; otherToken = otherToken.Next; } } } }