ExpressionLexer.cs source code in C# .NET

Source code for the .NET framework in C#

                        

Code:

/ 4.0 / 4.0 / untmp / DEVDIV_TFS / Dev10 / Releases / RTMRel / ndp / fx / src / DataWeb / Server / System / Data / Services / Parsing / ExpressionLexer.cs / 1305376 / ExpressionLexer.cs

                            //---------------------------------------------------------------------- 
// 
//      Copyright (c) Microsoft Corporation.  All rights reserved.
// 
//  
//      Provides a type to tokenize text.
//  
// 
// @owner  [....]
//--------------------------------------------------------------------- 

namespace System.Data.Services.Parsing
{
    using System; 
    using System.Diagnostics;
    using System.Text; 
 
    /// Use this class to parse an expression in the Astoria URI format.
    ///  
    /// Literals (non-normative "handy" reference - see spec for correct expression):
    /// Null        null
    /// Boolean     true | false
    /// Int32       (digit+) 
    /// Int64       (digit+)(L|l)
    /// Decimal     (digit+ ['.' digit+])(M|m) 
    /// Float       (digit+ ['.' digit+][e|E [+|-] digit+)(f|F) 
    /// Double      (digit+ ['.' digit+][e|E [+|-] digit+)
    /// String      "'" .* "'" 
    /// DateTime    datetime"'"dddd-dd-dd[T|' ']dd:mm[ss[.fffffff]]"'"
    /// Binary      (binary|X)'digit*'
    /// GUID        guid'digit*'
    ///  
    [DebuggerDisplay("ExpressionLexer ({text} @ {textPos} [{token}]")]
    internal class ExpressionLexer 
    { 
        #region Private fields.
 
        /// Suffix for single literals.
        private const char SingleSuffixLower = 'f';

        /// Suffix for single literals. 
        private const char SingleSuffixUpper = 'F';
 
        /// Text being parsed. 
        private readonly string text;
 
        /// Length of text being parsed.
        private readonly int textLen;

        /// Position on text being parsed. 
        private int textPos;
 
        /// Character being processed. 
        private char ch;
 
        /// Token being processed.
        private Token token;

        #endregion Private fields. 

        #region Constructors. 
 
        /// Initializes a new .
        /// Expression to parse. 
        internal ExpressionLexer(string expression)
        {
            Debug.Assert(expression != null, "expression != null");
 
            this.text = expression;
            this.textLen = this.text.Length; 
            this.SetTextPos(0); 
            this.NextToken();
        } 

        #endregion Constructors.

        #region Internal properties. 

        /// Token being processed. 
        internal Token CurrentToken 
        {
            get { return this.token; } 
            set { this.token = value; }
        }

        /// Text being parsed. 
        internal string ExpressionText
        { 
            get { return this.text; } 
        }
 
        /// Position on text being parsed.
        internal int Position
        {
            get { return this.token.Position; } 
        }
 
        #endregion Internal properties. 

        #region Internal methods. 

        /// Whether the specified token identifier is a numeric literal.
        /// Token to check.
        /// true if it's a numeric literal; false otherwise. 
        internal static bool IsNumeric(TokenId id)
        { 
            return 
                id == TokenId.IntegerLiteral || id == TokenId.DecimalLiteral ||
                id == TokenId.DoubleLiteral || id == TokenId.Int64Literal || 
                id == TokenId.SingleLiteral;
        }

        /// Reads the next token, skipping whitespace as necessary. 
        internal void NextToken()
        { 
            while (Char.IsWhiteSpace(this.ch)) 
            {
                this.NextChar(); 
            }

            TokenId t;
            int tokenPos = this.textPos; 
            switch (this.ch)
            { 
                case '(': 
                    this.NextChar();
                    t = TokenId.OpenParen; 
                    break;
                case ')':
                    this.NextChar();
                    t = TokenId.CloseParen; 
                    break;
                case ',': 
                    this.NextChar(); 
                    t = TokenId.Comma;
                    break; 
                case '-':
                    bool hasNext = this.textPos + 1 < this.textLen;
                    if (hasNext && Char.IsDigit(this.text[this.textPos + 1]))
                    { 
                        this.NextChar();
                        t = this.ParseFromDigit(); 
                        if (IsNumeric(t)) 
                        {
                            break; 
                        }

                        // If it looked like a numeric but wasn't (because it was a binary 0x... value for example),
                        // we'll rewind and fall through to a simple '-' token. 
                        this.SetTextPos(tokenPos);
                    } 
                    else if (hasNext && this.text[tokenPos + 1] == XmlConstants.XmlInfinityLiteral[0]) 
                    {
                        this.NextChar(); 
                        this.ParseIdentifier();
                        string currentIdentifier = this.text.Substring(tokenPos + 1, this.textPos - tokenPos - 1);

                        if (IsInfinityLiteralDouble(currentIdentifier)) 
                        {
                            t = TokenId.DoubleLiteral; 
                            break; 
                        }
                        else if (IsInfinityLiteralSingle(currentIdentifier)) 
                        {
                            t = TokenId.SingleLiteral;
                            break;
                        } 

                        // If it looked like '-INF' but wasn't we'll rewind and fall through to a simple '-' token. 
                        this.SetTextPos(tokenPos); 
                    }
 
                    this.NextChar();
                    t = TokenId.Minus;
                    break;
                case '=': 
                    this.NextChar();
                    t = TokenId.Equal; 
                    break; 
                case '/':
                    this.NextChar(); 
                    t = TokenId.Slash;
                    break;
                case '?':
                    this.NextChar(); 
                    t = TokenId.Question;
                    break; 
                case '.': 
                    this.NextChar();
                    t = TokenId.Dot; 
                    break;
                case '\'':
                    char quote = this.ch;
                    do 
                    {
                        this.NextChar(); 
                        while (this.textPos < this.textLen && this.ch != quote) 
                        {
                            this.NextChar(); 
                        }

                        if (this.textPos == this.textLen)
                        { 
                            throw ParseError(Strings.RequestQueryParser_UnterminatedStringLiteral(this.textPos, this.text));
                        } 
 
                        this.NextChar();
                    } 
                    while (this.ch == quote);
                    t = TokenId.StringLiteral;
                    break;
                case '*': 
                    this.NextChar();
                    t = TokenId.Star; 
                    break; 
                default:
                    if (Char.IsLetter(this.ch) || this.ch == '_') 
                    {
                        this.ParseIdentifier();
                        t = TokenId.Identifier;
                        break; 
                    }
 
                    if (Char.IsDigit(this.ch)) 
                    {
                        t = this.ParseFromDigit(); 
                        break;
                    }

                    if (this.textPos == this.textLen) 
                    {
                        t = TokenId.End; 
                        break; 
                    }
 
                    throw ParseError(Strings.RequestQueryParser_InvalidCharacter(this.ch, this.textPos));
            }

            this.token.Id = t; 
            this.token.Text = this.text.Substring(tokenPos, this.textPos - tokenPos);
            this.token.Position = tokenPos; 
 
            // Handle type-prefixed literals such as binary, datetime or guid.
            this.HandleTypePrefixedLiterals(); 

            // Handle keywords.
            if (this.token.Id == TokenId.Identifier)
            { 
                if (IsInfinityOrNaNDouble(this.token.Text))
                { 
                    this.token.Id = TokenId.DoubleLiteral; 
                }
                else if (IsInfinityOrNanSingle(this.token.Text)) 
                {
                    this.token.Id = TokenId.SingleLiteral;
                }
                else if (this.token.Text == ExpressionConstants.KeywordTrue || this.token.Text == ExpressionConstants.KeywordFalse) 
                {
                    this.token.Id = TokenId.BooleanLiteral; 
                } 
                else if (this.token.Text == ExpressionConstants.KeywordNull)
                { 
                    this.token.Id = TokenId.NullLiteral;
                }
            }
        } 

        ///  
        /// Starting from an identifier, reads a sequence of dots and 
        /// identifiers, and returns the text for it, with whitespace
        /// stripped. 
        /// 
        /// The dotted identifier starting at the current identifie.
        internal string ReadDottedIdentifier()
        { 
            this.ValidateToken(TokenId.Identifier);
            StringBuilder builder = null; 
            string result = this.CurrentToken.Text; 
            this.NextToken();
            while (this.CurrentToken.Id == TokenId.Dot) 
            {
                this.NextToken();
                this.ValidateToken(TokenId.Identifier);
                if (builder == null) 
                {
                    builder = new StringBuilder(result, result.Length + 1 + this.CurrentToken.Text.Length); 
                } 

                builder.Append('.'); 
                builder.Append(this.CurrentToken.Text);
                this.NextToken();
            }
 
            if (builder != null)
            { 
                result = builder.ToString(); 
            }
 
            return result;
        }

        /// Returns the next token without advancing the lexer. 
        /// The next token.
        internal Token PeekNextToken() 
        { 
            int savedTextPos = this.textPos;
            char savedChar = this.ch; 
            Token savedToken = this.token;

            this.NextToken();
            Token result = this.token; 

            this.textPos = savedTextPos; 
            this.ch = savedChar; 
            this.token = savedToken;
 
            return result;
        }

        /// Validates the current token is of the specified kind. 
        /// Expected token kind.
        internal void ValidateToken(TokenId t) 
        { 
            if (this.token.Id != t)
            { 
                throw ParseError(Strings.RequestQueryParser_SyntaxError(this.textPos));
            }
        }
 
        #endregion Internal methods.
 
        #region Private methods. 

        /// Checks if the  is INF or NaN. 
        /// Input token.
        /// true if match found, false otherwise.
        private static bool IsInfinityOrNaNDouble(string tokenText)
        { 
            if (tokenText.Length == 3)
            { 
                if (tokenText[0] == XmlConstants.XmlInfinityLiteral[0]) 
                {
                    return IsInfinityLiteralDouble(tokenText); 
                }
                else
                if (tokenText[0] == XmlConstants.XmlNaNLiteral[0])
                { 
                    return String.CompareOrdinal(tokenText, 0, XmlConstants.XmlNaNLiteral, 0, 3) == 0;
                } 
            } 

            return false; 
        }

        /// 
        /// Checks whether  equals to 'INF' 
        /// 
        /// Text to look in. 
        /// true if the substring is equal using an ordinal comparison; false otherwise. 
        private static bool IsInfinityLiteralDouble(string text)
        { 
            Debug.Assert(text != null, "text != null");
            return String.CompareOrdinal(text, 0, XmlConstants.XmlInfinityLiteral, 0, text.Length) == 0;
        }
 
        /// Checks if the  is INFf/INFF or NaNf/NaNF.
        /// Input token. 
        /// true if match found, false otherwise. 
        private static bool IsInfinityOrNanSingle(string tokenText)
        { 
            if (tokenText.Length == 4)
            {
                if (tokenText[0] == XmlConstants.XmlInfinityLiteral[0])
                { 
                    return IsInfinityLiteralSingle(tokenText);
                } 
                else if (tokenText[0] == XmlConstants.XmlNaNLiteral[0]) 
                {
                    return (tokenText[3] == ExpressionLexer.SingleSuffixLower || tokenText[3] == ExpressionLexer.SingleSuffixUpper) && 
                            String.CompareOrdinal(tokenText, 0, XmlConstants.XmlNaNLiteral, 0, 3) == 0;
                }
            }
 
            return false;
        } 
 
        /// 
        /// Checks whether  EQUALS to 'INFf' or 'INFF' at position 
        /// 
        /// Text to look in.
        /// true if the substring is equal using an ordinal comparison; false otherwise.
        private static bool IsInfinityLiteralSingle(string text) 
        {
            Debug.Assert(text != null, "text != null"); 
            return text.Length == 4 && 
                   (text[3] == ExpressionLexer.SingleSuffixLower || text[3] == ExpressionLexer.SingleSuffixUpper) &&
                   String.CompareOrdinal(text, 0, XmlConstants.XmlInfinityLiteral, 0, 3) == 0; 
        }

        /// Creates an exception for a parse error.
        /// Message text. 
        /// A new Exception.
        private static Exception ParseError(string message) 
        { 
            return DataServiceException.CreateSyntaxError(message);
        } 

        /// Handles lexemes that are formed by an identifier followed by a quoted string.
        /// This method modified the token field as necessary.
        private void HandleTypePrefixedLiterals() 
        {
            TokenId id = this.token.Id; 
            if (id != TokenId.Identifier) 
            {
                return; 
            }

            bool quoteFollows = this.ch == '\'';
            if (!quoteFollows) 
            {
                return; 
            } 

            string tokenText = this.token.Text; 
            if (String.Equals(tokenText, "datetime", StringComparison.OrdinalIgnoreCase))
            {
                id = TokenId.DateTimeLiteral;
            } 
            else if (String.Equals(tokenText, "guid", StringComparison.OrdinalIgnoreCase))
            { 
                id = TokenId.GuidLiteral; 
            }
            else if (String.Equals(tokenText, "binary", StringComparison.OrdinalIgnoreCase) || tokenText == "X" || tokenText == "x") 
            {
                id = TokenId.BinaryLiteral;
            }
            else 
            {
                return; 
            } 

            int tokenPos = this.token.Position; 
            do
            {
                this.NextChar();
            } 
            while (this.ch != '\0' && this.ch != '\'');
 
            if (this.ch == '\0') 
            {
                throw ParseError(Strings.RequestQueryParser_UnterminatedLiteral(this.textPos, this.text)); 
            }

            this.NextChar();
            this.token.Id = id; 
            this.token.Text = this.text.Substring(tokenPos, this.textPos - tokenPos);
        } 
 
        /// Advanced to the next character.
        private void NextChar() 
        {
            if (this.textPos < this.textLen)
            {
                this.textPos++; 
            }
 
            this.ch = this.textPos < this.textLen ? this.text[this.textPos] : '\0'; 
        }
 
        /// Parses a token that starts with a digit.
        /// The kind of token recognized.
        private TokenId ParseFromDigit()
        { 
            Debug.Assert(Char.IsDigit(this.ch), "Char.IsDigit(this.ch)");
            TokenId result; 
            char startChar = this.ch; 
            this.NextChar();
            if (startChar == '0' && this.ch == 'x' || this.ch == 'X') 
            {
                result = TokenId.BinaryLiteral;
                do
                { 
                    this.NextChar();
                } 
                while (WebConvert.IsCharHexDigit(this.ch)); 
            }
            else 
            {
                result = TokenId.IntegerLiteral;
                while (Char.IsDigit(this.ch))
                { 
                    this.NextChar();
                } 
 
                if (this.ch == '.')
                { 
                    result = TokenId.DoubleLiteral;
                    this.NextChar();
                    this.ValidateDigit();
 
                    do
                    { 
                        this.NextChar(); 
                    }
                    while (Char.IsDigit(this.ch)); 
                }

                if (this.ch == 'E' || this.ch == 'e')
                { 
                    result = TokenId.DoubleLiteral;
                    this.NextChar(); 
                    if (this.ch == '+' || this.ch == '-') 
                    {
                        this.NextChar(); 
                    }

                    this.ValidateDigit();
                    do 
                    {
                        this.NextChar(); 
                    } 
                    while (Char.IsDigit(this.ch));
                } 

                if (this.ch == 'M' || this.ch == 'm')
                {
                    result = TokenId.DecimalLiteral; 
                    this.NextChar();
                } 
                else 
                if (this.ch == 'd' || this.ch == 'D')
                { 
                    result = TokenId.DoubleLiteral;
                    this.NextChar();
                }
                else if (this.ch == 'L' || this.ch == 'l') 
                {
                    result = TokenId.Int64Literal; 
                    this.NextChar(); 
                }
                else if (this.ch == 'f' || this.ch == 'F') 
                {
                    result = TokenId.SingleLiteral;
                    this.NextChar();
                } 
            }
 
            return result; 
        }
 
        /// Parses an identifier by advancing the current character.
        private void ParseIdentifier()
        {
            Debug.Assert(Char.IsLetter(this.ch) || this.ch == '_', "Char.IsLetter(this.ch) || this.ch == '_'"); 
            do
            { 
                this.NextChar(); 
            }
            while (Char.IsLetterOrDigit(this.ch) || this.ch == '_'); 
        }

        /// Sets the text position.
        /// New text position. 
        private void SetTextPos(int pos)
        { 
            this.textPos = pos; 
            this.ch = this.textPos < this.textLen ? this.text[this.textPos] : '\0';
        } 

        /// Validates the current character is a digit.
        private void ValidateDigit()
        { 
            if (!Char.IsDigit(this.ch))
            { 
                throw ParseError(Strings.RequestQueryParser_DigitExpected(this.textPos)); 
            }
        } 

        #endregion Private methods.
    }
} 

// File provided for Reference Use Only by Microsoft Corporation (c) 2007.
                        

Link Menu

Network programming in C#, Network Programming in VB.NET, Network Programming in .NET
This book is available now!
Buy at Amazon US or
Buy at Amazon UK