Imported Upstream version 3.6.0

Former-commit-id: da6be194a6b1221998fc28233f2503bd61dd9d14

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/CSharpTokenizer.cs

@@ -0,0 +1,427 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Web.Razor.Parser;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Resources;
+using System.Web.Razor.Text;
+using System.Web.Razor.Tokenizer.Symbols;
+
+namespace System.Web.Razor.Tokenizer
+{
+    public class CSharpTokenizer : Tokenizer<CSharpSymbol, CSharpSymbolType>
+    {
+        private Dictionary<char, Func<CSharpSymbolType>> _operatorHandlers;
+
+        public CSharpTokenizer(ITextDocument source)
+            : base(source)
+        {
+            CurrentState = Data;
+
+            _operatorHandlers = new Dictionary<char, Func<CSharpSymbolType>>()
+            {
+                { '-', MinusOperator },
+                { '<', LessThanOperator },
+                { '>', GreaterThanOperator },
+                { '&', CreateTwoCharOperatorHandler(CSharpSymbolType.And, '=', CSharpSymbolType.AndAssign, '&', CSharpSymbolType.DoubleAnd) },
+                { '|', CreateTwoCharOperatorHandler(CSharpSymbolType.Or, '=', CSharpSymbolType.OrAssign, '|', CSharpSymbolType.DoubleOr) },
+                { '+', CreateTwoCharOperatorHandler(CSharpSymbolType.Plus, '=', CSharpSymbolType.PlusAssign, '+', CSharpSymbolType.Increment) },
+                { '=', CreateTwoCharOperatorHandler(CSharpSymbolType.Assign, '=', CSharpSymbolType.Equals, '>', CSharpSymbolType.GreaterThanEqual) },
+                { '!', CreateTwoCharOperatorHandler(CSharpSymbolType.Not, '=', CSharpSymbolType.NotEqual) },
+                { '%', CreateTwoCharOperatorHandler(CSharpSymbolType.Modulo, '=', CSharpSymbolType.ModuloAssign) },
+                { '*', CreateTwoCharOperatorHandler(CSharpSymbolType.Star, '=', CSharpSymbolType.MultiplyAssign) },
+                { ':', CreateTwoCharOperatorHandler(CSharpSymbolType.Colon, ':', CSharpSymbolType.DoubleColon) },
+                { '?', CreateTwoCharOperatorHandler(CSharpSymbolType.QuestionMark, '?', CSharpSymbolType.NullCoalesce) },
+                { '^', CreateTwoCharOperatorHandler(CSharpSymbolType.Xor, '=', CSharpSymbolType.XorAssign) },
+                { '(', () => CSharpSymbolType.LeftParenthesis },
+                { ')', () => CSharpSymbolType.RightParenthesis },
+                { '{', () => CSharpSymbolType.LeftBrace },
+                { '}', () => CSharpSymbolType.RightBrace },
+                { '[', () => CSharpSymbolType.LeftBracket },
+                { ']', () => CSharpSymbolType.RightBracket },
+                { ',', () => CSharpSymbolType.Comma },
+                { ';', () => CSharpSymbolType.Semicolon },
+                { '~', () => CSharpSymbolType.Tilde },
+                { '#', () => CSharpSymbolType.Hash }
+            };
+        }
+
+        protected override State StartState
+        {
+            get { return Data; }
+        }
+
+        public override CSharpSymbolType RazorCommentType
+        {
+            get { return CSharpSymbolType.RazorComment; }
+        }
+
+        public override CSharpSymbolType RazorCommentTransitionType
+        {
+            get { return CSharpSymbolType.RazorCommentTransition; }
+        }
+
+        public override CSharpSymbolType RazorCommentStarType
+        {
+            get { return CSharpSymbolType.RazorCommentStar; }
+        }
+
+        protected override CSharpSymbol CreateSymbol(SourceLocation start, string content, CSharpSymbolType type, IEnumerable<RazorError> errors)
+        {
+            return new CSharpSymbol(start, content, type, errors);
+        }
+
+        private StateResult Data()
+        {
+            if (ParserHelpers.IsNewLine(CurrentCharacter))
+            {
+                // CSharp Spec §2.3.1
+                bool checkTwoCharNewline = CurrentCharacter == '\r';
+                TakeCurrent();
+                if (checkTwoCharNewline && CurrentCharacter == '\n')
+                {
+                    TakeCurrent();
+                }
+                return Stay(EndSymbol(CSharpSymbolType.NewLine));
+            }
+            else if (ParserHelpers.IsWhitespace(CurrentCharacter))
+            {
+                // CSharp Spec §2.3.3
+                TakeUntil(c => !ParserHelpers.IsWhitespace(c));
+                return Stay(EndSymbol(CSharpSymbolType.WhiteSpace));
+            }
+            else if (CSharpHelpers.IsIdentifierStart(CurrentCharacter))
+            {
+                return Identifier();
+            }
+            else if (Char.IsDigit(CurrentCharacter))
+            {
+                return NumericLiteral();
+            }
+            switch (CurrentCharacter)
+            {
+                case '@':
+                    return AtSymbol();
+                case '\'':
+                    TakeCurrent();
+                    return Transition(() => QuotedLiteral('\'', CSharpSymbolType.CharacterLiteral));
+                case '"':
+                    TakeCurrent();
+                    return Transition(() => QuotedLiteral('"', CSharpSymbolType.StringLiteral));
+                case '.':
+                    if (Char.IsDigit(Peek()))
+                    {
+                        return RealLiteral();
+                    }
+                    return Stay(Single(CSharpSymbolType.Dot));
+                case '/':
+                    TakeCurrent();
+                    if (CurrentCharacter == '/')
+                    {
+                        TakeCurrent();
+                        return SingleLineComment();
+                    }
+                    else if (CurrentCharacter == '*')
+                    {
+                        TakeCurrent();
+                        return Transition(BlockComment);
+                    }
+                    else if (CurrentCharacter == '=')
+                    {
+                        TakeCurrent();
+                        return Stay(EndSymbol(CSharpSymbolType.DivideAssign));
+                    }
+                    else
+                    {
+                        return Stay(EndSymbol(CSharpSymbolType.Slash));
+                    }
+                default:
+                    return Stay(EndSymbol(Operator()));
+            }
+        }
+
+        private StateResult AtSymbol()
+        {
+            TakeCurrent();
+            if (CurrentCharacter == '"')
+            {
+                TakeCurrent();
+                return Transition(VerbatimStringLiteral);
+            }
+            else if (CurrentCharacter == '*')
+            {
+                return Transition(EndSymbol(CSharpSymbolType.RazorCommentTransition), AfterRazorCommentTransition);
+            }
+            else if (CurrentCharacter == '@')
+            {
+                // Could be escaped comment transition
+                return Transition(EndSymbol(CSharpSymbolType.Transition), () =>
+                {
+                    TakeCurrent();
+                    return Transition(EndSymbol(CSharpSymbolType.Transition), Data);
+                });
+            }
+            return Stay(EndSymbol(CSharpSymbolType.Transition));
+        }
+
+        private CSharpSymbolType Operator()
+        {
+            char first = CurrentCharacter;
+            TakeCurrent();
+            Func<CSharpSymbolType> handler;
+            if (_operatorHandlers.TryGetValue(first, out handler))
+            {
+                return handler();
+            }
+            return CSharpSymbolType.Unknown;
+        }
+
+        private CSharpSymbolType LessThanOperator()
+        {
+            if (CurrentCharacter == '=')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.LessThanEqual;
+            }
+            return CSharpSymbolType.LessThan;
+        }
+
+        private CSharpSymbolType GreaterThanOperator()
+        {
+            if (CurrentCharacter == '=')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.GreaterThanEqual;
+            }
+            return CSharpSymbolType.GreaterThan;
+        }
+
+        private CSharpSymbolType MinusOperator()
+        {
+            if (CurrentCharacter == '>')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.Arrow;
+            }
+            else if (CurrentCharacter == '-')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.Decrement;
+            }
+            else if (CurrentCharacter == '=')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.MinusAssign;
+            }
+            return CSharpSymbolType.Minus;
+        }
+
+        private Func<CSharpSymbolType> CreateTwoCharOperatorHandler(CSharpSymbolType typeIfOnlyFirst, char second, CSharpSymbolType typeIfBoth)
+        {
+            return () =>
+            {
+                if (CurrentCharacter == second)
+                {
+                    TakeCurrent();
+                    return typeIfBoth;
+                }
+                return typeIfOnlyFirst;
+            };
+        }
+
+        private Func<CSharpSymbolType> CreateTwoCharOperatorHandler(CSharpSymbolType typeIfOnlyFirst, char option1, CSharpSymbolType typeIfOption1, char option2, CSharpSymbolType typeIfOption2)
+        {
+            return () =>
+            {
+                if (CurrentCharacter == option1)
+                {
+                    TakeCurrent();
+                    return typeIfOption1;
+                }
+                else if (CurrentCharacter == option2)
+                {
+                    TakeCurrent();
+                    return typeIfOption2;
+                }
+                return typeIfOnlyFirst;
+            };
+        }
+
+        private StateResult VerbatimStringLiteral()
+        {
+            TakeUntil(c => c == '"');
+            if (CurrentCharacter == '"')
+            {
+                TakeCurrent();
+                if (CurrentCharacter == '"')
+                {
+                    TakeCurrent();
+                    // Stay in the literal, this is an escaped "
+                    return Stay();
+                }
+            }
+            else if (EndOfFile)
+            {
+                CurrentErrors.Add(new RazorError(RazorResources.ParseError_Unterminated_String_Literal, CurrentStart));
+            }
+            return Transition(EndSymbol(CSharpSymbolType.StringLiteral), Data);
+        }
+
+        private StateResult QuotedLiteral(char quote, CSharpSymbolType literalType)
+        {
+            TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
+            if (CurrentCharacter == '\\')
+            {
+                TakeCurrent(); // Take the '\'
+                TakeCurrent(); // Take the next char as well (multi-char escapes don't matter)
+                return Stay();
+            }
+            else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter))
+            {
+                CurrentErrors.Add(new RazorError(RazorResources.ParseError_Unterminated_String_Literal, CurrentStart));
+            }
+            else
+            {
+                TakeCurrent(); // No-op if at EOF
+            }
+            return Transition(EndSymbol(literalType), Data);
+        }
+
+        // CSharp Spec §2.3.2
+        private StateResult BlockComment()
+        {
+            TakeUntil(c => c == '*');
+            if (EndOfFile)
+            {
+                CurrentErrors.Add(new RazorError(RazorResources.ParseError_BlockComment_Not_Terminated, CurrentStart));
+                return Transition(EndSymbol(CSharpSymbolType.Comment), Data);
+            }
+            if (CurrentCharacter == '*')
+            {
+                TakeCurrent();
+                if (CurrentCharacter == '/')
+                {
+                    TakeCurrent();
+                    return Transition(EndSymbol(CSharpSymbolType.Comment), Data);
+                }
+            }
+            return Stay();
+        }
+
+        // CSharp Spec §2.3.2
+        private StateResult SingleLineComment()
+        {
+            TakeUntil(c => ParserHelpers.IsNewLine(c));
+            return Stay(EndSymbol(CSharpSymbolType.Comment));
+        }
+
+        // CSharp Spec §2.4.4
+        private StateResult NumericLiteral()
+        {
+            if (TakeAll("0x", caseSensitive: true))
+            {
+                return HexLiteral();
+            }
+            else
+            {
+                return DecimalLiteral();
+            }
+        }
+
+        private StateResult HexLiteral()
+        {
+            TakeUntil(c => !ParserHelpers.IsHexDigit(c));
+            TakeIntegerSuffix();
+            return Stay(EndSymbol(CSharpSymbolType.IntegerLiteral));
+        }
+
+        private StateResult DecimalLiteral()
+        {
+            TakeUntil(c => !Char.IsDigit(c));
+            if (CurrentCharacter == '.' && Char.IsDigit(Peek()))
+            {
+                return RealLiteral();
+            }
+            else if (CSharpHelpers.IsRealLiteralSuffix(CurrentCharacter) ||
+                     CurrentCharacter == 'E' || CurrentCharacter == 'e')
+            {
+                return RealLiteralExponentPart();
+            }
+            else
+            {
+                TakeIntegerSuffix();
+                return Stay(EndSymbol(CSharpSymbolType.IntegerLiteral));
+            }
+        }
+
+        private StateResult RealLiteralExponentPart()
+        {
+            if (CurrentCharacter == 'E' || CurrentCharacter == 'e')
+            {
+                TakeCurrent();
+                if (CurrentCharacter == '+' || CurrentCharacter == '-')
+                {
+                    TakeCurrent();
+                }
+                TakeUntil(c => !Char.IsDigit(c));
+            }
+            if (CSharpHelpers.IsRealLiteralSuffix(CurrentCharacter))
+            {
+                TakeCurrent();
+            }
+            return Stay(EndSymbol(CSharpSymbolType.RealLiteral));
+        }
+
+        // CSharp Spec §2.4.4.3
+        private StateResult RealLiteral()
+        {
+            AssertCurrent('.');
+            TakeCurrent();
+            Debug.Assert(Char.IsDigit(CurrentCharacter));
+            TakeUntil(c => !Char.IsDigit(c));
+            return RealLiteralExponentPart();
+        }
+
+        private void TakeIntegerSuffix()
+        {
+            if (Char.ToLowerInvariant(CurrentCharacter) == 'u')
+            {
+                TakeCurrent();
+                if (Char.ToLowerInvariant(CurrentCharacter) == 'l')
+                {
+                    TakeCurrent();
+                }
+            }
+            else if (Char.ToLowerInvariant(CurrentCharacter) == 'l')
+            {
+                TakeCurrent();
+                if (Char.ToLowerInvariant(CurrentCharacter) == 'u')
+                {
+                    TakeCurrent();
+                }
+            }
+        }
+
+        // CSharp Spec §2.4.2
+        private StateResult Identifier()
+        {
+            Debug.Assert(CSharpHelpers.IsIdentifierStart(CurrentCharacter));
+            TakeCurrent();
+            TakeUntil(c => !CSharpHelpers.IsIdentifierPart(c));
+            CSharpSymbol sym = null;
+            if (HaveContent)
+            {
+                CSharpKeyword? kwd = CSharpKeywordDetector.SymbolTypeForIdentifier(Buffer.ToString());
+                CSharpSymbolType type = CSharpSymbolType.Identifier;
+                if (kwd != null)
+                {
+                    type = CSharpSymbolType.Keyword;
+                }
+                sym = new CSharpSymbol(CurrentStart, Buffer.ToString(), type) { Keyword = kwd };
+            }
+            StartSymbol();
+            return Stay(sym);
+        }
+    }
+}