Imported Upstream version 3.6.0

Former-commit-id: da6be194a6b1221998fc28233f2503bd61dd9d14

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/CSharpHelpers.cs

@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Globalization;
+
+namespace System.Web.Razor.Tokenizer
+{
+    public static class CSharpHelpers
+    {
+        // CSharp Spec §2.4.2
+        public static bool IsIdentifierStart(char character)
+        {
+            return Char.IsLetter(character) ||
+                   character == '_' ||
+                   Char.GetUnicodeCategory(character) == UnicodeCategory.LetterNumber; // Ln
+        }
+
+        public static bool IsIdentifierPart(char character)
+        {
+            return Char.IsDigit(character) ||
+                   IsIdentifierStart(character) ||
+                   IsIdentifierPartByUnicodeCategory(character);
+        }
+
+        public static bool IsRealLiteralSuffix(char character)
+        {
+            return character == 'F' ||
+                   character == 'f' ||
+                   character == 'D' ||
+                   character == 'd' ||
+                   character == 'M' ||
+                   character == 'm';
+        }
+
+        private static bool IsIdentifierPartByUnicodeCategory(char character)
+        {
+            UnicodeCategory category = Char.GetUnicodeCategory(character);
+            return category == UnicodeCategory.NonSpacingMark || // Mn
+                   category == UnicodeCategory.SpacingCombiningMark || // Mc
+                   category == UnicodeCategory.ConnectorPunctuation || // Pc
+                   category == UnicodeCategory.Format; // Cf
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/CSharpKeywordDetector.cs

@@ -0,0 +1,101 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Web.Razor.Tokenizer.Symbols;
+
+namespace System.Web.Razor.Tokenizer
+{
+    internal static class CSharpKeywordDetector
+    {
+        private static readonly Dictionary<string, CSharpKeyword> _keywords = new Dictionary<string, CSharpKeyword>(StringComparer.Ordinal)
+        {
+            { "abstract", CSharpKeyword.Abstract },
+            { "byte", CSharpKeyword.Byte },
+            { "class", CSharpKeyword.Class },
+            { "delegate", CSharpKeyword.Delegate },
+            { "event", CSharpKeyword.Event },
+            { "fixed", CSharpKeyword.Fixed },
+            { "if", CSharpKeyword.If },
+            { "internal", CSharpKeyword.Internal },
+            { "new", CSharpKeyword.New },
+            { "override", CSharpKeyword.Override },
+            { "readonly", CSharpKeyword.Readonly },
+            { "short", CSharpKeyword.Short },
+            { "struct", CSharpKeyword.Struct },
+            { "try", CSharpKeyword.Try },
+            { "unsafe", CSharpKeyword.Unsafe },
+            { "volatile", CSharpKeyword.Volatile },
+            { "as", CSharpKeyword.As },
+            { "do", CSharpKeyword.Do },
+            { "is", CSharpKeyword.Is },
+            { "params", CSharpKeyword.Params },
+            { "ref", CSharpKeyword.Ref },
+            { "switch", CSharpKeyword.Switch },
+            { "ushort", CSharpKeyword.Ushort },
+            { "while", CSharpKeyword.While },
+            { "case", CSharpKeyword.Case },
+            { "const", CSharpKeyword.Const },
+            { "explicit", CSharpKeyword.Explicit },
+            { "float", CSharpKeyword.Float },
+            { "null", CSharpKeyword.Null },
+            { "sizeof", CSharpKeyword.Sizeof },
+            { "typeof", CSharpKeyword.Typeof },
+            { "implicit", CSharpKeyword.Implicit },
+            { "private", CSharpKeyword.Private },
+            { "this", CSharpKeyword.This },
+            { "using", CSharpKeyword.Using },
+            { "extern", CSharpKeyword.Extern },
+            { "return", CSharpKeyword.Return },
+            { "stackalloc", CSharpKeyword.Stackalloc },
+            { "uint", CSharpKeyword.Uint },
+            { "base", CSharpKeyword.Base },
+            { "catch", CSharpKeyword.Catch },
+            { "continue", CSharpKeyword.Continue },
+            { "double", CSharpKeyword.Double },
+            { "for", CSharpKeyword.For },
+            { "in", CSharpKeyword.In },
+            { "lock", CSharpKeyword.Lock },
+            { "object", CSharpKeyword.Object },
+            { "protected", CSharpKeyword.Protected },
+            { "static", CSharpKeyword.Static },
+            { "false", CSharpKeyword.False },
+            { "public", CSharpKeyword.Public },
+            { "sbyte", CSharpKeyword.Sbyte },
+            { "throw", CSharpKeyword.Throw },
+            { "virtual", CSharpKeyword.Virtual },
+            { "decimal", CSharpKeyword.Decimal },
+            { "else", CSharpKeyword.Else },
+            { "operator", CSharpKeyword.Operator },
+            { "string", CSharpKeyword.String },
+            { "ulong", CSharpKeyword.Ulong },
+            { "bool", CSharpKeyword.Bool },
+            { "char", CSharpKeyword.Char },
+            { "default", CSharpKeyword.Default },
+            { "foreach", CSharpKeyword.Foreach },
+            { "long", CSharpKeyword.Long },
+            { "void", CSharpKeyword.Void },
+            { "enum", CSharpKeyword.Enum },
+            { "finally", CSharpKeyword.Finally },
+            { "int", CSharpKeyword.Int },
+            { "out", CSharpKeyword.Out },
+            { "sealed", CSharpKeyword.Sealed },
+            { "true", CSharpKeyword.True },
+            { "goto", CSharpKeyword.Goto },
+            { "unchecked", CSharpKeyword.Unchecked },
+            { "interface", CSharpKeyword.Interface },
+            { "break", CSharpKeyword.Break },
+            { "checked", CSharpKeyword.Checked },
+            { "namespace", CSharpKeyword.Namespace }
+        };
+
+        public static CSharpKeyword? SymbolTypeForIdentifier(string id)
+        {
+            CSharpKeyword type;
+            if (!_keywords.TryGetValue(id, out type))
+            {
+                return null;
+            }
+            return type;
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/CSharpTokenizer.cs

@@ -0,0 +1,427 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Web.Razor.Parser;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Resources;
+using System.Web.Razor.Text;
+using System.Web.Razor.Tokenizer.Symbols;
+
+namespace System.Web.Razor.Tokenizer
+{
+    public class CSharpTokenizer : Tokenizer<CSharpSymbol, CSharpSymbolType>
+    {
+        private Dictionary<char, Func<CSharpSymbolType>> _operatorHandlers;
+
+        public CSharpTokenizer(ITextDocument source)
+            : base(source)
+        {
+            CurrentState = Data;
+
+            _operatorHandlers = new Dictionary<char, Func<CSharpSymbolType>>()
+            {
+                { '-', MinusOperator },
+                { '<', LessThanOperator },
+                { '>', GreaterThanOperator },
+                { '&', CreateTwoCharOperatorHandler(CSharpSymbolType.And, '=', CSharpSymbolType.AndAssign, '&', CSharpSymbolType.DoubleAnd) },
+                { '|', CreateTwoCharOperatorHandler(CSharpSymbolType.Or, '=', CSharpSymbolType.OrAssign, '|', CSharpSymbolType.DoubleOr) },
+                { '+', CreateTwoCharOperatorHandler(CSharpSymbolType.Plus, '=', CSharpSymbolType.PlusAssign, '+', CSharpSymbolType.Increment) },
+                { '=', CreateTwoCharOperatorHandler(CSharpSymbolType.Assign, '=', CSharpSymbolType.Equals, '>', CSharpSymbolType.GreaterThanEqual) },
+                { '!', CreateTwoCharOperatorHandler(CSharpSymbolType.Not, '=', CSharpSymbolType.NotEqual) },
+                { '%', CreateTwoCharOperatorHandler(CSharpSymbolType.Modulo, '=', CSharpSymbolType.ModuloAssign) },
+                { '*', CreateTwoCharOperatorHandler(CSharpSymbolType.Star, '=', CSharpSymbolType.MultiplyAssign) },
+                { ':', CreateTwoCharOperatorHandler(CSharpSymbolType.Colon, ':', CSharpSymbolType.DoubleColon) },
+                { '?', CreateTwoCharOperatorHandler(CSharpSymbolType.QuestionMark, '?', CSharpSymbolType.NullCoalesce) },
+                { '^', CreateTwoCharOperatorHandler(CSharpSymbolType.Xor, '=', CSharpSymbolType.XorAssign) },
+                { '(', () => CSharpSymbolType.LeftParenthesis },
+                { ')', () => CSharpSymbolType.RightParenthesis },
+                { '{', () => CSharpSymbolType.LeftBrace },
+                { '}', () => CSharpSymbolType.RightBrace },
+                { '[', () => CSharpSymbolType.LeftBracket },
+                { ']', () => CSharpSymbolType.RightBracket },
+                { ',', () => CSharpSymbolType.Comma },
+                { ';', () => CSharpSymbolType.Semicolon },
+                { '~', () => CSharpSymbolType.Tilde },
+                { '#', () => CSharpSymbolType.Hash }
+            };
+        }
+
+        protected override State StartState
+        {
+            get { return Data; }
+        }
+
+        public override CSharpSymbolType RazorCommentType
+        {
+            get { return CSharpSymbolType.RazorComment; }
+        }
+
+        public override CSharpSymbolType RazorCommentTransitionType
+        {
+            get { return CSharpSymbolType.RazorCommentTransition; }
+        }
+
+        public override CSharpSymbolType RazorCommentStarType
+        {
+            get { return CSharpSymbolType.RazorCommentStar; }
+        }
+
+        protected override CSharpSymbol CreateSymbol(SourceLocation start, string content, CSharpSymbolType type, IEnumerable<RazorError> errors)
+        {
+            return new CSharpSymbol(start, content, type, errors);
+        }
+
+        private StateResult Data()
+        {
+            if (ParserHelpers.IsNewLine(CurrentCharacter))
+            {
+                // CSharp Spec §2.3.1
+                bool checkTwoCharNewline = CurrentCharacter == '\r';
+                TakeCurrent();
+                if (checkTwoCharNewline && CurrentCharacter == '\n')
+                {
+                    TakeCurrent();
+                }
+                return Stay(EndSymbol(CSharpSymbolType.NewLine));
+            }
+            else if (ParserHelpers.IsWhitespace(CurrentCharacter))
+            {
+                // CSharp Spec §2.3.3
+                TakeUntil(c => !ParserHelpers.IsWhitespace(c));
+                return Stay(EndSymbol(CSharpSymbolType.WhiteSpace));
+            }
+            else if (CSharpHelpers.IsIdentifierStart(CurrentCharacter))
+            {
+                return Identifier();
+            }
+            else if (Char.IsDigit(CurrentCharacter))
+            {
+                return NumericLiteral();
+            }
+            switch (CurrentCharacter)
+            {
+                case '@':
+                    return AtSymbol();
+                case '\'':
+                    TakeCurrent();
+                    return Transition(() => QuotedLiteral('\'', CSharpSymbolType.CharacterLiteral));
+                case '"':
+                    TakeCurrent();
+                    return Transition(() => QuotedLiteral('"', CSharpSymbolType.StringLiteral));
+                case '.':
+                    if (Char.IsDigit(Peek()))
+                    {
+                        return RealLiteral();
+                    }
+                    return Stay(Single(CSharpSymbolType.Dot));
+                case '/':
+                    TakeCurrent();
+                    if (CurrentCharacter == '/')
+                    {
+                        TakeCurrent();
+                        return SingleLineComment();
+                    }
+                    else if (CurrentCharacter == '*')
+                    {
+                        TakeCurrent();
+                        return Transition(BlockComment);
+                    }
+                    else if (CurrentCharacter == '=')
+                    {
+                        TakeCurrent();
+                        return Stay(EndSymbol(CSharpSymbolType.DivideAssign));
+                    }
+                    else
+                    {
+                        return Stay(EndSymbol(CSharpSymbolType.Slash));
+                    }
+                default:
+                    return Stay(EndSymbol(Operator()));
+            }
+        }
+
+        private StateResult AtSymbol()
+        {
+            TakeCurrent();
+            if (CurrentCharacter == '"')
+            {
+                TakeCurrent();
+                return Transition(VerbatimStringLiteral);
+            }
+            else if (CurrentCharacter == '*')
+            {
+                return Transition(EndSymbol(CSharpSymbolType.RazorCommentTransition), AfterRazorCommentTransition);
+            }
+            else if (CurrentCharacter == '@')
+            {
+                // Could be escaped comment transition
+                return Transition(EndSymbol(CSharpSymbolType.Transition), () =>
+                {
+                    TakeCurrent();
+                    return Transition(EndSymbol(CSharpSymbolType.Transition), Data);
+                });
+            }
+            return Stay(EndSymbol(CSharpSymbolType.Transition));
+        }
+
+        private CSharpSymbolType Operator()
+        {
+            char first = CurrentCharacter;
+            TakeCurrent();
+            Func<CSharpSymbolType> handler;
+            if (_operatorHandlers.TryGetValue(first, out handler))
+            {
+                return handler();
+            }
+            return CSharpSymbolType.Unknown;
+        }
+
+        private CSharpSymbolType LessThanOperator()
+        {
+            if (CurrentCharacter == '=')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.LessThanEqual;
+            }
+            return CSharpSymbolType.LessThan;
+        }
+
+        private CSharpSymbolType GreaterThanOperator()
+        {
+            if (CurrentCharacter == '=')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.GreaterThanEqual;
+            }
+            return CSharpSymbolType.GreaterThan;
+        }
+
+        private CSharpSymbolType MinusOperator()
+        {
+            if (CurrentCharacter == '>')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.Arrow;
+            }
+            else if (CurrentCharacter == '-')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.Decrement;
+            }
+            else if (CurrentCharacter == '=')
+            {
+                TakeCurrent();
+                return CSharpSymbolType.MinusAssign;
+            }
+            return CSharpSymbolType.Minus;
+        }
+
+        private Func<CSharpSymbolType> CreateTwoCharOperatorHandler(CSharpSymbolType typeIfOnlyFirst, char second, CSharpSymbolType typeIfBoth)
+        {
+            return () =>
+            {
+                if (CurrentCharacter == second)
+                {
+                    TakeCurrent();
+                    return typeIfBoth;
+                }
+                return typeIfOnlyFirst;
+            };
+        }
+
+        private Func<CSharpSymbolType> CreateTwoCharOperatorHandler(CSharpSymbolType typeIfOnlyFirst, char option1, CSharpSymbolType typeIfOption1, char option2, CSharpSymbolType typeIfOption2)
+        {
+            return () =>
+            {
+                if (CurrentCharacter == option1)
+                {
+                    TakeCurrent();
+                    return typeIfOption1;
+                }
+                else if (CurrentCharacter == option2)
+                {
+                    TakeCurrent();
+                    return typeIfOption2;
+                }
+                return typeIfOnlyFirst;
+            };
+        }
+
+        private StateResult VerbatimStringLiteral()
+        {
+            TakeUntil(c => c == '"');
+            if (CurrentCharacter == '"')
+            {
+                TakeCurrent();
+                if (CurrentCharacter == '"')
+                {
+                    TakeCurrent();
+                    // Stay in the literal, this is an escaped "
+                    return Stay();
+                }
+            }
+            else if (EndOfFile)
+            {
+                CurrentErrors.Add(new RazorError(RazorResources.ParseError_Unterminated_String_Literal, CurrentStart));
+            }
+            return Transition(EndSymbol(CSharpSymbolType.StringLiteral), Data);
+        }
+
+        private StateResult QuotedLiteral(char quote, CSharpSymbolType literalType)
+        {
+            TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
+            if (CurrentCharacter == '\\')
+            {
+                TakeCurrent(); // Take the '\'
+                TakeCurrent(); // Take the next char as well (multi-char escapes don't matter)
+                return Stay();
+            }
+            else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter))
+            {
+                CurrentErrors.Add(new RazorError(RazorResources.ParseError_Unterminated_String_Literal, CurrentStart));
+            }
+            else
+            {
+                TakeCurrent(); // No-op if at EOF
+            }
+            return Transition(EndSymbol(literalType), Data);
+        }
+
+        // CSharp Spec §2.3.2
+        private StateResult BlockComment()
+        {
+            TakeUntil(c => c == '*');
+            if (EndOfFile)
+            {
+                CurrentErrors.Add(new RazorError(RazorResources.ParseError_BlockComment_Not_Terminated, CurrentStart));
+                return Transition(EndSymbol(CSharpSymbolType.Comment), Data);
+            }
+            if (CurrentCharacter == '*')
+            {
+                TakeCurrent();
+                if (CurrentCharacter == '/')
+                {
+                    TakeCurrent();
+                    return Transition(EndSymbol(CSharpSymbolType.Comment), Data);
+                }
+            }
+            return Stay();
+        }
+
+        // CSharp Spec §2.3.2
+        private StateResult SingleLineComment()
+        {
+            TakeUntil(c => ParserHelpers.IsNewLine(c));
+            return Stay(EndSymbol(CSharpSymbolType.Comment));
+        }
+
+        // CSharp Spec §2.4.4
+        private StateResult NumericLiteral()
+        {
+            if (TakeAll("0x", caseSensitive: true))
+            {
+                return HexLiteral();
+            }
+            else
+            {
+                return DecimalLiteral();
+            }
+        }
+
+        private StateResult HexLiteral()
+        {
+            TakeUntil(c => !ParserHelpers.IsHexDigit(c));
+            TakeIntegerSuffix();
+            return Stay(EndSymbol(CSharpSymbolType.IntegerLiteral));
+        }
+
+        private StateResult DecimalLiteral()
+        {
+            TakeUntil(c => !Char.IsDigit(c));
+            if (CurrentCharacter == '.' && Char.IsDigit(Peek()))
+            {
+                return RealLiteral();
+            }
+            else if (CSharpHelpers.IsRealLiteralSuffix(CurrentCharacter) ||
+                     CurrentCharacter == 'E' || CurrentCharacter == 'e')
+            {
+                return RealLiteralExponentPart();
+            }
+            else
+            {
+                TakeIntegerSuffix();
+                return Stay(EndSymbol(CSharpSymbolType.IntegerLiteral));
+            }
+        }
+
+        private StateResult RealLiteralExponentPart()
+        {
+            if (CurrentCharacter == 'E' || CurrentCharacter == 'e')
+            {
+                TakeCurrent();
+                if (CurrentCharacter == '+' || CurrentCharacter == '-')
+                {
+                    TakeCurrent();
+                }
+                TakeUntil(c => !Char.IsDigit(c));
+            }
+            if (CSharpHelpers.IsRealLiteralSuffix(CurrentCharacter))
+            {
+                TakeCurrent();
+            }
+            return Stay(EndSymbol(CSharpSymbolType.RealLiteral));
+        }
+
+        // CSharp Spec §2.4.4.3
+        private StateResult RealLiteral()
+        {
+            AssertCurrent('.');
+            TakeCurrent();
+            Debug.Assert(Char.IsDigit(CurrentCharacter));
+            TakeUntil(c => !Char.IsDigit(c));
+            return RealLiteralExponentPart();
+        }
+
+        private void TakeIntegerSuffix()
+        {
+            if (Char.ToLowerInvariant(CurrentCharacter) == 'u')
+            {
+                TakeCurrent();
+                if (Char.ToLowerInvariant(CurrentCharacter) == 'l')
+                {
+                    TakeCurrent();
+                }
+            }
+            else if (Char.ToLowerInvariant(CurrentCharacter) == 'l')
+            {
+                TakeCurrent();
+                if (Char.ToLowerInvariant(CurrentCharacter) == 'u')
+                {
+                    TakeCurrent();
+                }
+            }
+        }
+
+        // CSharp Spec §2.4.2
+        private StateResult Identifier()
+        {
+            Debug.Assert(CSharpHelpers.IsIdentifierStart(CurrentCharacter));
+            TakeCurrent();
+            TakeUntil(c => !CSharpHelpers.IsIdentifierPart(c));
+            CSharpSymbol sym = null;
+            if (HaveContent)
+            {
+                CSharpKeyword? kwd = CSharpKeywordDetector.SymbolTypeForIdentifier(Buffer.ToString());
+                CSharpSymbolType type = CSharpSymbolType.Identifier;
+                if (kwd != null)
+                {
+                    type = CSharpSymbolType.Keyword;
+                }
+                sym = new CSharpSymbol(CurrentStart, Buffer.ToString(), type) { Keyword = kwd };
+            }
+            StartSymbol();
+            return Stay(sym);
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/HtmlTokenizer.cs

@@ -0,0 +1,199 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Web.Razor.Parser;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Text;
+using System.Web.Razor.Tokenizer.Symbols;
+
+namespace System.Web.Razor.Tokenizer
+{
+    // Tokenizer _loosely_ based on http://dev.w3.org/html5/spec/Overview.html#tokenization
+    public class HtmlTokenizer : Tokenizer<HtmlSymbol, HtmlSymbolType>
+    {
+        private const char TransitionChar = '@';
+
+        public HtmlTokenizer(ITextDocument source)
+            : base(source)
+        {
+            CurrentState = Data;
+        }
+
+        protected override State StartState
+        {
+            get { return Data; }
+        }
+
+        public override HtmlSymbolType RazorCommentType
+        {
+            get { return HtmlSymbolType.RazorComment; }
+        }
+
+        public override HtmlSymbolType RazorCommentTransitionType
+        {
+            get { return HtmlSymbolType.RazorCommentTransition; }
+        }
+
+        public override HtmlSymbolType RazorCommentStarType
+        {
+            get { return HtmlSymbolType.RazorCommentStar; }
+        }
+
+        internal static IEnumerable<HtmlSymbol> Tokenize(string content)
+        {
+            using (SeekableTextReader reader = new SeekableTextReader(content))
+            {
+                HtmlTokenizer tok = new HtmlTokenizer(reader);
+                HtmlSymbol sym;
+                while ((sym = tok.NextSymbol()) != null)
+                {
+                    yield return sym;
+                }
+            }
+        }
+
+        protected override HtmlSymbol CreateSymbol(SourceLocation start, string content, HtmlSymbolType type, IEnumerable<RazorError> errors)
+        {
+            return new HtmlSymbol(start, content, type, errors);
+        }
+
+        // http://dev.w3.org/html5/spec/Overview.html#data-state
+        private StateResult Data()
+        {
+            if (ParserHelpers.IsWhitespace(CurrentCharacter))
+            {
+                return Stay(Whitespace());
+            }
+            else if (ParserHelpers.IsNewLine(CurrentCharacter))
+            {
+                return Stay(Newline());
+            }
+            else if (CurrentCharacter == '@')
+            {
+                TakeCurrent();
+                if (CurrentCharacter == '*')
+                {
+                    return Transition(EndSymbol(HtmlSymbolType.RazorCommentTransition), AfterRazorCommentTransition);
+                }
+                else if (CurrentCharacter == '@')
+                {
+                    // Could be escaped comment transition
+                    return Transition(EndSymbol(HtmlSymbolType.Transition), () =>
+                    {
+                        TakeCurrent();
+                        return Transition(EndSymbol(HtmlSymbolType.Transition), Data);
+                    });
+                }
+                return Stay(EndSymbol(HtmlSymbolType.Transition));
+            }
+            else if (AtSymbol())
+            {
+                return Stay(Symbol());
+            }
+            else
+            {
+                return Transition(Text);
+            }
+        }
+
+        private StateResult Text()
+        {
+            char prev = '\0';
+            while (!EndOfFile && !ParserHelpers.IsWhitespaceOrNewLine(CurrentCharacter) && !AtSymbol())
+            {
+                prev = CurrentCharacter;
+                TakeCurrent();
+            }
+
+            if (CurrentCharacter == '@')
+            {
+                char next = Peek();
+                if (ParserHelpers.IsLetterOrDecimalDigit(prev) && ParserHelpers.IsLetterOrDecimalDigit(next))
+                {
+                    TakeCurrent(); // Take the "@"
+                    return Stay(); // Stay in the Text state
+                }
+            }
+
+            // Output the Text token and return to the Data state to tokenize the next character (if there is one)
+            return Transition(EndSymbol(HtmlSymbolType.Text), Data);
+        }
+
+        private HtmlSymbol Symbol()
+        {
+            Debug.Assert(AtSymbol());
+            char sym = CurrentCharacter;
+            TakeCurrent();
+            switch (sym)
+            {
+                case '<':
+                    return EndSymbol(HtmlSymbolType.OpenAngle);
+                case '!':
+                    return EndSymbol(HtmlSymbolType.Bang);
+                case '/':
+                    return EndSymbol(HtmlSymbolType.Solidus);
+                case '?':
+                    return EndSymbol(HtmlSymbolType.QuestionMark);
+                case '[':
+                    return EndSymbol(HtmlSymbolType.LeftBracket);
+                case '>':
+                    return EndSymbol(HtmlSymbolType.CloseAngle);
+                case ']':
+                    return EndSymbol(HtmlSymbolType.RightBracket);
+                case '=':
+                    return EndSymbol(HtmlSymbolType.Equals);
+                case '"':
+                    return EndSymbol(HtmlSymbolType.DoubleQuote);
+                case '\'':
+                    return EndSymbol(HtmlSymbolType.SingleQuote);
+                case '-':
+                    Debug.Assert(CurrentCharacter == '-');
+                    TakeCurrent();
+                    return EndSymbol(HtmlSymbolType.DoubleHyphen);
+                default:
+                    Debug.Fail("Unexpected symbol!");
+                    return EndSymbol(HtmlSymbolType.Unknown);
+            }
+        }
+
+        private HtmlSymbol Whitespace()
+        {
+            while (ParserHelpers.IsWhitespace(CurrentCharacter))
+            {
+                TakeCurrent();
+            }
+            return EndSymbol(HtmlSymbolType.WhiteSpace);
+        }
+
+        private HtmlSymbol Newline()
+        {
+            Debug.Assert(ParserHelpers.IsNewLine(CurrentCharacter));
+            // CSharp Spec §2.3.1
+            bool checkTwoCharNewline = CurrentCharacter == '\r';
+            TakeCurrent();
+            if (checkTwoCharNewline && CurrentCharacter == '\n')
+            {
+                TakeCurrent();
+            }
+            return EndSymbol(HtmlSymbolType.NewLine);
+        }
+
+        private bool AtSymbol()
+        {
+            return CurrentCharacter == '<' ||
+                   CurrentCharacter == '<' ||
+                   CurrentCharacter == '!' ||
+                   CurrentCharacter == '/' ||
+                   CurrentCharacter == '?' ||
+                   CurrentCharacter == '[' ||
+                   CurrentCharacter == '>' ||
+                   CurrentCharacter == ']' ||
+                   CurrentCharacter == '=' ||
+                   CurrentCharacter == '"' ||
+                   CurrentCharacter == '\'' ||
+                   CurrentCharacter == '@' ||
+                   (CurrentCharacter == '-' && Peek() == '-');
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/ITokenizer.cs

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Web.Razor.Tokenizer.Symbols;
+
+namespace System.Web.Razor.Tokenizer
+{
+    public interface ITokenizer
+    {
+        ISymbol NextSymbol();
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/CSharpKeyword.cs

@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public enum CSharpKeyword
+    {
+        Abstract,
+        Byte,
+        Class,
+        Delegate,
+        Event,
+        Fixed,
+        If,
+        Internal,
+        New,
+        Override,
+        Readonly,
+        Short,
+        Struct,
+        Try,
+        Unsafe,
+        Volatile,
+        As,
+        Do,
+        Is,
+        Params,
+        Ref,
+        Switch,
+        Ushort,
+        While,
+        Case,
+        Const,
+        Explicit,
+        Float,
+        Null,
+        Sizeof,
+        Typeof,
+        Implicit,
+        Private,
+        This,
+        Using,
+        Extern,
+        Return,
+        Stackalloc,
+        Uint,
+        Base,
+        Catch,
+        Continue,
+        Double,
+        For,
+        In,
+        Lock,
+        Object,
+        Protected,
+        Static,
+        False,
+        Public,
+        Sbyte,
+        Throw,
+        Virtual,
+        Decimal,
+        Else,
+        Operator,
+        String,
+        Ulong,
+        Bool,
+        Char,
+        Default,
+        Foreach,
+        Long,
+        Void,
+        Enum,
+        Finally,
+        Int,
+        Out,
+        Sealed,
+        True,
+        Goto,
+        Unchecked,
+        Interface,
+        Break,
+        Checked,
+        Namespace
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/CSharpSymbol.cs

@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Text;
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public class CSharpSymbol : SymbolBase<CSharpSymbolType>
+    {
+        // Helper constructor
+        public CSharpSymbol(int offset, int line, int column, string content, CSharpSymbolType type)
+            : this(new SourceLocation(offset, line, column), content, type, Enumerable.Empty<RazorError>())
+        {
+        }
+
+        public CSharpSymbol(SourceLocation start, string content, CSharpSymbolType type)
+            : this(start, content, type, Enumerable.Empty<RazorError>())
+        {
+        }
+
+        public CSharpSymbol(int offset, int line, int column, string content, CSharpSymbolType type, IEnumerable<RazorError> errors)
+            : base(new SourceLocation(offset, line, column), content, type, errors)
+        {
+        }
+
+        public CSharpSymbol(SourceLocation start, string content, CSharpSymbolType type, IEnumerable<RazorError> errors)
+            : base(start, content, type, errors)
+        {
+        }
+
+        public bool? EscapedIdentifier { get; set; }
+        public CSharpKeyword? Keyword { get; set; }
+
+        public override bool Equals(object obj)
+        {
+            CSharpSymbol other = obj as CSharpSymbol;
+            return base.Equals(obj) && other.Keyword == Keyword;
+        }
+
+        public override int GetHashCode()
+        {
+            return base.GetHashCode() ^ Keyword.GetHashCode();
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/CSharpSymbolType.cs

@@ -0,0 +1,74 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public enum CSharpSymbolType
+    {
+        Unknown,
+        Identifier,
+        Keyword,
+        IntegerLiteral,
+        NewLine,
+        WhiteSpace,
+        Comment,
+        RealLiteral,
+        CharacterLiteral,
+        StringLiteral,
+
+        // Operators
+        Arrow,
+        Minus,
+        Decrement,
+        MinusAssign,
+        NotEqual,
+        Not,
+        Modulo,
+        ModuloAssign,
+        AndAssign,
+        And,
+        DoubleAnd,
+        LeftParenthesis,
+        RightParenthesis,
+        Star,
+        MultiplyAssign,
+        Comma,
+        Dot,
+        Slash,
+        DivideAssign,
+        DoubleColon,
+        Colon,
+        Semicolon,
+        QuestionMark,
+        NullCoalesce,
+        RightBracket,
+        LeftBracket,
+        XorAssign,
+        Xor,
+        LeftBrace,
+        OrAssign,
+        DoubleOr,
+        Or,
+        RightBrace,
+        Tilde,
+        Plus,
+        PlusAssign,
+        Increment,
+        LessThan,
+        LessThanEqual,
+        LeftShift,
+        LeftShiftAssign,
+        Assign,
+        Equals,
+        GreaterThan,
+        GreaterThanEqual,
+        RightShift,
+        RightShiftAssign,
+        Hash,
+        Transition,
+
+        // Razor specific
+        RazorCommentTransition,
+        RazorCommentStar,
+        RazorComment
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/HtmlSymbol.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Text;
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public class HtmlSymbol : SymbolBase<HtmlSymbolType>
+    {
+        // Helper constructor
+        public HtmlSymbol(int offset, int line, int column, string content, HtmlSymbolType type)
+            : this(new SourceLocation(offset, line, column), content, type, Enumerable.Empty<RazorError>())
+        {
+        }
+
+        public HtmlSymbol(SourceLocation start, string content, HtmlSymbolType type)
+            : base(start, content, type, Enumerable.Empty<RazorError>())
+        {
+        }
+
+        public HtmlSymbol(int offset, int line, int column, string content, HtmlSymbolType type, IEnumerable<RazorError> errors)
+            : base(new SourceLocation(offset, line, column), content, type, errors)
+        {
+        }
+
+        public HtmlSymbol(SourceLocation start, string content, HtmlSymbolType type, IEnumerable<RazorError> errors)
+            : base(start, content, type, errors)
+        {
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/HtmlSymbolType.cs

@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public enum HtmlSymbolType
+    {
+        Unknown,
+        Text, // Text which isn't one of the below
+        WhiteSpace, // Non-newline Whitespace
+        NewLine, // Newline
+        OpenAngle, // <
+        Bang, // !
+        Solidus, // /
+        QuestionMark, // ?
+        DoubleHyphen, // --
+        LeftBracket, // [
+        CloseAngle, // >
+        RightBracket, // ]
+        Equals, // =
+        DoubleQuote, // "
+        SingleQuote, // '
+        Transition, // @
+        Colon,
+        RazorComment,
+        RazorCommentStar,
+        RazorCommentTransition
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/ISymbol.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Web.Razor.Text;
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public interface ISymbol
+    {
+        SourceLocation Start { get; }
+        string Content { get; }
+
+        void OffsetStart(SourceLocation documentStart);
+        void ChangeStart(SourceLocation newStart);
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/KnownSymbolType.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public enum KnownSymbolType
+    {
+        WhiteSpace,
+        NewLine,
+        Identifier,
+        Keyword,
+        Transition,
+        Unknown,
+        CommentStart,
+        CommentStar,
+        CommentBody
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/SymbolBase.cs

@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Text;
+using Microsoft.Internal.Web.Utils;
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public abstract class SymbolBase<TType> : ISymbol
+    {
+        protected SymbolBase(SourceLocation start, string content, TType type, IEnumerable<RazorError> errors)
+        {
+            if (content == null)
+            {
+                throw new ArgumentNullException("content");
+            }
+            if (type == null)
+            {
+                throw new ArgumentNullException("type");
+            }
+
+            Start = start;
+            Content = content;
+            Type = type;
+            Errors = errors;
+        }
+
+        public SourceLocation Start { get; private set; }
+        public string Content { get; private set; }
+        public IEnumerable<RazorError> Errors { get; private set; }
+
+        [SuppressMessage("Microsoft.Naming", "CA1721:PropertyNamesShouldNotMatchGetMethods", Justification = "This is the most appropriate name for this property and conflicts are unlikely")]
+        public TType Type { get; private set; }
+
+        public override bool Equals(object obj)
+        {
+            SymbolBase<TType> other = obj as SymbolBase<TType>;
+            return other != null &&
+                   Start.Equals(other.Start) &&
+                   String.Equals(Content, other.Content, StringComparison.Ordinal) &&
+                   Type.Equals(other.Type);
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCodeCombiner.Start()
+                .Add(Start)
+                .Add(Content)
+                .Add(Type)
+                .CombinedHash;
+        }
+
+        public override string ToString()
+        {
+            return String.Format(CultureInfo.InvariantCulture, "{0} {1} - [{2}]", Start, Type, Content);
+        }
+
+        public void OffsetStart(SourceLocation documentStart)
+        {
+            Start = documentStart + Start;
+        }
+
+        public void ChangeStart(SourceLocation newStart)
+        {
+            Start = newStart;
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/SymbolExtensions.cs

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Text;
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public static class SymbolExtensions
+    {
+        public static LocationTagged<string> GetContent(this SpanBuilder span)
+        {
+            return GetContent(span, e => e);
+        }
+
+        [SuppressMessage("Microsoft.Design", "CA1006:DoNotNestGenericTypesInMemberSignatures", Justification = "Func<T> is the recommended type for generic delegates and requires this level of nesting")]
+        public static LocationTagged<string> GetContent(this SpanBuilder span, Func<IEnumerable<ISymbol>, IEnumerable<ISymbol>> filter)
+        {
+            return GetContent(filter(span.Symbols), span.Start);
+        }
+
+        public static LocationTagged<string> GetContent(this IEnumerable<ISymbol> symbols, SourceLocation spanStart)
+        {
+            if (symbols.Any())
+            {
+                return new LocationTagged<string>(String.Concat(symbols.Select(s => s.Content)), spanStart + symbols.First().Start);
+            }
+            else
+            {
+                return new LocationTagged<string>(String.Empty, spanStart);
+            }
+        }
+
+        public static LocationTagged<string> GetContent(this ISymbol symbol)
+        {
+            return new LocationTagged<string>(symbol.Content, symbol.Start);
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/SymbolTypeSuppressions.cs

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+// Centralized all the supressions for the CSharpSymbolType and VBSymbolType enum members here for clarity. They are
+// not in the CodeAnalysisDictionary because they are special case exclusions
+
+using System.Diagnostics.CodeAnalysis;
+
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Foreach", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Foreach", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "Readonly", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Readonly", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Sbyte", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Sbyte", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Sizeof", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Sizeof", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Stackalloc", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Stackalloc", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Typeof", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Typeof", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Uint", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Uint", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Ulong", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Ulong", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Ushort", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.CSharpKeyword.#Ushort", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Val", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.VBKeyword.#ByVal", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Sng", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.VBKeyword.#CSng", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "ReDim", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.VBKeyword.#ReDim", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Re", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.VBKeyword.#ReDim", Justification = Justifications.SymbolTypeNames)]
+[assembly: SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Str", Scope = "member", Target = "System.Web.Razor.Tokenizer.Symbols.VBKeyword.#CStr", Justification = Justifications.SymbolTypeNames)]
+
+internal static partial class Justifications
+{
+    internal const string SymbolTypeNames = "Symbol Type Names are spelled according to the language keyword or token they represent";
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/VBKeyword.cs

@@ -0,0 +1,168 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public enum VBKeyword
+    {
+        AddHandler,
+        AndAlso,
+        Byte,
+        Catch,
+        CDate,
+        CInt,
+        Const,
+        CSng,
+
+        [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Lng", Justification = "This is a VB Keyword. Note: Excluded here because it is a specific case")]
+        CULng,
+        Declare,
+        DirectCast,
+        Else,
+        Enum,
+        Exit,
+        Friend,
+        GetXmlNamespace,
+        Handles,
+        In,
+        Is,
+        Like,
+        Mod,
+        MyBase,
+        New,
+        AddressOf,
+        As,
+        ByVal,
+        CBool,
+
+        [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Dbl", Justification = "This is a VB Keyword. Note: Excluded here because it is a specific case")]
+        CDbl,
+        Class,
+        Continue,
+        CStr,
+        CUShort,
+        Default,
+        Do,
+        ElseIf,
+        Erase,
+        False,
+        Function,
+        Global,
+        If,
+        Inherits,
+        IsNot,
+        Long,
+        Module,
+        MyClass,
+        Next,
+        Alias,
+        Boolean,
+        Call,
+        CByte,
+        CDec,
+
+        [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Lng", Justification = "This is a VB Keyword. Note: Excluded here because it is a specific case")]
+        CLng,
+        CSByte,
+        CType,
+        Date,
+        Delegate,
+        Double,
+        End,
+        Error,
+        Finally,
+        Get,
+        GoSub,
+        Implements,
+        Integer,
+        Let,
+        Loop,
+        MustInherit,
+        Namespace,
+        Not,
+        And,
+        ByRef,
+        Case,
+        CChar,
+        Char,
+        CObj,
+        CShort,
+        CUInt,
+        Decimal,
+        Dim,
+        Each,
+        EndIf,
+        Event,
+        For,
+        GetType,
+        GoTo,
+        Imports,
+        Interface,
+        Lib,
+        Me,
+        MustOverride,
+        Narrowing,
+        Nothing,
+        NotInheritable,
+        On,
+        Or,
+        Overrides,
+        Property,
+        ReadOnly,
+        Resume,
+        Set,
+        Single,
+        String,
+        Then,
+        Try,
+        ULong,
+        Wend,
+        With,
+        NotOverridable,
+        Operator,
+        OrElse,
+        ParamArray,
+        Protected,
+        ReDim,
+        Return,
+        Shadows,
+        Static,
+        Structure,
+        Throw,
+        TryCast,
+        UShort,
+        When,
+        WithEvents,
+        Object,
+        Option,
+        Overloads,
+        Partial,
+        Public,
+        Rem,
+        SByte,
+        Shared,
+        Step,
+        Sub,
+        To,
+        TypeOf,
+        Using,
+        While,
+        WriteOnly,
+        Of,
+        Optional,
+        Overridable,
+        Private,
+        RaiseEvent,
+        RemoveHandler,
+        Select,
+        Short,
+        Stop,
+        SyncLock,
+        True,
+        UInteger,
+        Variant,
+        Widening,
+        Xor
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/VBSymbol.cs

@@ -0,0 +1,114 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Resources;
+using System.Web.Razor.Text;
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public class VBSymbol : SymbolBase<VBSymbolType>
+    {
+        // Helper constructor
+        private static Dictionary<VBSymbolType, string> _symbolSamples = new Dictionary<VBSymbolType, string>()
+        {
+            { VBSymbolType.LineContinuation, "_" },
+            { VBSymbolType.LeftParenthesis, "(" },
+            { VBSymbolType.RightParenthesis, ")" },
+            { VBSymbolType.LeftBracket, "[" },
+            { VBSymbolType.RightBracket, "]" },
+            { VBSymbolType.LeftBrace, "{" },
+            { VBSymbolType.RightBrace, "}" },
+            { VBSymbolType.Bang, "!" },
+            { VBSymbolType.Hash, "#" },
+            { VBSymbolType.Comma, "," },
+            { VBSymbolType.Dot, "." },
+            { VBSymbolType.Colon, ":" },
+            { VBSymbolType.QuestionMark, "?" },
+            { VBSymbolType.Concatenation, "&" },
+            { VBSymbolType.Multiply, "*" },
+            { VBSymbolType.Add, "+" },
+            { VBSymbolType.Subtract, "-" },
+            { VBSymbolType.Divide, "/" },
+            { VBSymbolType.IntegerDivide, "\\" },
+            { VBSymbolType.Exponentiation, "^" },
+            { VBSymbolType.Equal, "=" },
+            { VBSymbolType.LessThan, "<" },
+            { VBSymbolType.GreaterThan, ">" },
+            { VBSymbolType.Dollar, "$" },
+            { VBSymbolType.Transition, "@" },
+            { VBSymbolType.RazorCommentTransition, "@" },
+            { VBSymbolType.RazorCommentStar, "*" }
+        };
+
+        public VBSymbol(int offset, int line, int column, string content, VBSymbolType type)
+            : this(new SourceLocation(offset, line, column), content, type, Enumerable.Empty<RazorError>())
+        {
+        }
+
+        public VBSymbol(SourceLocation start, string content, VBSymbolType type)
+            : this(start, content, type, Enumerable.Empty<RazorError>())
+        {
+        }
+
+        public VBSymbol(int offset, int line, int column, string content, VBSymbolType type, IEnumerable<RazorError> errors)
+            : base(new SourceLocation(offset, line, column), content, type, errors)
+        {
+        }
+
+        public VBSymbol(SourceLocation start, string content, VBSymbolType type, IEnumerable<RazorError> errors)
+            : base(start, content, type, errors)
+        {
+        }
+
+        public VBKeyword? Keyword { get; set; }
+
+        public override bool Equals(object obj)
+        {
+            VBSymbol other = obj as VBSymbol;
+            return base.Equals(obj) && other.Keyword == Keyword;
+        }
+
+        public override int GetHashCode()
+        {
+            return base.GetHashCode() ^ Keyword.GetHashCode();
+        }
+
+        public static string GetSample(VBSymbolType type)
+        {
+            string sample;
+            if (!_symbolSamples.TryGetValue(type, out sample))
+            {
+                switch (type)
+                {
+                    case VBSymbolType.WhiteSpace:
+                        return RazorResources.VBSymbol_WhiteSpace;
+                    case VBSymbolType.NewLine:
+                        return RazorResources.VBSymbol_NewLine;
+                    case VBSymbolType.Comment:
+                        return RazorResources.VBSymbol_Comment;
+                    case VBSymbolType.Identifier:
+                        return RazorResources.VBSymbol_Identifier;
+                    case VBSymbolType.Keyword:
+                        return RazorResources.VBSymbol_Keyword;
+                    case VBSymbolType.IntegerLiteral:
+                        return RazorResources.VBSymbol_IntegerLiteral;
+                    case VBSymbolType.FloatingPointLiteral:
+                        return RazorResources.VBSymbol_FloatingPointLiteral;
+                    case VBSymbolType.StringLiteral:
+                        return RazorResources.VBSymbol_StringLiteral;
+                    case VBSymbolType.CharacterLiteral:
+                        return RazorResources.VBSymbol_CharacterLiteral;
+                    case VBSymbolType.DateLiteral:
+                        return RazorResources.VBSymbol_DateLiteral;
+                    case VBSymbolType.RazorComment:
+                        return RazorResources.VBSymbol_RazorComment;
+                    default:
+                        return RazorResources.Symbol_Unknown;
+                }
+            }
+            return sample;
+        }
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Symbols/VBSymbolType.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+namespace System.Web.Razor.Tokenizer.Symbols
+{
+    public enum VBSymbolType
+    {
+        Unknown,
+        WhiteSpace,
+        NewLine,
+        LineContinuation,
+        Comment,
+        Identifier,
+        Keyword,
+        IntegerLiteral,
+        FloatingPointLiteral,
+        StringLiteral,
+        CharacterLiteral,
+        DateLiteral,
+        LeftParenthesis,
+        RightBrace,
+        LeftBrace,
+        RightParenthesis,
+        Hash,
+        Bang,
+        Comma,
+        Dot,
+        Colon,
+        Concatenation,
+        QuestionMark,
+        Subtract,
+        Multiply,
+        Add,
+        Divide,
+        IntegerDivide,
+        Exponentiation,
+        LessThan,
+        GreaterThan,
+        Equal,
+        RightBracket,
+        LeftBracket,
+        Dollar,
+        Transition,
+
+        RazorCommentTransition,
+        RazorCommentStar,
+        RazorComment
+    }
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/Tokenizer.cs

@@ -0,0 +1,354 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Web.Razor.Parser;
+using System.Web.Razor.Parser.SyntaxTree;
+using System.Web.Razor.Resources;
+using System.Web.Razor.Text;
+using System.Web.Razor.Tokenizer.Symbols;
+
+namespace System.Web.Razor.Tokenizer
+{
+    public abstract partial class Tokenizer<TSymbol, TSymbolType> : StateMachine<TSymbol>, ITokenizer
+        where TSymbol : SymbolBase<TSymbolType>
+    {
+        [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification = "TextDocumentReader does not require disposal")]
+        protected Tokenizer(ITextDocument source)
+        {
+            if (source == null)
+            {
+                throw new ArgumentNullException("source");
+            }
+            Source = new TextDocumentReader(source);
+            Buffer = new StringBuilder();
+            CurrentErrors = new List<RazorError>();
+            StartSymbol();
+        }
+
+        public TextDocumentReader Source { get; private set; }
+
+        protected StringBuilder Buffer { get; private set; }
+
+        protected bool EndOfFile
+        {
+            get { return Source.Peek() == -1; }
+        }
+
+        protected IList<RazorError> CurrentErrors { get; private set; }
+
+        public abstract TSymbolType RazorCommentStarType { get; }
+        public abstract TSymbolType RazorCommentType { get; }
+        public abstract TSymbolType RazorCommentTransitionType { get; }
+
+        protected bool HaveContent
+        {
+            get { return Buffer.Length > 0; }
+        }
+
+        protected char CurrentCharacter
+        {
+            get
+            {
+                int peek = Source.Peek();
+                return peek == -1 ? '\0' : (char)peek;
+            }
+        }
+
+        protected SourceLocation CurrentLocation
+        {
+            get { return Source.Location; }
+        }
+
+        protected SourceLocation CurrentStart { get; private set; }
+
+        public virtual TSymbol NextSymbol()
+        {
+            // Post-Condition: Buffer should be empty at the start of Next()
+            Debug.Assert(Buffer.Length == 0);
+            StartSymbol();
+
+            if (EndOfFile)
+            {
+                return null;
+            }
+            TSymbol sym = Turn();
+
+            // Post-Condition: Buffer should be empty at the end of Next()
+            Debug.Assert(Buffer.Length == 0);
+
+            return sym;
+        }
+
+        public void Reset()
+        {
+            CurrentState = StartState;
+        }
+
+        protected abstract TSymbol CreateSymbol(SourceLocation start, string content, TSymbolType type, IEnumerable<RazorError> errors);
+
+        protected TSymbol Single(TSymbolType type)
+        {
+            TakeCurrent();
+            return EndSymbol(type);
+        }
+
+        protected bool TakeString(string input, bool caseSensitive)
+        {
+            int position = 0;
+            Func<char, char> charFilter = c => c;
+            if (caseSensitive)
+            {
+                charFilter = Char.ToLower;
+            }
+            while (!EndOfFile && position < input.Length && charFilter(CurrentCharacter) == charFilter(input[position++]))
+            {
+                TakeCurrent();
+            }
+            return position == input.Length;
+        }
+
+        protected void StartSymbol()
+        {
+            Buffer.Clear();
+            CurrentStart = CurrentLocation;
+            CurrentErrors.Clear();
+        }
+
+        protected TSymbol EndSymbol(TSymbolType type)
+        {
+            return EndSymbol(CurrentStart, type);
+        }
+
+        protected TSymbol EndSymbol(SourceLocation start, TSymbolType type)
+        {
+            TSymbol sym = null;
+            if (HaveContent)
+            {
+                sym = CreateSymbol(start, Buffer.ToString(), type, CurrentErrors.ToArray());
+            }
+            StartSymbol();
+            return sym;
+        }
+
+        protected void ResumeSymbol(TSymbol previous)
+        {
+            // Verify the symbol can be resumed
+            if (previous.Start.AbsoluteIndex + previous.Content.Length != CurrentStart.AbsoluteIndex)
+            {
+                throw new InvalidOperationException(RazorResources.Tokenizer_CannotResumeSymbolUnlessIsPrevious);
+            }
+
+            // Reset the start point
+            CurrentStart = previous.Start;
+
+            // Capture the current buffer content
+            string newContent = Buffer.ToString();
+
+            // Clear the buffer, then put the old content back and add the new content to the end
+            Buffer.Clear();
+            Buffer.Append(previous.Content);
+            Buffer.Append(newContent);
+        }
+
+        protected bool TakeUntil(Func<char, bool> predicate)
+        {
+            // Take all the characters up to the end character
+            while (!EndOfFile && !predicate(CurrentCharacter))
+            {
+                TakeCurrent();
+            }
+
+            // Why did we end?
+            return !EndOfFile;
+        }
+
+        protected Func<char, bool> CharOrWhiteSpace(char character)
+        {
+            return c => c == character || ParserHelpers.IsWhitespace(c) || ParserHelpers.IsNewLine(c);
+        }
+
+        protected void TakeCurrent()
+        {
+            if (EndOfFile)
+            {
+                return;
+            } // No-op
+            Buffer.Append(CurrentCharacter);
+            MoveNext();
+        }
+
+        protected void MoveNext()
+        {
+#if DEBUG
+            _read.Append(CurrentCharacter);
+#endif
+            Source.Read();
+        }
+
+        protected bool TakeAll(string expected, bool caseSensitive)
+        {
+            return Lookahead(expected, takeIfMatch: true, caseSensitive: caseSensitive);
+        }
+
+        protected bool At(string expected, bool caseSensitive)
+        {
+            return Lookahead(expected, takeIfMatch: false, caseSensitive: caseSensitive);
+        }
+
+        protected char Peek()
+        {
+            using (LookaheadToken lookahead = Source.BeginLookahead())
+            {
+                MoveNext();
+                return CurrentCharacter;
+            }
+        }
+
+        protected StateResult AfterRazorCommentTransition()
+        {
+            if (CurrentCharacter != '*')
+            {
+                // We've been moved since last time we were asked for a symbol... reset the state
+                return Transition(StartState);
+            }
+            AssertCurrent('*');
+            TakeCurrent();
+            return Transition(EndSymbol(RazorCommentStarType), RazorCommentBody);
+        }
+
+        protected StateResult RazorCommentBody()
+        {
+            TakeUntil(c => c == '*');
+            if (CurrentCharacter == '*')
+            {
+                char star = CurrentCharacter;
+                SourceLocation start = CurrentLocation;
+                MoveNext();
+                if (!EndOfFile && CurrentCharacter == '@')
+                {
+                    State next = () =>
+                    {
+                        Buffer.Append(star);
+                        return Transition(EndSymbol(start, RazorCommentStarType), () =>
+                        {
+                            if (CurrentCharacter != '@')
+                            {
+                                // We've been moved since last time we were asked for a symbol... reset the state
+                                return Transition(StartState);
+                            }
+                            TakeCurrent();
+                            return Transition(EndSymbol(RazorCommentTransitionType), StartState);
+                        });
+                    };
+
+                    if (HaveContent)
+                    {
+                        return Transition(EndSymbol(RazorCommentType), next);
+                    }
+                    else
+                    {
+                        return Transition(next);
+                    }
+                }
+                else
+                {
+                    Buffer.Append(star);
+                    return Stay();
+                }
+            }
+            return Transition(EndSymbol(RazorCommentType), StartState);
+        }
+
+        private bool Lookahead(string expected, bool takeIfMatch, bool caseSensitive)
+        {
+            Func<char, char> filter = c => c;
+            if (!caseSensitive)
+            {
+                filter = Char.ToLowerInvariant;
+            }
+
+            if (expected.Length == 0 || filter(CurrentCharacter) != filter(expected[0]))
+            {
+                return false;
+            }
+
+            // Capture the current buffer content in case we have to backtrack
+            string oldBuffer = null;
+            if (takeIfMatch)
+            {
+                Buffer.ToString();
+            }
+
+            using (LookaheadToken lookahead = Source.BeginLookahead())
+            {
+                for (int i = 0; i < expected.Length; i++)
+                {
+                    if (filter(CurrentCharacter) != filter(expected[i]))
+                    {
+                        if (takeIfMatch)
+                        {
+                            // Clear the buffer and put the old buffer text back
+                            Buffer.Clear();
+                            Buffer.Append(oldBuffer);
+                        }
+                        // Return without accepting lookahead (thus rejecting it)
+                        return false;
+                    }
+                    if (takeIfMatch)
+                    {
+                        TakeCurrent();
+                    }
+                    else
+                    {
+                        MoveNext();
+                    }
+                }
+                if (takeIfMatch)
+                {
+                    lookahead.Accept();
+                }
+            }
+            return true;
+        }
+
+        [SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Justification = "This only occurs in Release builds, where this method is empty by design")]
+        [Conditional("DEBUG")]
+        internal void AssertCurrent(char current)
+        {
+            Debug.Assert(CurrentCharacter == current, "CurrentCharacter Assumption violated", "Assumed that the current character would be {0}, but it is actually {1}", current, CurrentCharacter);
+        }
+
+        ISymbol ITokenizer.NextSymbol()
+        {
+            return (ISymbol)NextSymbol();
+        }
+    }
+
+#if DEBUG
+    [DebuggerDisplay("{DebugDisplay}")]
+    public partial class Tokenizer<TSymbol, TSymbolType>
+    {
+        private StringBuilder _read = new StringBuilder();
+
+        public string DebugDisplay
+        {
+            get { return String.Format(CultureInfo.InvariantCulture, "[{0}] [{1}] [{2}]", _read.ToString(), CurrentCharacter, Remaining); }
+        }
+
+        public string Remaining
+        {
+            get
+            {
+                string remaining = Source.ReadToEnd();
+                Source.Seek(-remaining.Length);
+                return remaining;
+            }
+        }
+    }
+#endif
+}

external/aspnetwebstack/src/System.Web.Razor/Tokenizer/TokenizerView.cs

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
+
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using System.Web.Razor.Resources;
+using System.Web.Razor.Text;
+using System.Web.Razor.Tokenizer.Symbols;
+
+namespace System.Web.Razor.Tokenizer
+{
+    [SuppressMessage("Microsoft.Design", "CA1005:AvoidExcessiveParametersOnGenericTypes", Justification = "All generic parameters are required")]
+    public class TokenizerView<TTokenizer, TSymbol, TSymbolType>
+        where TTokenizer : Tokenizer<TSymbol, TSymbolType>
+        where TSymbol : SymbolBase<TSymbolType>
+    {
+        public TokenizerView(TTokenizer tokenizer)
+        {
+            Tokenizer = tokenizer;
+        }
+
+        public TTokenizer Tokenizer { get; private set; }
+        public bool EndOfFile { get; private set; }
+        public TSymbol Current { get; private set; }
+
+        public ITextDocument Source
+        {
+            get { return Tokenizer.Source; }
+        }
+
+        public bool Next()
+        {
+            Current = Tokenizer.NextSymbol();
+            EndOfFile = (Current == null);
+            return !EndOfFile;
+        }
+
+        public void PutBack(TSymbol symbol)
+        {
+            Debug.Assert(Source.Position == symbol.Start.AbsoluteIndex + symbol.Content.Length);
+            if (Source.Position != symbol.Start.AbsoluteIndex + symbol.Content.Length)
+            {
+                // We've already passed this symbol
+                throw new InvalidOperationException(
+                    String.Format(CultureInfo.CurrentCulture,
+                                  RazorResources.TokenizerView_CannotPutBack,
+                                  symbol.Start.AbsoluteIndex + symbol.Content.Length,
+                                  Source.Position));
+            }
+            Source.Position -= symbol.Content.Length;
+            Current = null;
+            EndOfFile = Source.Position >= Source.Length;
+            Tokenizer.Reset();
+        }
+    }
+}