mono/linux-packaging-mono
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Imported Upstream version 3.6.0

Browse Source 
Former-commit-id: da6be194a6b1221998fc28233f2503bd61dd9d14
This commit is contained in:
Jo Shields
2014-08-13 10:39:27 +01:00
commit a575963da9
50588 changed files with 8155799 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/CSharpTokenizerCommentTest.cs vendored Normal file
View File

@@ -0,0 +1,89 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class CSharpTokenizerCommentTest : CSharpTokenizerTestBase
{
[Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment()
{
TestTokenizer("@* Foo * Bar * Baz *",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(2, 0, 2, " Foo * Bar * Baz *", CSharpSymbolType.RazorComment));
}
[Fact]
public void Next_Ignores_Star_Without_Trailing_At()
{
TestTokenizer("@* Foo * Bar * Baz *@",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(2, 0, 2, " Foo * Bar * Baz ", CSharpSymbolType.RazorComment),
new CSharpSymbol(19, 0, 19, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(20, 0, 20, "@", CSharpSymbolType.RazorCommentTransition));
}
[Fact]
public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment()
{
TestTokenizer("@* Foo Bar Baz *@",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.RazorCommentTransition),
new CSharpSymbol(1, 0, 1, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(2, 0, 2, " Foo Bar Baz ", CSharpSymbolType.RazorComment),
new CSharpSymbol(15, 0, 15, "*", CSharpSymbolType.RazorCommentStar),
new CSharpSymbol(16, 0, 16, "@", CSharpSymbolType.RazorCommentTransition));
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
{
TestTokenizer("// Foo Bar Baz", new CSharpSymbol(0, 0, 0, "// Foo Bar Baz", CSharpSymbolType.Comment));
}
[Fact]
public void Single_Line_Comment_Is_Terminated_By_Newline()
{
TestTokenizer("// Foo Bar Baz\na", new CSharpSymbol(0, 0, 0, "// Foo Bar Baz", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
{
TestTokenizer("// Foo/*Bar*/ Baz\na", new CSharpSymbol(0, 0, 0, "// Foo/*Bar*/ Baz", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
{
TestTokenizer("/* Foo\nBar\nBaz */", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz */", CSharpSymbolType.Comment));
}
[Fact]
public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
{
TestTokenizer("/* Foo\nBar\nBaz */a", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz */", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
{
TestTokenizer("/* Foo\nBar\nBaz", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
{
TestTokenizer("/* Foo/*\nBar\nBaz*/ */", new CSharpSymbol(0, 0, 0, "/* Foo/*\nBar\nBaz*/", CSharpSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
{
TestTokenizer("/* Foo\nBar\nBaz* */", new CSharpSymbol(0, 0, 0, "/* Foo\nBar\nBaz* */", CSharpSymbolType.Comment), IgnoreRemaining);
}
}
}

169
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/CSharpTokenizerIdentifierTest.cs vendored Normal file
View File

@@ -0,0 +1,169 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class CSharpTokenizerIdentifierTest : CSharpTokenizerTestBase
{
[Fact]
public void Simple_Identifier_Is_Recognized()
{
TestTokenizer("foo", new CSharpSymbol(0, 0, 0, "foo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized()
{
TestTokenizer("_foo", new CSharpSymbol(0, 0, 0, "_foo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Digits()
{
TestTokenizer("foo4", new CSharpSymbol(0, 0, 0, "foo4", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Titlecase_Letter()
{
TestTokenizer("ῼfoo", new CSharpSymbol(0, 0, 0, "ῼfoo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Letter_Modifier()
{
TestTokenizer("ᵊfoo", new CSharpSymbol(0, 0, 0, "ᵊfoo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Other_Letter()
{
TestTokenizer("ƻfoo", new CSharpSymbol(0, 0, 0, "ƻfoo", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Number_Letter()
{
TestTokenizer("Ⅽool", new CSharpSymbol(0, 0, 0, "Ⅽool", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark()
{
TestTokenizer("foo\u0300", new CSharpSymbol(0, 0, 0, "foo\u0300", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark()
{
TestTokenizer("fooः", new CSharpSymbol(0, 0, 0, "fooः", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Non_English_Digit()
{
TestTokenizer("foo١", new CSharpSymbol(0, 0, 0, "foo١", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Connector_Punctuation()
{
TestTokenizer("foo‿bar", new CSharpSymbol(0, 0, 0, "foo‿bar", CSharpSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Format_Character()
{
TestTokenizer("foo؃bar", new CSharpSymbol(0, 0, 0, "foo؃bar", CSharpSymbolType.Identifier));
}
[Fact]
public void Keywords_Are_Recognized_As_Keyword_Tokens()
{
TestKeyword("abstract", CSharpKeyword.Abstract);
TestKeyword("byte", CSharpKeyword.Byte);
TestKeyword("class", CSharpKeyword.Class);
TestKeyword("delegate", CSharpKeyword.Delegate);
TestKeyword("event", CSharpKeyword.Event);
TestKeyword("fixed", CSharpKeyword.Fixed);
TestKeyword("if", CSharpKeyword.If);
TestKeyword("internal", CSharpKeyword.Internal);
TestKeyword("new", CSharpKeyword.New);
TestKeyword("override", CSharpKeyword.Override);
TestKeyword("readonly", CSharpKeyword.Readonly);
TestKeyword("short", CSharpKeyword.Short);
TestKeyword("struct", CSharpKeyword.Struct);
TestKeyword("try", CSharpKeyword.Try);
TestKeyword("unsafe", CSharpKeyword.Unsafe);
TestKeyword("volatile", CSharpKeyword.Volatile);
TestKeyword("as", CSharpKeyword.As);
TestKeyword("do", CSharpKeyword.Do);
TestKeyword("is", CSharpKeyword.Is);
TestKeyword("params", CSharpKeyword.Params);
TestKeyword("ref", CSharpKeyword.Ref);
TestKeyword("switch", CSharpKeyword.Switch);
TestKeyword("ushort", CSharpKeyword.Ushort);
TestKeyword("while", CSharpKeyword.While);
TestKeyword("case", CSharpKeyword.Case);
TestKeyword("const", CSharpKeyword.Const);
TestKeyword("explicit", CSharpKeyword.Explicit);
TestKeyword("float", CSharpKeyword.Float);
TestKeyword("null", CSharpKeyword.Null);
TestKeyword("sizeof", CSharpKeyword.Sizeof);
TestKeyword("typeof", CSharpKeyword.Typeof);
TestKeyword("implicit", CSharpKeyword.Implicit);
TestKeyword("private", CSharpKeyword.Private);
TestKeyword("this", CSharpKeyword.This);
TestKeyword("using", CSharpKeyword.Using);
TestKeyword("extern", CSharpKeyword.Extern);
TestKeyword("return", CSharpKeyword.Return);
TestKeyword("stackalloc", CSharpKeyword.Stackalloc);
TestKeyword("uint", CSharpKeyword.Uint);
TestKeyword("base", CSharpKeyword.Base);
TestKeyword("catch", CSharpKeyword.Catch);
TestKeyword("continue", CSharpKeyword.Continue);
TestKeyword("double", CSharpKeyword.Double);
TestKeyword("for", CSharpKeyword.For);
TestKeyword("in", CSharpKeyword.In);
TestKeyword("lock", CSharpKeyword.Lock);
TestKeyword("object", CSharpKeyword.Object);
TestKeyword("protected", CSharpKeyword.Protected);
TestKeyword("static", CSharpKeyword.Static);
TestKeyword("false", CSharpKeyword.False);
TestKeyword("public", CSharpKeyword.Public);
TestKeyword("sbyte", CSharpKeyword.Sbyte);
TestKeyword("throw", CSharpKeyword.Throw);
TestKeyword("virtual", CSharpKeyword.Virtual);
TestKeyword("decimal", CSharpKeyword.Decimal);
TestKeyword("else", CSharpKeyword.Else);
TestKeyword("operator", CSharpKeyword.Operator);
TestKeyword("string", CSharpKeyword.String);
TestKeyword("ulong", CSharpKeyword.Ulong);
TestKeyword("bool", CSharpKeyword.Bool);
TestKeyword("char", CSharpKeyword.Char);
TestKeyword("default", CSharpKeyword.Default);
TestKeyword("foreach", CSharpKeyword.Foreach);
TestKeyword("long", CSharpKeyword.Long);
TestKeyword("void", CSharpKeyword.Void);
TestKeyword("enum", CSharpKeyword.Enum);
TestKeyword("finally", CSharpKeyword.Finally);
TestKeyword("int", CSharpKeyword.Int);
TestKeyword("out", CSharpKeyword.Out);
TestKeyword("sealed", CSharpKeyword.Sealed);
TestKeyword("true", CSharpKeyword.True);
TestKeyword("goto", CSharpKeyword.Goto);
TestKeyword("unchecked", CSharpKeyword.Unchecked);
TestKeyword("interface", CSharpKeyword.Interface);
TestKeyword("break", CSharpKeyword.Break);
TestKeyword("checked", CSharpKeyword.Checked);
TestKeyword("namespace", CSharpKeyword.Namespace);
}
private void TestKeyword(string keyword, CSharpKeyword keywordType)
{
TestTokenizer(keyword, new CSharpSymbol(0, 0, 0, keyword, CSharpSymbolType.Keyword) { Keyword = keywordType });
}
}
}

224
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/CSharpTokenizerLiteralTest.cs vendored Normal file
View File

@@ -0,0 +1,224 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class CSharpTokenizerLiteralTest : CSharpTokenizerTestBase
{
[Fact]
public void Simple_Integer_Literal_Is_Recognized()
{
TestSingleToken("01189998819991197253", CSharpSymbolType.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffix_Is_Recognized()
{
TestSingleToken("42U", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42u", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42L", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42l", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42UL", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42Ul", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42uL", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42ul", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42LU", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42Lu", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42lU", CSharpSymbolType.IntegerLiteral);
TestSingleToken("42lu", CSharpSymbolType.IntegerLiteral);
}
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Integer_Literal_If_Not_Type_Sufix()
{
TestTokenizer("42a", new CSharpSymbol(0, 0, 0, "42", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
public void Simple_Hex_Literal_Is_Recognized()
{
TestSingleToken("0x0123456789ABCDEF", CSharpSymbolType.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffix_Is_Recognized_In_Hex_Literal()
{
TestSingleToken("0xDEADBEEFU", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFu", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFL", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFl", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFUL", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFUl", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFuL", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFul", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFLU", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFLu", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFlU", CSharpSymbolType.IntegerLiteral);
TestSingleToken("0xDEADBEEFlu", CSharpSymbolType.IntegerLiteral);
}
[Fact]
public void Trailing_Letter_Is_Not_Part_Of_Hex_Literal_If_Not_Type_Sufix()
{
TestTokenizer("0xDEADBEEFz", new CSharpSymbol(0, 0, 0, "0xDEADBEEF", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
public void Dot_Followed_By_Non_Digit_Is_Not_Part_Of_Real_Literal()
{
TestTokenizer("3.a", new CSharpSymbol(0, 0, 0, "3", CSharpSymbolType.IntegerLiteral), IgnoreRemaining);
}
[Fact]
public void Simple_Real_Literal_Is_Recognized()
{
TestTokenizer("3.14159", new CSharpSymbol(0, 0, 0, "3.14159", CSharpSymbolType.RealLiteral));
}
[Fact]
public void Real_Literal_Between_Zero_And_One_Is_Recognized()
{
TestTokenizer(".14159", new CSharpSymbol(0, 0, 0, ".14159", CSharpSymbolType.RealLiteral));
}
[Fact]
public void Integer_With_Real_Type_Suffix_Is_Recognized()
{
TestSingleToken("42F", CSharpSymbolType.RealLiteral);
TestSingleToken("42f", CSharpSymbolType.RealLiteral);
TestSingleToken("42D", CSharpSymbolType.RealLiteral);
TestSingleToken("42d", CSharpSymbolType.RealLiteral);
TestSingleToken("42M", CSharpSymbolType.RealLiteral);
TestSingleToken("42m", CSharpSymbolType.RealLiteral);
}
[Fact]
public void Integer_With_Exponent_Is_Recognized()
{
TestSingleToken("1e10", CSharpSymbolType.RealLiteral);
TestSingleToken("1E10", CSharpSymbolType.RealLiteral);
TestSingleToken("1e+10", CSharpSymbolType.RealLiteral);
TestSingleToken("1E+10", CSharpSymbolType.RealLiteral);
TestSingleToken("1e-10", CSharpSymbolType.RealLiteral);
TestSingleToken("1E-10", CSharpSymbolType.RealLiteral);
}
[Fact]
public void Real_Number_With_Type_Suffix_Is_Recognized()
{
TestSingleToken("3.14F", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14f", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14D", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14d", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14M", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14m", CSharpSymbolType.RealLiteral);
}
[Fact]
public void Real_Number_With_Exponent_Is_Recognized()
{
TestSingleToken("3.14E10", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14e10", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14E+10", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14e+10", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14E-10", CSharpSymbolType.RealLiteral);
TestSingleToken("3.14e-10", CSharpSymbolType.RealLiteral);
}
[Fact]
public void Real_Number_With_Exponent_And_Type_Suffix_Is_Recognized()
{
TestSingleToken("3.14E+10F", CSharpSymbolType.RealLiteral);
}
[Fact]
public void Single_Character_Literal_Is_Recognized()
{
TestSingleToken("'f'", CSharpSymbolType.CharacterLiteral);
}
[Fact]
public void Multi_Character_Literal_Is_Recognized()
{
TestSingleToken("'foo'", CSharpSymbolType.CharacterLiteral);
}
[Fact]
public void Character_Literal_Is_Terminated_By_EOF_If_Unterminated()
{
TestSingleToken("'foo bar", CSharpSymbolType.CharacterLiteral);
}
[Fact]
public void Character_Literal_Not_Terminated_By_Escaped_Quote()
{
TestSingleToken("'foo\\'bar'", CSharpSymbolType.CharacterLiteral);
}
[Fact]
public void Character_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
TestTokenizer("'foo\n", new CSharpSymbol(0, 0, 0, "'foo", CSharpSymbolType.CharacterLiteral), IgnoreRemaining);
}
[Fact]
public void String_Literal_Is_Recognized()
{
TestSingleToken("\"foo\"", CSharpSymbolType.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_By_EOF_If_Unterminated()
{
TestSingleToken("\"foo bar", CSharpSymbolType.StringLiteral);
}
[Fact]
public void String_Literal_Not_Terminated_By_Escaped_Quote()
{
TestSingleToken("\"foo\\\"bar\"", CSharpSymbolType.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_By_EOL_If_Unterminated()
{
TestTokenizer("\"foo\n", new CSharpSymbol(0, 0, 0, "\"foo", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void Verbatim_String_Literal_Can_Contain_Newlines()
{
TestSingleToken("@\"foo\nbar\nbaz\"", CSharpSymbolType.StringLiteral);
}
[Fact]
public void Verbatim_String_Literal_Not_Terminated_By_Escaped_Double_Quote()
{
TestSingleToken("@\"foo\"\"bar\"", CSharpSymbolType.StringLiteral);
}
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_Slash_Double_Quote()
{
TestTokenizer("@\"foo\\\"bar\"", new CSharpSymbol(0, 0, 0, "@\"foo\\\"", CSharpSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void Verbatim_String_Literal_Is_Terminated_By_EOF()
{
TestSingleToken("@\"foo", CSharpSymbolType.StringLiteral);
}
}
}

296
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/CSharpTokenizerOperatorsTest.cs vendored Normal file
View File

@@ -0,0 +1,296 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class CSharpTokenizerOperatorsTest : CSharpTokenizerTestBase
{
[Fact]
public void LeftBrace_Is_Recognized()
{
TestSingleToken("{", CSharpSymbolType.LeftBrace);
}
[Fact]
public void Plus_Is_Recognized()
{
TestSingleToken("+", CSharpSymbolType.Plus);
}
[Fact]
public void Assign_Is_Recognized()
{
TestSingleToken("=", CSharpSymbolType.Assign);
}
[Fact]
public void Arrow_Is_Recognized()
{
TestSingleToken("->", CSharpSymbolType.Arrow);
}
[Fact]
public void AndAssign_Is_Recognized()
{
TestSingleToken("&=", CSharpSymbolType.AndAssign);
}
[Fact]
public void RightBrace_Is_Recognized()
{
TestSingleToken("}", CSharpSymbolType.RightBrace);
}
[Fact]
public void Minus_Is_Recognized()
{
TestSingleToken("-", CSharpSymbolType.Minus);
}
[Fact]
public void LessThan_Is_Recognized()
{
TestSingleToken("<", CSharpSymbolType.LessThan);
}
[Fact]
public void Equals_Is_Recognized()
{
TestSingleToken("==", CSharpSymbolType.Equals);
}
[Fact]
public void OrAssign_Is_Recognized()
{
TestSingleToken("|=", CSharpSymbolType.OrAssign);
}
[Fact]
public void LeftBracket_Is_Recognized()
{
TestSingleToken("[", CSharpSymbolType.LeftBracket);
}
[Fact]
public void Star_Is_Recognized()
{
TestSingleToken("*", CSharpSymbolType.Star);
}
[Fact]
public void GreaterThan_Is_Recognized()
{
TestSingleToken(">", CSharpSymbolType.GreaterThan);
}
[Fact]
public void NotEqual_Is_Recognized()
{
TestSingleToken("!=", CSharpSymbolType.NotEqual);
}
[Fact]
public void XorAssign_Is_Recognized()
{
TestSingleToken("^=", CSharpSymbolType.XorAssign);
}
[Fact]
public void RightBracket_Is_Recognized()
{
TestSingleToken("]", CSharpSymbolType.RightBracket);
}
[Fact]
public void Slash_Is_Recognized()
{
TestSingleToken("/", CSharpSymbolType.Slash);
}
[Fact]
public void QuestionMark_Is_Recognized()
{
TestSingleToken("?", CSharpSymbolType.QuestionMark);
}
[Fact]
public void LessThanEqual_Is_Recognized()
{
TestSingleToken("<=", CSharpSymbolType.LessThanEqual);
}
[Fact]
public void LeftShift_Is_Not_Specially_Recognized()
{
TestTokenizer("<<",
new CSharpSymbol(0, 0, 0, "<", CSharpSymbolType.LessThan),
new CSharpSymbol(1, 0, 1, "<", CSharpSymbolType.LessThan));
}
[Fact]
public void LeftParen_Is_Recognized()
{
TestSingleToken("(", CSharpSymbolType.LeftParenthesis);
}
[Fact]
public void Modulo_Is_Recognized()
{
TestSingleToken("%", CSharpSymbolType.Modulo);
}
[Fact]
public void NullCoalesce_Is_Recognized()
{
TestSingleToken("??", CSharpSymbolType.NullCoalesce);
}
[Fact]
public void GreaterThanEqual_Is_Recognized()
{
TestSingleToken(">=", CSharpSymbolType.GreaterThanEqual);
}
[Fact]
public void EqualGreaterThan_Is_Recognized()
{
TestSingleToken("=>", CSharpSymbolType.GreaterThanEqual);
}
[Fact]
public void RightParen_Is_Recognized()
{
TestSingleToken(")", CSharpSymbolType.RightParenthesis);
}
[Fact]
public void And_Is_Recognized()
{
TestSingleToken("&", CSharpSymbolType.And);
}
[Fact]
public void DoubleColon_Is_Recognized()
{
TestSingleToken("::", CSharpSymbolType.DoubleColon);
}
[Fact]
public void PlusAssign_Is_Recognized()
{
TestSingleToken("+=", CSharpSymbolType.PlusAssign);
}
[Fact]
public void Semicolon_Is_Recognized()
{
TestSingleToken(";", CSharpSymbolType.Semicolon);
}
[Fact]
public void Tilde_Is_Recognized()
{
TestSingleToken("~", CSharpSymbolType.Tilde);
}
[Fact]
public void DoubleOr_Is_Recognized()
{
TestSingleToken("||", CSharpSymbolType.DoubleOr);
}
[Fact]
public void ModuloAssign_Is_Recognized()
{
TestSingleToken("%=", CSharpSymbolType.ModuloAssign);
}
[Fact]
public void Colon_Is_Recognized()
{
TestSingleToken(":", CSharpSymbolType.Colon);
}
[Fact]
public void Not_Is_Recognized()
{
TestSingleToken("!", CSharpSymbolType.Not);
}
[Fact]
public void DoubleAnd_Is_Recognized()
{
TestSingleToken("&&", CSharpSymbolType.DoubleAnd);
}
[Fact]
public void DivideAssign_Is_Recognized()
{
TestSingleToken("/=", CSharpSymbolType.DivideAssign);
}
[Fact]
public void Comma_Is_Recognized()
{
TestSingleToken(",", CSharpSymbolType.Comma);
}
[Fact]
public void Xor_Is_Recognized()
{
TestSingleToken("^", CSharpSymbolType.Xor);
}
[Fact]
public void Decrement_Is_Recognized()
{
TestSingleToken("--", CSharpSymbolType.Decrement);
}
[Fact]
public void MultiplyAssign_Is_Recognized()
{
TestSingleToken("*=", CSharpSymbolType.MultiplyAssign);
}
[Fact]
public void Dot_Is_Recognized()
{
TestSingleToken(".", CSharpSymbolType.Dot);
}
[Fact]
public void Or_Is_Recognized()
{
TestSingleToken("|", CSharpSymbolType.Or);
}
[Fact]
public void Increment_Is_Recognized()
{
TestSingleToken("++", CSharpSymbolType.Increment);
}
[Fact]
public void MinusAssign_Is_Recognized()
{
TestSingleToken("-=", CSharpSymbolType.MinusAssign);
}
[Fact]
public void RightShift_Is_Not_Specially_Recognized()
{
TestTokenizer(">>",
new CSharpSymbol(0, 0, 0, ">", CSharpSymbolType.GreaterThan),
new CSharpSymbol(1, 0, 1, ">", CSharpSymbolType.GreaterThan));
}
[Fact]
public void Hash_Is_Recognized()
{
TestSingleToken("#", CSharpSymbolType.Hash);
}
}
}

104
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/CSharpTokenizerTest.cs vendored Normal file
View File

@@ -0,0 +1,104 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
using Assert = Microsoft.TestCommon.AssertEx;
namespace System.Web.Razor.Test.Tokenizer
{
public class CSharpTokenizerTest : CSharpTokenizerTestBase
{
[Fact]
public void Constructor_Throws_ArgNull_If_Null_Source_Provided()
{
Assert.ThrowsArgumentNull(() => new CSharpTokenizer(null), "source");
}
[Fact]
public void Next_Returns_Null_When_EOF_Reached()
{
TestTokenizer("");
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_CR()
{
TestTokenizer("\r\ra",
new CSharpSymbol(0, 0, 0, "\r", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\r", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_LF()
{
TestTokenizer("\n\na",
new CSharpSymbol(0, 0, 0, "\n", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_NEL()
{
// NEL: Unicode "Next Line" U+0085
TestTokenizer("\u0085\u0085a",
new CSharpSymbol(0, 0, 0, "\u0085", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u0085", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_Line_Separator()
{
// Unicode "Line Separator" U+2028
TestTokenizer("\u2028\u2028a",
new CSharpSymbol(0, 0, 0, "\u2028", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u2028", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator()
{
// Unicode "Paragraph Separator" U+2029
TestTokenizer("\u2029\u2029a",
new CSharpSymbol(0, 0, 0, "\u2029", CSharpSymbolType.NewLine),
new CSharpSymbol(1, 1, 0, "\u2029", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Single_Newline_Token_For_CRLF()
{
TestTokenizer("\r\n\r\na",
new CSharpSymbol(0, 0, 0, "\r\n", CSharpSymbolType.NewLine),
new CSharpSymbol(2, 1, 0, "\r\n", CSharpSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Token_For_Whitespace_Characters()
{
TestTokenizer(" \f\t\u000B \n ",
new CSharpSymbol(0, 0, 0, " \f\t\u000B ", CSharpSymbolType.WhiteSpace),
new CSharpSymbol(5, 0, 5, "\n", CSharpSymbolType.NewLine),
new CSharpSymbol(6, 1, 0, " ", CSharpSymbolType.WhiteSpace));
}
[Fact]
public void Transition_Is_Recognized()
{
TestSingleToken("@", CSharpSymbolType.Transition);
}
[Fact]
public void Transition_Is_Recognized_As_SingleCharacter()
{
TestTokenizer("@(",
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.Transition),
new CSharpSymbol(1, 0, 1, "(", CSharpSymbolType.LeftParenthesis));
}
}
}

28
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/CSharpTokenizerTestBase.cs vendored Normal file
View File

@@ -0,0 +1,28 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Text;
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
namespace System.Web.Razor.Test.Tokenizer
{
public abstract class CSharpTokenizerTestBase : TokenizerTestBase<CSharpSymbol, CSharpSymbolType>
{
private static CSharpSymbol _ignoreRemaining = new CSharpSymbol(0, 0, 0, String.Empty, CSharpSymbolType.Unknown);
protected override CSharpSymbol IgnoreRemaining
{
get { return _ignoreRemaining; }
}
protected override Tokenizer<CSharpSymbol, CSharpSymbolType> CreateTokenizer(ITextDocument source)
{
return new CSharpTokenizer(source);
}
protected void TestSingleToken(string text, CSharpSymbolType expectedSymbolType)
{
TestTokenizer(text, new CSharpSymbol(0, 0, 0, text, expectedSymbolType));
}
}
}

168
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/HtmlTokenizerTest.cs vendored Normal file
View File

@@ -0,0 +1,168 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
using Assert = Microsoft.TestCommon.AssertEx;
namespace System.Web.Razor.Test.Tokenizer
{
public class HtmlTokenizerTest : HtmlTokenizerTestBase
{
[Fact]
public void Constructor_Throws_ArgNull_If_Null_Source_Provided()
{
Assert.ThrowsArgumentNull(() => new HtmlTokenizer(null), "source");
}
[Fact]
public void Next_Returns_Null_When_EOF_Reached()
{
TestTokenizer("");
}
[Fact]
public void Text_Is_Recognized()
{
TestTokenizer("foo-9309&smlkmb;::-3029022,.sdkq92384",
new HtmlSymbol(0, 0, 0, "foo-9309&smlkmb;::-3029022,.sdkq92384", HtmlSymbolType.Text));
}
[Fact]
public void Whitespace_Is_Recognized()
{
TestTokenizer(" \t\f ",
new HtmlSymbol(0, 0, 0, " \t\f ", HtmlSymbolType.WhiteSpace));
}
[Fact]
public void Newline_Is_Recognized()
{
TestTokenizer("\n\r\r\n",
new HtmlSymbol(0, 0, 0, "\n", HtmlSymbolType.NewLine),
new HtmlSymbol(1, 1, 0, "\r", HtmlSymbolType.NewLine),
new HtmlSymbol(2, 2, 0, "\r\n", HtmlSymbolType.NewLine));
}
[Fact]
public void Transition_Is_Not_Recognized_Mid_Text_If_Surrounded_By_Alphanumeric_Characters()
{
TestSingleToken("foo@bar", HtmlSymbolType.Text);
}
[Fact]
public void OpenAngle_Is_Recognized()
{
TestSingleToken("<", HtmlSymbolType.OpenAngle);
}
[Fact]
public void Bang_Is_Recognized()
{
TestSingleToken("!", HtmlSymbolType.Bang);
}
[Fact]
public void Solidus_Is_Recognized()
{
TestSingleToken("/", HtmlSymbolType.Solidus);
}
[Fact]
public void QuestionMark_Is_Recognized()
{
TestSingleToken("?", HtmlSymbolType.QuestionMark);
}
[Fact]
public void LeftBracket_Is_Recognized()
{
TestSingleToken("[", HtmlSymbolType.LeftBracket);
}
[Fact]
public void CloseAngle_Is_Recognized()
{
TestSingleToken(">", HtmlSymbolType.CloseAngle);
}
[Fact]
public void RightBracket_Is_Recognized()
{
TestSingleToken("]", HtmlSymbolType.RightBracket);
}
[Fact]
public void Equals_Is_Recognized()
{
TestSingleToken("=", HtmlSymbolType.Equals);
}
[Fact]
public void DoubleQuote_Is_Recognized()
{
TestSingleToken("\"", HtmlSymbolType.DoubleQuote);
}
[Fact]
public void SingleQuote_Is_Recognized()
{
TestSingleToken("'", HtmlSymbolType.SingleQuote);
}
[Fact]
public void Transition_Is_Recognized()
{
TestSingleToken("@", HtmlSymbolType.Transition);
}
[Fact]
public void DoubleHyphen_Is_Recognized()
{
TestSingleToken("--", HtmlSymbolType.DoubleHyphen);
}
[Fact]
public void SingleHyphen_Is_Not_Recognized()
{
TestSingleToken("-", HtmlSymbolType.Text);
}
[Fact]
public void SingleHyphen_Mid_Text_Is_Not_Recognized_As_Separate_Token()
{
TestSingleToken("foo-bar", HtmlSymbolType.Text);
}
[Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment()
{
TestTokenizer("@* Foo * Bar * Baz *",
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(2, 0, 2, " Foo * Bar * Baz *", HtmlSymbolType.RazorComment));
}
[Fact]
public void Next_Ignores_Star_Without_Trailing_At()
{
TestTokenizer("@* Foo * Bar * Baz *@",
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(2, 0, 2, " Foo * Bar * Baz ", HtmlSymbolType.RazorComment),
new HtmlSymbol(19, 0, 19, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(20, 0, 20, "@", HtmlSymbolType.RazorCommentTransition));
}
[Fact]
public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment()
{
TestTokenizer("@* Foo Bar Baz *@",
new HtmlSymbol(0, 0, 0, "@", HtmlSymbolType.RazorCommentTransition),
new HtmlSymbol(1, 0, 1, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(2, 0, 2, " Foo Bar Baz ", HtmlSymbolType.RazorComment),
new HtmlSymbol(15, 0, 15, "*", HtmlSymbolType.RazorCommentStar),
new HtmlSymbol(16, 0, 16, "@", HtmlSymbolType.RazorCommentTransition));
}
}
}

28
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/HtmlTokenizerTestBase.cs vendored Normal file
View File

@@ -0,0 +1,28 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Text;
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
namespace System.Web.Razor.Test.Tokenizer
{
public abstract class HtmlTokenizerTestBase : TokenizerTestBase<HtmlSymbol, HtmlSymbolType>
{
private static HtmlSymbol _ignoreRemaining = new HtmlSymbol(0, 0, 0, String.Empty, HtmlSymbolType.Unknown);
protected override HtmlSymbol IgnoreRemaining
{
get { return _ignoreRemaining; }
}
protected override Tokenizer<HtmlSymbol, HtmlSymbolType> CreateTokenizer(ITextDocument source)
{
return new HtmlTokenizer(source);
}
protected void TestSingleToken(string text, HtmlSymbolType expectedSymbolType)
{
TestTokenizer(text, new HtmlSymbol(0, 0, 0, text, expectedSymbolType));
}
}
}

41
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/TokenizerLookaheadTest.cs vendored Normal file
View File

@@ -0,0 +1,41 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.IO;
using System.Web.Razor.Text;
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class TokenizerLookaheadTest : HtmlTokenizerTestBase
{
[Fact]
public void After_Cancelling_Lookahead_Tokenizer_Returns_Same_Tokens_As_It_Did_Before_Lookahead()
{
HtmlTokenizer tokenizer = new HtmlTokenizer(new SeekableTextReader(new StringReader("<foo>")));
using (tokenizer.Source.BeginLookahead())
{
Assert.Equal(new HtmlSymbol(0, 0, 0, "<", HtmlSymbolType.OpenAngle), tokenizer.NextSymbol());
Assert.Equal(new HtmlSymbol(1, 0, 1, "foo", HtmlSymbolType.Text), tokenizer.NextSymbol());
Assert.Equal(new HtmlSymbol(4, 0, 4, ">", HtmlSymbolType.CloseAngle), tokenizer.NextSymbol());
}
Assert.Equal(new HtmlSymbol(0, 0, 0, "<", HtmlSymbolType.OpenAngle), tokenizer.NextSymbol());
Assert.Equal(new HtmlSymbol(1, 0, 1, "foo", HtmlSymbolType.Text), tokenizer.NextSymbol());
Assert.Equal(new HtmlSymbol(4, 0, 4, ">", HtmlSymbolType.CloseAngle), tokenizer.NextSymbol());
}
[Fact]
public void After_Accepting_Lookahead_Tokenizer_Returns_Next_Token()
{
HtmlTokenizer tokenizer = new HtmlTokenizer(new SeekableTextReader(new StringReader("<foo>")));
using (LookaheadToken lookahead = tokenizer.Source.BeginLookahead())
{
Assert.Equal(new HtmlSymbol(0, 0, 0, "<", HtmlSymbolType.OpenAngle), tokenizer.NextSymbol());
Assert.Equal(new HtmlSymbol(1, 0, 1, "foo", HtmlSymbolType.Text), tokenizer.NextSymbol());
lookahead.Accept();
}
Assert.Equal(new HtmlSymbol(4, 0, 4, ">", HtmlSymbolType.CloseAngle), tokenizer.NextSymbol());
}
}
}

76
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/TokenizerTestBase.cs vendored Normal file
View File

@@ -0,0 +1,76 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Web.Razor.Text;
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public abstract class TokenizerTestBase<TSymbol, TSymbolType>
where TSymbol : SymbolBase<TSymbolType>
{
protected abstract TSymbol IgnoreRemaining { get; }
protected abstract Tokenizer<TSymbol, TSymbolType> CreateTokenizer(ITextDocument source);
protected void TestTokenizer(string input, params TSymbol[] expectedSymbols)
{
// Arrange
bool success = true;
StringBuilder output = new StringBuilder();
using (StringReader reader = new StringReader(input))
{
using (SeekableTextReader source = new SeekableTextReader(reader))
{
Tokenizer<TSymbol, TSymbolType> tokenizer = CreateTokenizer(source);
int counter = 0;
TSymbol current = null;
while ((current = tokenizer.NextSymbol()) != null)
{
if (counter >= expectedSymbols.Length)
{
output.AppendLine(String.Format("F: Expected: << Nothing >>; Actual: {0}", current));
success = false;
}
else if (ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
{
output.AppendLine(String.Format("P: Ignored {0}", current));
}
else
{
if (!Equals(expectedSymbols[counter], current))
{
output.AppendLine(String.Format("F: Expected: {0}; Actual: {1}", expectedSymbols[counter], current));
success = false;
}
else
{
output.AppendLine(String.Format("P: Expected: {0}", expectedSymbols[counter]));
}
counter++;
}
}
if (counter < expectedSymbols.Length && !ReferenceEquals(expectedSymbols[counter], IgnoreRemaining))
{
success = false;
for (; counter < expectedSymbols.Length; counter++)
{
output.AppendLine(String.Format("F: Expected: {0}; Actual: << None >>", expectedSymbols[counter]));
}
}
}
}
Assert.True(success, "\r\n" + output.ToString());
WriteTraceLine(output.Replace("{", "{{").Replace("}", "}}").ToString());
}
[Conditional("PARSER_TRACE")]
private static void WriteTraceLine(string format, params object[] args)
{
Trace.WriteLine(String.Format(format, args));
}
}
}

93
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/VBTokenizerCommentTest.cs vendored Normal file
View File

@@ -0,0 +1,93 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class VBTokenizerCommentTest : VBTokenizerTestBase
{
[Fact]
public void Next_Ignores_Star_At_EOF_In_RazorComment()
{
TestTokenizer("@* Foo * Bar * Baz *",
new VBSymbol(0, 0, 0, "@", VBSymbolType.RazorCommentTransition),
new VBSymbol(1, 0, 1, "*", VBSymbolType.RazorCommentStar),
new VBSymbol(2, 0, 2, " Foo * Bar * Baz *", VBSymbolType.RazorComment));
}
[Fact]
public void Next_Ignores_Star_Without_Trailing_At()
{
TestTokenizer("@* Foo * Bar * Baz *@",
new VBSymbol(0, 0, 0, "@", VBSymbolType.RazorCommentTransition),
new VBSymbol(1, 0, 1, "*", VBSymbolType.RazorCommentStar),
new VBSymbol(2, 0, 2, " Foo * Bar * Baz ", VBSymbolType.RazorComment),
new VBSymbol(19, 0, 19, "*", VBSymbolType.RazorCommentStar),
new VBSymbol(20, 0, 20, "@", VBSymbolType.RazorCommentTransition));
}
[Fact]
public void Next_Returns_RazorComment_Token_For_Entire_Razor_Comment()
{
TestTokenizer("@* Foo Bar Baz *@",
new VBSymbol(0, 0, 0, "@", VBSymbolType.RazorCommentTransition),
new VBSymbol(1, 0, 1, "*", VBSymbolType.RazorCommentStar),
new VBSymbol(2, 0, 2, " Foo Bar Baz ", VBSymbolType.RazorComment),
new VBSymbol(15, 0, 15, "*", VBSymbolType.RazorCommentStar),
new VBSymbol(16, 0, 16, "@", VBSymbolType.RazorCommentTransition));
}
[Fact]
public void Tick_Comment_Is_Recognized()
{
TestTokenizer("' Foo Bar Baz", new VBSymbol(0, 0, 0, "' Foo Bar Baz", VBSymbolType.Comment));
}
[Fact]
public void Tick_Comment_Is_Terminated_By_Newline()
{
TestTokenizer("' Foo Bar Baz\na", new VBSymbol(0, 0, 0, "' Foo Bar Baz", VBSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void LeftQuote_Comment_Is_Recognized()
{
// U+2018 - Left Quote: ‘
TestTokenizer("‘ Foo Bar Baz", new VBSymbol(0, 0, 0, "‘ Foo Bar Baz", VBSymbolType.Comment));
}
[Fact]
public void LeftQuote_Comment_Is_Terminated_By_Newline()
{
// U+2018 - Left Quote: ‘
TestTokenizer("‘ Foo Bar Baz\na", new VBSymbol(0, 0, 0, "‘ Foo Bar Baz", VBSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void RightQuote_Comment_Is_Recognized()
{
// U+2019 - Right Quote: ’
TestTokenizer("’ Foo Bar Baz", new VBSymbol(0, 0, 0, "’ Foo Bar Baz", VBSymbolType.Comment));
}
[Fact]
public void RightQuote_Comment_Is_Terminated_By_Newline()
{
// U+2019 - Right Quote: ’
TestTokenizer("’ Foo Bar Baz\na", new VBSymbol(0, 0, 0, "’ Foo Bar Baz", VBSymbolType.Comment), IgnoreRemaining);
}
[Fact]
public void Rem_Comment_Is_Recognized()
{
TestTokenizer("REM Foo Bar Baz", new VBSymbol(0, 0, 0, "REM Foo Bar Baz", VBSymbolType.Comment));
}
[Fact]
public void Rem_Comment_Is_Terminated_By_Newline()
{
TestTokenizer("REM Foo Bar Baz\na", new VBSymbol(0, 0, 0, "REM Foo Bar Baz", VBSymbolType.Comment), IgnoreRemaining);
}
}
}

267
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/VBTokenizerIdentifierTest.cs vendored Normal file
View File

@@ -0,0 +1,267 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class VBTokenizerIdentifierTest : VBTokenizerTestBase
{
[Fact]
public void Simple_Identifier_Is_Recognized()
{
TestTokenizer("foo", new VBSymbol(0, 0, 0, "foo", VBSymbolType.Identifier));
}
[Fact]
public void Escaped_Identifier_Terminates_At_EOF()
{
TestTokenizer("[foo", new VBSymbol(0, 0, 0, "[foo", VBSymbolType.Identifier));
}
[Fact]
public void Escaped_Identifier_Terminates_At_Whitespace()
{
TestTokenizer("[foo ", new VBSymbol(0, 0, 0, "[foo", VBSymbolType.Identifier), IgnoreRemaining);
}
[Fact]
public void Escaped_Identifier_Terminates_At_RightBracket_And_Does_Not_Read_TypeCharacter()
{
TestTokenizer("[foo]&", new VBSymbol(0, 0, 0, "[foo]", VBSymbolType.Identifier), IgnoreRemaining);
}
[Fact]
public void Identifier_Starting_With_Underscore_Is_Recognized()
{
TestTokenizer("_foo", new VBSymbol(0, 0, 0, "_foo", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Digits()
{
TestTokenizer("foo4", new VBSymbol(0, 0, 0, "foo4", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Titlecase_Letter()
{
TestTokenizer("ῼfoo", new VBSymbol(0, 0, 0, "ῼfoo", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Letter_Modifier()
{
TestTokenizer("ᵊfoo", new VBSymbol(0, 0, 0, "ᵊfoo", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Other_Letter()
{
TestTokenizer("ƻfoo", new VBSymbol(0, 0, 0, "ƻfoo", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Start_With_Number_Letter()
{
TestTokenizer("Ⅽool", new VBSymbol(0, 0, 0, "Ⅽool", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Non_Spacing_Mark()
{
TestTokenizer("foo\u0300", new VBSymbol(0, 0, 0, "foo\u0300", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Spacing_Combining_Mark()
{
TestTokenizer("fooः", new VBSymbol(0, 0, 0, "fooः", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Non_English_Digit()
{
TestTokenizer("foo١", new VBSymbol(0, 0, 0, "foo١", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Connector_Punctuation()
{
TestTokenizer("foo‿bar", new VBSymbol(0, 0, 0, "foo‿bar", VBSymbolType.Identifier));
}
[Fact]
public void Identifier_Can_Contain_Format_Character()
{
TestTokenizer("foo؃bar", new VBSymbol(0, 0, 0, "foo؃bar", VBSymbolType.Identifier));
}
[Fact]
public void Escaped_Keyword_Is_Recognized_As_Identifier()
{
TestSingleToken("[AddHandler]", VBSymbolType.Identifier);
}
[Fact]
public void Keywords_Are_Recognized_As_Keyword_Tokens()
{
TestKeyword("AddHandler", VBKeyword.AddHandler);
TestKeyword("AndAlso", VBKeyword.AndAlso);
TestKeyword("Byte", VBKeyword.Byte);
TestKeyword("Catch", VBKeyword.Catch);
TestKeyword("CDate", VBKeyword.CDate);
TestKeyword("CInt", VBKeyword.CInt);
TestKeyword("Const", VBKeyword.Const);
TestKeyword("CSng", VBKeyword.CSng);
TestKeyword("CULng", VBKeyword.CULng);
TestKeyword("Declare", VBKeyword.Declare);
TestKeyword("DirectCast", VBKeyword.DirectCast);
TestKeyword("Else", VBKeyword.Else);
TestKeyword("Enum", VBKeyword.Enum);
TestKeyword("Exit", VBKeyword.Exit);
TestKeyword("Friend", VBKeyword.Friend);
TestKeyword("GetXmlNamespace", VBKeyword.GetXmlNamespace);
TestKeyword("Handles", VBKeyword.Handles);
TestKeyword("In", VBKeyword.In);
TestKeyword("Is", VBKeyword.Is);
TestKeyword("Like", VBKeyword.Like);
TestKeyword("Mod", VBKeyword.Mod);
TestKeyword("MyBase", VBKeyword.MyBase);
TestKeyword("New", VBKeyword.New);
TestKeyword("AddressOf", VBKeyword.AddressOf);
TestKeyword("As", VBKeyword.As);
TestKeyword("ByVal", VBKeyword.ByVal);
TestKeyword("CBool", VBKeyword.CBool);
TestKeyword("CDbl", VBKeyword.CDbl);
TestKeyword("Class", VBKeyword.Class);
TestKeyword("Continue", VBKeyword.Continue);
TestKeyword("CStr", VBKeyword.CStr);
TestKeyword("CUShort", VBKeyword.CUShort);
TestKeyword("Default", VBKeyword.Default);
TestKeyword("Do", VBKeyword.Do);
TestKeyword("ElseIf", VBKeyword.ElseIf);
TestKeyword("Erase", VBKeyword.Erase);
TestKeyword("False", VBKeyword.False);
TestKeyword("Function", VBKeyword.Function);
TestKeyword("Global", VBKeyword.Global);
TestKeyword("If", VBKeyword.If);
TestKeyword("Inherits", VBKeyword.Inherits);
TestKeyword("IsNot", VBKeyword.IsNot);
TestKeyword("Long", VBKeyword.Long);
TestKeyword("Module", VBKeyword.Module);
TestKeyword("MyClass", VBKeyword.MyClass);
TestKeyword("Next", VBKeyword.Next);
TestKeyword("Alias", VBKeyword.Alias);
TestKeyword("Boolean", VBKeyword.Boolean);
TestKeyword("Call", VBKeyword.Call);
TestKeyword("CByte", VBKeyword.CByte);
TestKeyword("CDec", VBKeyword.CDec);
TestKeyword("CLng", VBKeyword.CLng);
TestKeyword("CSByte", VBKeyword.CSByte);
TestKeyword("CType", VBKeyword.CType);
TestKeyword("Date", VBKeyword.Date);
TestKeyword("Delegate", VBKeyword.Delegate);
TestKeyword("Double", VBKeyword.Double);
TestKeyword("End", VBKeyword.End);
TestKeyword("Error", VBKeyword.Error);
TestKeyword("Finally", VBKeyword.Finally);
TestKeyword("Get", VBKeyword.Get);
TestKeyword("GoSub", VBKeyword.GoSub);
TestKeyword("Implements", VBKeyword.Implements);
TestKeyword("Integer", VBKeyword.Integer);
TestKeyword("Let", VBKeyword.Let);
TestKeyword("Loop", VBKeyword.Loop);
TestKeyword("MustInherit", VBKeyword.MustInherit);
TestKeyword("Namespace", VBKeyword.Namespace);
TestKeyword("Not", VBKeyword.Not);
TestKeyword("And", VBKeyword.And);
TestKeyword("ByRef", VBKeyword.ByRef);
TestKeyword("Case", VBKeyword.Case);
TestKeyword("CChar", VBKeyword.CChar);
TestKeyword("Char", VBKeyword.Char);
TestKeyword("CObj", VBKeyword.CObj);
TestKeyword("CShort", VBKeyword.CShort);
TestKeyword("CUInt", VBKeyword.CUInt);
TestKeyword("Decimal", VBKeyword.Decimal);
TestKeyword("Dim", VBKeyword.Dim);
TestKeyword("Each", VBKeyword.Each);
TestKeyword("EndIf", VBKeyword.EndIf);
TestKeyword("Event", VBKeyword.Event);
TestKeyword("For", VBKeyword.For);
TestKeyword("GetType", VBKeyword.GetType);
TestKeyword("GoTo", VBKeyword.GoTo);
TestKeyword("Imports", VBKeyword.Imports);
TestKeyword("Interface", VBKeyword.Interface);
TestKeyword("Lib", VBKeyword.Lib);
TestKeyword("Me", VBKeyword.Me);
TestKeyword("MustOverride", VBKeyword.MustOverride);
TestKeyword("Narrowing", VBKeyword.Narrowing);
TestKeyword("Nothing", VBKeyword.Nothing);
TestKeyword("NotInheritable", VBKeyword.NotInheritable);
TestKeyword("On", VBKeyword.On);
TestKeyword("Or", VBKeyword.Or);
TestKeyword("Overrides", VBKeyword.Overrides);
TestKeyword("Property", VBKeyword.Property);
TestKeyword("ReadOnly", VBKeyword.ReadOnly);
TestKeyword("Resume", VBKeyword.Resume);
TestKeyword("Set", VBKeyword.Set);
TestKeyword("Single", VBKeyword.Single);
TestKeyword("String", VBKeyword.String);
TestKeyword("Then", VBKeyword.Then);
TestKeyword("Try", VBKeyword.Try);
TestKeyword("ULong", VBKeyword.ULong);
TestKeyword("Wend", VBKeyword.Wend);
TestKeyword("With", VBKeyword.With);
TestKeyword("NotOverridable", VBKeyword.NotOverridable);
TestKeyword("Operator", VBKeyword.Operator);
TestKeyword("OrElse", VBKeyword.OrElse);
TestKeyword("ParamArray", VBKeyword.ParamArray);
TestKeyword("Protected", VBKeyword.Protected);
TestKeyword("ReDim", VBKeyword.ReDim);
TestKeyword("Return", VBKeyword.Return);
TestKeyword("Shadows", VBKeyword.Shadows);
TestKeyword("Static", VBKeyword.Static);
TestKeyword("Structure", VBKeyword.Structure);
TestKeyword("Throw", VBKeyword.Throw);
TestKeyword("TryCast", VBKeyword.TryCast);
TestKeyword("UShort", VBKeyword.UShort);
TestKeyword("When", VBKeyword.When);
TestKeyword("WithEvents", VBKeyword.WithEvents);
TestKeyword("Object", VBKeyword.Object);
TestKeyword("Option", VBKeyword.Option);
TestKeyword("Overloads", VBKeyword.Overloads);
TestKeyword("Partial", VBKeyword.Partial);
TestKeyword("Public", VBKeyword.Public);
TestKeyword("SByte", VBKeyword.SByte);
TestKeyword("Shared", VBKeyword.Shared);
TestKeyword("Step", VBKeyword.Step);
TestKeyword("Sub", VBKeyword.Sub);
TestKeyword("To", VBKeyword.To);
TestKeyword("TypeOf", VBKeyword.TypeOf);
TestKeyword("Using", VBKeyword.Using);
TestKeyword("While", VBKeyword.While);
TestKeyword("WriteOnly", VBKeyword.WriteOnly);
TestKeyword("Of", VBKeyword.Of);
TestKeyword("Optional", VBKeyword.Optional);
TestKeyword("Overridable", VBKeyword.Overridable);
TestKeyword("Private", VBKeyword.Private);
TestKeyword("RaiseEvent", VBKeyword.RaiseEvent);
TestKeyword("RemoveHandler", VBKeyword.RemoveHandler);
TestKeyword("Select", VBKeyword.Select);
TestKeyword("Short", VBKeyword.Short);
TestKeyword("Stop", VBKeyword.Stop);
TestKeyword("SyncLock", VBKeyword.SyncLock);
TestKeyword("True", VBKeyword.True);
TestKeyword("UInteger", VBKeyword.UInteger);
TestKeyword("Variant", VBKeyword.Variant);
TestKeyword("Widening", VBKeyword.Widening);
TestKeyword("Xor", VBKeyword.Xor);
}
private void TestKeyword(string keyword, VBKeyword keywordType)
{
TestTokenizer(keyword, new VBSymbol(0, 0, 0, keyword, VBSymbolType.Keyword) { Keyword = keywordType });
}
}
}

182
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/VBTokenizerLiteralTest.cs vendored Normal file
View File

@@ -0,0 +1,182 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class VBTokenizerLiteralTest : VBTokenizerTestBase
{
[Fact]
public void Decimal_Integer_Literal_Is_Recognized()
{
TestSingleToken("01189998819991197253", VBSymbolType.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffixes_Are_Recognized_In_Decimal_Literal()
{
TestSingleToken("42S", VBSymbolType.IntegerLiteral);
TestSingleToken("42I", VBSymbolType.IntegerLiteral);
TestSingleToken("42L", VBSymbolType.IntegerLiteral);
TestSingleToken("42US", VBSymbolType.IntegerLiteral);
TestSingleToken("42UI", VBSymbolType.IntegerLiteral);
TestSingleToken("42UL", VBSymbolType.IntegerLiteral);
}
[Fact]
public void Hex_Integer_Literal_Is_Recognized()
{
TestSingleToken("&HDeadBeef", VBSymbolType.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffixes_Are_Recognized_In_Hex_Literal()
{
TestSingleToken("&HDeadBeefS", VBSymbolType.IntegerLiteral);
TestSingleToken("&HDeadBeefI", VBSymbolType.IntegerLiteral);
TestSingleToken("&HDeadBeefL", VBSymbolType.IntegerLiteral);
TestSingleToken("&HDeadBeefUS", VBSymbolType.IntegerLiteral);
TestSingleToken("&HDeadBeefUI", VBSymbolType.IntegerLiteral);
TestSingleToken("&HDeadBeefUL", VBSymbolType.IntegerLiteral);
}
[Fact]
public void Octal_Integer_Literal_Is_Recognized()
{
TestSingleToken("&O77", VBSymbolType.IntegerLiteral);
}
[Fact]
public void Integer_Type_Suffixes_Are_Recognized_In_Octal_Literal()
{
TestSingleToken("&O77S", VBSymbolType.IntegerLiteral);
TestSingleToken("&O77I", VBSymbolType.IntegerLiteral);
TestSingleToken("&O77L", VBSymbolType.IntegerLiteral);
TestSingleToken("&O77US", VBSymbolType.IntegerLiteral);
TestSingleToken("&O77UI", VBSymbolType.IntegerLiteral);
TestSingleToken("&O77UL", VBSymbolType.IntegerLiteral);
}
[Fact]
public void Incomplete_Type_Suffix_Is_Recognized()
{
TestSingleToken("42U", VBSymbolType.IntegerLiteral);
TestSingleToken("&H42U", VBSymbolType.IntegerLiteral);
TestSingleToken("&O77U", VBSymbolType.IntegerLiteral);
}
[Fact]
public void Integer_With_FloatingPoint_Type_Suffix_Is_Recognized_As_FloatingPointLiteral()
{
TestSingleToken("42F", VBSymbolType.FloatingPointLiteral);
TestSingleToken("42R", VBSymbolType.FloatingPointLiteral);
TestSingleToken("42D", VBSymbolType.FloatingPointLiteral);
}
[Fact]
public void Simple_FloatingPoint_Is_Recognized()
{
TestSingleToken("3.14159", VBSymbolType.FloatingPointLiteral);
}
[Fact]
public void Integer_With_Exponent_Is_Recognized()
{
TestSingleToken("1E10", VBSymbolType.FloatingPointLiteral);
TestSingleToken("1e10", VBSymbolType.FloatingPointLiteral);
TestSingleToken("1E+10", VBSymbolType.FloatingPointLiteral);
TestSingleToken("1e+10", VBSymbolType.FloatingPointLiteral);
TestSingleToken("1E-10", VBSymbolType.FloatingPointLiteral);
TestSingleToken("1e-10", VBSymbolType.FloatingPointLiteral);
}
[Fact]
public void Simple_FloatingPoint_With_Exponent_Is_Recognized()
{
TestSingleToken("3.14159e10", VBSymbolType.FloatingPointLiteral);
}
[Fact]
public void FloatingPoint_Between_Zero_And_One_Is_Recognized()
{
TestSingleToken(".314159e1", VBSymbolType.FloatingPointLiteral);
}
[Fact]
public void Simple_String_Literal_Is_Recognized()
{
TestSingleToken("\"Foo Bar Baz\"", VBSymbolType.StringLiteral);
}
[Fact]
public void Two_Double_Quotes_Are_Recognized_As_Escape_Sequence()
{
TestSingleToken("\"Foo \"\"Bar\"\" Baz\"", VBSymbolType.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_At_EOF()
{
TestSingleToken("\"Foo", VBSymbolType.StringLiteral);
}
[Fact]
public void String_Literal_Is_Terminated_At_EOL()
{
TestTokenizer("\"Foo\nBar", new VBSymbol(0, 0, 0, "\"Foo", VBSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void Character_Literal_Is_Recognized_By_Trailing_C_After_String_Literal()
{
TestSingleToken("\"abc\"c", VBSymbolType.CharacterLiteral);
}
[Fact]
public void LeftDoubleQuote_Is_Valid_DoubleQuote()
{
// Repeat all the above tests with Unicode Left Double Quote Character U+201C: “
TestSingleToken("“Foo Bar Baz“", VBSymbolType.StringLiteral);
TestSingleToken("“Foo ““Bar““ Baz“", VBSymbolType.StringLiteral);
TestSingleToken("“Foo", VBSymbolType.StringLiteral);
TestSingleToken("“abc“c", VBSymbolType.CharacterLiteral);
TestTokenizer("“Foo\nBar", new VBSymbol(0, 0, 0, "“Foo", VBSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void RightDoubleQuote_Is_Valid_DoubleQuote()
{
// Repeat all the above tests with Unicode Right Double Quote Character U+201D: ”
TestSingleToken("”Foo Bar Baz”", VBSymbolType.StringLiteral);
TestSingleToken("”Foo ””Bar”” Baz”", VBSymbolType.StringLiteral);
TestSingleToken("”Foo", VBSymbolType.StringLiteral);
TestSingleToken("”abc”c", VBSymbolType.CharacterLiteral);
TestTokenizer("”Foo\nBar", new VBSymbol(0, 0, 0, "”Foo", VBSymbolType.StringLiteral), IgnoreRemaining);
}
[Fact]
public void DateLiteral_Is_Recognized()
{
TestSingleToken("# 8/23/1970 3:45:39AM #", VBSymbolType.DateLiteral);
}
[Fact]
public void DateLiteral_Is_Terminated_At_EndHash()
{
TestTokenizer("# 8/23/1970 # 3:45:39AM", new VBSymbol(0, 0, 0, "# 8/23/1970 #", VBSymbolType.DateLiteral), IgnoreRemaining);
}
[Fact]
public void DateLiteral_Is_Terminated_At_EOF()
{
TestSingleToken("# 8/23/1970 3:45:39AM", VBSymbolType.DateLiteral);
}
[Fact]
public void DateLiteral_Is_Terminated_At_EOL()
{
TestTokenizer("# 8/23/1970\n3:45:39AM", new VBSymbol(0, 0, 0, "# 8/23/1970", VBSymbolType.DateLiteral), IgnoreRemaining);
}
}
}

154
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/VBTokenizerOperatorsTest.cs vendored Normal file
View File

@@ -0,0 +1,154 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
namespace System.Web.Razor.Test.Tokenizer
{
public class VBTokenizerOperatorsTest : VBTokenizerTestBase
{
[Fact]
public void Line_Continuation_Character_Is_Recognized()
{
TestSingleToken("_", VBSymbolType.LineContinuation);
}
[Fact]
public void LeftParen_Is_Recognized()
{
TestSingleToken("(", VBSymbolType.LeftParenthesis);
}
[Fact]
public void RightParen_Is_Recognized()
{
TestSingleToken(")", VBSymbolType.RightParenthesis);
}
[Fact]
public void LeftBracket_Is_Recognized()
{
TestSingleToken("[", VBSymbolType.LeftBracket);
}
[Fact]
public void RightBracket_Is_Recognized()
{
TestSingleToken("]", VBSymbolType.RightBracket);
}
[Fact]
public void LeftBrace_Is_Recognized()
{
TestSingleToken("{", VBSymbolType.LeftBrace);
}
[Fact]
public void RightBrace_Is_Recognized()
{
TestSingleToken("}", VBSymbolType.RightBrace);
}
[Fact]
public void Bang_Is_Recognized()
{
TestSingleToken("!", VBSymbolType.Bang);
}
[Fact]
public void Hash_Is_Recognized()
{
TestSingleToken("#", VBSymbolType.Hash);
}
[Fact]
public void Comma_Is_Recognized()
{
TestSingleToken(",", VBSymbolType.Comma);
}
[Fact]
public void Dot_Is_Recognized()
{
TestSingleToken(".", VBSymbolType.Dot);
}
[Fact]
public void Colon_Is_Recognized()
{
TestSingleToken(":", VBSymbolType.Colon);
}
[Fact]
public void QuestionMark_Is_Recognized()
{
TestSingleToken("?", VBSymbolType.QuestionMark);
}
[Fact]
public void Concatenation_Is_Recognized()
{
TestSingleToken("&", VBSymbolType.Concatenation);
}
[Fact]
public void Multiply_Is_Recognized()
{
TestSingleToken("*", VBSymbolType.Multiply);
}
[Fact]
public void Add_Is_Recognized()
{
TestSingleToken("+", VBSymbolType.Add);
}
[Fact]
public void Subtract_Is_Recognized()
{
TestSingleToken("-", VBSymbolType.Subtract);
}
[Fact]
public void Divide_Is_Recognized()
{
TestSingleToken("/", VBSymbolType.Divide);
}
[Fact]
public void IntegerDivide_Is_Recognized()
{
TestSingleToken("\\", VBSymbolType.IntegerDivide);
}
[Fact]
public void Exponentiation_Is_Recognized()
{
TestSingleToken("^", VBSymbolType.Exponentiation);
}
[Fact]
public void Equal_Is_Recognized()
{
TestSingleToken("=", VBSymbolType.Equal);
}
[Fact]
public void LessThan_Is_Recognized()
{
TestSingleToken("<", VBSymbolType.LessThan);
}
[Fact]
public void GreaterThan_Is_Recognized()
{
TestSingleToken(">", VBSymbolType.GreaterThan);
}
[Fact]
public void Dollar_Is_Recognized()
{
TestSingleToken("$", VBSymbolType.Dollar);
}
}
}

96
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/VBTokenizerTest.cs vendored Normal file
View File

@@ -0,0 +1,96 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
using Xunit;
using Assert = Microsoft.TestCommon.AssertEx;
namespace System.Web.Razor.Test.Tokenizer
{
public class VBTokenizerTest : VBTokenizerTestBase
{
[Fact]
public void Constructor_Throws_ArgNull_If_Null_Source_Provided()
{
Assert.ThrowsArgumentNull(() => new CSharpTokenizer(null), "source");
}
[Fact]
public void Next_Returns_Null_When_EOF_Reached()
{
TestTokenizer("");
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_CR()
{
TestTokenizer("\r\ra",
new VBSymbol(0, 0, 0, "\r", VBSymbolType.NewLine),
new VBSymbol(1, 1, 0, "\r", VBSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_LF()
{
TestTokenizer("\n\na",
new VBSymbol(0, 0, 0, "\n", VBSymbolType.NewLine),
new VBSymbol(1, 1, 0, "\n", VBSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_NEL()
{
// NEL: Unicode "Next Line" U+0085
TestTokenizer("\u0085\u0085a",
new VBSymbol(0, 0, 0, "\u0085", VBSymbolType.NewLine),
new VBSymbol(1, 1, 0, "\u0085", VBSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_Line_Separator()
{
// Unicode "Line Separator" U+2028
TestTokenizer("\u2028\u2028a",
new VBSymbol(0, 0, 0, "\u2028", VBSymbolType.NewLine),
new VBSymbol(1, 1, 0, "\u2028", VBSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator()
{
// Unicode "Paragraph Separator" U+2029
TestTokenizer("\u2029\u2029a",
new VBSymbol(0, 0, 0, "\u2029", VBSymbolType.NewLine),
new VBSymbol(1, 1, 0, "\u2029", VBSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Single_Newline_Token_For_CRLF()
{
TestTokenizer("\r\n\r\na",
new VBSymbol(0, 0, 0, "\r\n", VBSymbolType.NewLine),
new VBSymbol(2, 1, 0, "\r\n", VBSymbolType.NewLine),
IgnoreRemaining);
}
[Fact]
public void Next_Returns_Token_For_Whitespace_Characters()
{
TestTokenizer(" \f\t\u000B \n ",
new VBSymbol(0, 0, 0, " \f\t\u000B ", VBSymbolType.WhiteSpace),
new VBSymbol(5, 0, 5, "\n", VBSymbolType.NewLine),
new VBSymbol(6, 1, 0, " ", VBSymbolType.WhiteSpace));
}
[Fact]
public void Transition_Is_Recognized()
{
TestSingleToken("@", VBSymbolType.Transition);
}
}
}

28
external/aspnetwebstack/test/System.Web.Razor.Test/Tokenizer/VBTokenizerTestBase.cs vendored Normal file
View File

@@ -0,0 +1,28 @@
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
using System.Web.Razor.Text;
using System.Web.Razor.Tokenizer;
using System.Web.Razor.Tokenizer.Symbols;
namespace System.Web.Razor.Test.Tokenizer
{
public abstract class VBTokenizerTestBase : TokenizerTestBase<VBSymbol, VBSymbolType>
{
private static VBSymbol _ignoreRemaining = new VBSymbol(0, 0, 0, String.Empty, VBSymbolType.Unknown);
protected override VBSymbol IgnoreRemaining
{
get { return _ignoreRemaining; }
}
protected override Tokenizer<VBSymbol, VBSymbolType> CreateTokenizer(ITextDocument source)
{
return new VBTokenizer(source);
}
protected void TestSingleToken(string text, VBSymbolType expectedSymbolType)
{
TestTokenizer(text, new VBSymbol(0, 0, 0, text, expectedSymbolType));
}
}
}