105 lines
3.8 KiB
C#
105 lines
3.8 KiB
C#
// Copyright (c) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
|
|
|
|
using System.Web.Razor.Tokenizer;
|
|
using System.Web.Razor.Tokenizer.Symbols;
|
|
using Xunit;
|
|
using Assert = Microsoft.TestCommon.AssertEx;
|
|
|
|
namespace System.Web.Razor.Test.Tokenizer
|
|
{
|
|
public class CSharpTokenizerTest : CSharpTokenizerTestBase
|
|
{
|
|
[Fact]
|
|
public void Constructor_Throws_ArgNull_If_Null_Source_Provided()
|
|
{
|
|
Assert.ThrowsArgumentNull(() => new CSharpTokenizer(null), "source");
|
|
}
|
|
|
|
[Fact]
|
|
public void Next_Returns_Null_When_EOF_Reached()
|
|
{
|
|
TestTokenizer("");
|
|
}
|
|
|
|
[Fact]
|
|
public void Next_Returns_Newline_Token_For_Single_CR()
|
|
{
|
|
TestTokenizer("\r\ra",
|
|
new CSharpSymbol(0, 0, 0, "\r", CSharpSymbolType.NewLine),
|
|
new CSharpSymbol(1, 1, 0, "\r", CSharpSymbolType.NewLine),
|
|
IgnoreRemaining);
|
|
}
|
|
|
|
[Fact]
|
|
public void Next_Returns_Newline_Token_For_Single_LF()
|
|
{
|
|
TestTokenizer("\n\na",
|
|
new CSharpSymbol(0, 0, 0, "\n", CSharpSymbolType.NewLine),
|
|
new CSharpSymbol(1, 1, 0, "\n", CSharpSymbolType.NewLine),
|
|
IgnoreRemaining);
|
|
}
|
|
|
|
[Fact]
|
|
public void Next_Returns_Newline_Token_For_Single_NEL()
|
|
{
|
|
// NEL: Unicode "Next Line" U+0085
|
|
TestTokenizer("\u0085\u0085a",
|
|
new CSharpSymbol(0, 0, 0, "\u0085", CSharpSymbolType.NewLine),
|
|
new CSharpSymbol(1, 1, 0, "\u0085", CSharpSymbolType.NewLine),
|
|
IgnoreRemaining);
|
|
}
|
|
|
|
[Fact]
|
|
public void Next_Returns_Newline_Token_For_Single_Line_Separator()
|
|
{
|
|
// Unicode "Line Separator" U+2028
|
|
TestTokenizer("\u2028\u2028a",
|
|
new CSharpSymbol(0, 0, 0, "\u2028", CSharpSymbolType.NewLine),
|
|
new CSharpSymbol(1, 1, 0, "\u2028", CSharpSymbolType.NewLine),
|
|
IgnoreRemaining);
|
|
}
|
|
|
|
[Fact]
|
|
public void Next_Returns_Newline_Token_For_Single_Paragraph_Separator()
|
|
{
|
|
// Unicode "Paragraph Separator" U+2029
|
|
TestTokenizer("\u2029\u2029a",
|
|
new CSharpSymbol(0, 0, 0, "\u2029", CSharpSymbolType.NewLine),
|
|
new CSharpSymbol(1, 1, 0, "\u2029", CSharpSymbolType.NewLine),
|
|
IgnoreRemaining);
|
|
}
|
|
|
|
[Fact]
|
|
public void Next_Returns_Single_Newline_Token_For_CRLF()
|
|
{
|
|
TestTokenizer("\r\n\r\na",
|
|
new CSharpSymbol(0, 0, 0, "\r\n", CSharpSymbolType.NewLine),
|
|
new CSharpSymbol(2, 1, 0, "\r\n", CSharpSymbolType.NewLine),
|
|
IgnoreRemaining);
|
|
}
|
|
|
|
[Fact]
|
|
public void Next_Returns_Token_For_Whitespace_Characters()
|
|
{
|
|
TestTokenizer(" \f\t\u000B \n ",
|
|
new CSharpSymbol(0, 0, 0, " \f\t\u000B ", CSharpSymbolType.WhiteSpace),
|
|
new CSharpSymbol(5, 0, 5, "\n", CSharpSymbolType.NewLine),
|
|
new CSharpSymbol(6, 1, 0, " ", CSharpSymbolType.WhiteSpace));
|
|
}
|
|
|
|
[Fact]
|
|
public void Transition_Is_Recognized()
|
|
{
|
|
TestSingleToken("@", CSharpSymbolType.Transition);
|
|
}
|
|
|
|
[Fact]
|
|
public void Transition_Is_Recognized_As_SingleCharacter()
|
|
{
|
|
TestTokenizer("@(",
|
|
new CSharpSymbol(0, 0, 0, "@", CSharpSymbolType.Transition),
|
|
new CSharpSymbol(1, 0, 1, "(", CSharpSymbolType.LeftParenthesis));
|
|
}
|
|
}
|
|
}
|