From e8d450bb245249eaaafe329838a26750821bb649 Mon Sep 17 00:00:00 2001 From: Valentin Gosu Date: Fri, 5 Feb 2016 14:43:45 +0100 Subject: [PATCH] Bug 241788 - mozilla::Tokenizer - token type for \n should whitespace if given in constructor r=honzab --- xpcom/ds/Tokenizer.cpp | 4 ++-- xpcom/tests/gtest/TestTokenizer.cpp | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/xpcom/ds/Tokenizer.cpp b/xpcom/ds/Tokenizer.cpp index 81d6490e3ba..6fa9ff56a59 100644 --- a/xpcom/ds/Tokenizer.cpp +++ b/xpcom/ds/Tokenizer.cpp @@ -260,12 +260,12 @@ Tokenizer::Parse(Token& aToken) const state = PARSE_WORD; } else if (IsNumber(*next)) { state = PARSE_INTEGER; + } else if (strchr(mWhitespaces, *next)) { // not UTF-8 friendly? + state = PARSE_WS; } else if (*next == '\r') { state = PARSE_CRLF; } else if (*next == '\n') { state = PARSE_LF; - } else if (strchr(mWhitespaces, *next)) { // not UTF-8 friendly? - state = PARSE_WS; } else { state = PARSE_CHAR; } diff --git a/xpcom/tests/gtest/TestTokenizer.cpp b/xpcom/tests/gtest/TestTokenizer.cpp index c98d8bcbf5e..019ffedf50e 100644 --- a/xpcom/tests/gtest/TestTokenizer.cpp +++ b/xpcom/tests/gtest/TestTokenizer.cpp @@ -526,6 +526,21 @@ TEST(Tokenizer, SkipWhites) EXPECT_TRUE(p.CheckEOF()); } +TEST(Tokenizer, SkipCustomWhites) +{ + Tokenizer p("Text1 \n\r\t.Text2 \n\r\t.", " \n\r\t."); + + EXPECT_TRUE(p.CheckWord("Text1")); + p.SkipWhites(); + EXPECT_TRUE(p.CheckWord("Text2")); + EXPECT_TRUE(p.CheckWhite()); + EXPECT_TRUE(p.CheckWhite()); + EXPECT_TRUE(p.CheckWhite()); + EXPECT_TRUE(p.CheckWhite()); + EXPECT_TRUE(p.CheckWhite()); + EXPECT_TRUE(p.CheckEOF()); +} + TEST(Tokenizer, IntegerReading) { #define INT_6_BITS 64U