Bug 914072 - Have nsCSSScanner store any implied characters at EOF needed for correct serialization. r=dbaron

layout/style/nsCSSScanner.cpp

@@ -350,6 +350,7 @@ nsCSSScanner::nsCSSScanner(const nsAString& aBuffer, uint32_t aLineNumber)
   , mTokenLineOffset(0)
   , mTokenOffset(0)
   , mRecordStartOffset(0)
+  , mEOFCharacters(eEOFCharacters_None)
   , mReporter(nullptr)
   , mSVGMode(false)
   , mRecording(false)
@@ -512,13 +513,22 @@ nsCSSScanner::SkipComment()
     int32_t ch = Peek();
     if (ch < 0) {
       mReporter->ReportUnexpectedEOF("PECommentEOF");
+      SetEOFCharacters(eEOFCharacters_Asterisk | eEOFCharacters_Slash);
       return;
     }
-    if (ch == '*' && Peek(1) == '/') {
-      Advance(2);
-      return;
-    }
-    if (IsVertSpace(ch)) {
+    if (ch == '*') {
+      Advance();
+      ch = Peek();
+      if (ch < 0) {
+        mReporter->ReportUnexpectedEOF("PECommentEOF");
+        SetEOFCharacters(eEOFCharacters_Slash);
+        return;
+      }
+      if (ch == '/') {
+        Advance();
+        return;
+      }
+    } else if (IsVertSpace(ch)) {
       AdvanceLine();
     } else {
       Advance();
@@ -543,8 +553,11 @@ nsCSSScanner::GatherEscape(nsString& aOutput, bool aInString)
     // the backslash on the floor.  Otherwise, we want to treat it as a U+FFFD
     // character.
     Advance();
-    if (!aInString) {
+    if (aInString) {
+      SetEOFCharacters(eEOFCharacters_DropBackslash);
+    } else {
       aOutput.Append(UCS2_REPLACEMENT_CHAR);
+      SetEOFCharacters(eEOFCharacters_ReplacementChar);
     }
     return true;
   }
@@ -679,6 +692,8 @@ bool
 nsCSSScanner::ScanIdent(nsCSSToken& aToken)
 {
   if (MOZ_UNLIKELY(!GatherText(IS_IDCHAR, aToken.mIdent))) {
+    MOZ_ASSERT(Peek() == '\\',
+               "unexpected IsIdentStart character that did not begin an ident");
     aToken.mSymbol = Peek();
     Advance();
     return true;
@@ -911,6 +926,8 @@ nsCSSScanner::ScanString(nsCSSToken& aToken)
 
     int32_t ch = Peek();
     if (ch == -1) {
+      AddEOFCharacters(aStop == '"' ? eEOFCharacters_DoubleQuote :
+                                      eEOFCharacters_SingleQuote);
       break; // EOF ends a string token with no error.
     }
     if (ch == aStop) {
@@ -1016,6 +1033,79 @@ nsCSSScanner::ScanURange(nsCSSToken& aResult)
   return true;
 }
 
+#ifdef DEBUG
+/* static */ void
+nsCSSScanner::AssertEOFCharactersValid(uint32_t c)
+{
+  MOZ_ASSERT(c == eEOFCharacters_None ||
+             c == eEOFCharacters_ReplacementChar ||
+             c == eEOFCharacters_Slash ||
+             c == (eEOFCharacters_Asterisk |
+                   eEOFCharacters_Slash) ||
+             c == eEOFCharacters_DoubleQuote ||
+             c == eEOFCharacters_SingleQuote ||
+             c == (eEOFCharacters_DropBackslash |
+                   eEOFCharacters_DoubleQuote) ||
+             c == (eEOFCharacters_DropBackslash |
+                   eEOFCharacters_SingleQuote) ||
+             c == eEOFCharacters_CloseParen ||
+             c == (eEOFCharacters_ReplacementChar |
+                   eEOFCharacters_CloseParen) ||
+             c == (eEOFCharacters_DoubleQuote |
+                   eEOFCharacters_CloseParen) ||
+             c == (eEOFCharacters_SingleQuote |
+                   eEOFCharacters_CloseParen) ||
+             c == (eEOFCharacters_DropBackslash |
+                   eEOFCharacters_DoubleQuote |
+                   eEOFCharacters_CloseParen) ||
+             c == (eEOFCharacters_DropBackslash |
+                   eEOFCharacters_SingleQuote |
+                   eEOFCharacters_CloseParen),
+             "invalid EOFCharacters value");
+}
+#endif
+
+void
+nsCSSScanner::SetEOFCharacters(uint32_t aEOFCharacters)
+{
+  mEOFCharacters = EOFCharacters(aEOFCharacters);
+}
+
+void
+nsCSSScanner::AddEOFCharacters(uint32_t aEOFCharacters)
+{
+  mEOFCharacters = EOFCharacters(mEOFCharacters | aEOFCharacters);
+}
+
+static const PRUnichar kImpliedEOFCharacters[] = {
+  UCS2_REPLACEMENT_CHAR, '*', '/', '"', '\'', ')', 0
+};
+
+/* static */ void
+nsCSSScanner::AdjustTokenStreamForEOFCharacters(EOFCharacters aEOFCharacters,
+                                                nsAString& aResult)
+{
+  uint32_t c = aEOFCharacters;
+
+  // First, handle eEOFCharacters_DropBackslash.
+  if (c & eEOFCharacters_DropBackslash) {
+    MOZ_ASSERT(aResult[aResult.Length() - 1] == '\\');
+    aResult.SetLength(aResult.Length() - 1);
+  }
+
+  c >>= 1;
+
+  // All of the remaining EOFCharacters bits represent appended characters,
+  // and the bits are in the order that they need appending.
+  for (const PRUnichar* p = kImpliedEOFCharacters; *p && c; p++, c >>= 1) {
+    if (c & 1) {
+      aResult.Append(*p);
+    }
+  }
+
+  MOZ_ASSERT(c == 0, "too many bits in mEOFCharacters");
+}
+
 /**
  * Consume the part of an URL token after the initial 'url('.  Caller
  * is assumed to have consumed 'url(' already.  Will always produce
@@ -1058,6 +1148,9 @@ nsCSSScanner::NextURL(nsCSSToken& aToken)
   if (MOZ_LIKELY(ch < 0 || ch == ')')) {
     Advance();
     aToken.mType = eCSSToken_URL;
+    if (ch < 0) {
+      AddEOFCharacters(eEOFCharacters_CloseParen);
+    }
   } else {
     mSeenBadToken = true;
     aToken.mType = eCSSToken_Bad_URL;

layout/style/nsCSSScanner.h

@@ -194,6 +194,46 @@ class nsCSSScanner {
   // input to aBuffer.
   void StopRecording(nsString& aBuffer);
 
+  enum EOFCharacters {
+    eEOFCharacters_None =                    0x0000,
+
+    // to handle \<EOF> inside strings
+    eEOFCharacters_DropBackslash =           0x0001,
+
+    // to handle \<EOF> outside strings
+    eEOFCharacters_ReplacementChar =         0x0002,
+
+    // to close comments
+    eEOFCharacters_Asterisk =                0x0004,
+    eEOFCharacters_Slash =                   0x0008,
+
+    // to close double-quoted strings
+    eEOFCharacters_DoubleQuote =             0x0010,
+
+    // to close single-quoted strings
+    eEOFCharacters_SingleQuote =             0x0020,
+
+    // to close URLs
+    eEOFCharacters_CloseParen =              0x0040,
+  };
+
+  // Appends or drops any characters to/from the specified string
+  // the input stream to make the last token not rely on special EOF handling
+  // behavior.
+  static void AdjustTokenStreamForEOFCharacters(EOFCharacters aEOFCharacters,
+                                                nsAString& aString);
+
+  EOFCharacters GetEOFCharacters() const {
+#ifdef DEBUG
+    AssertEOFCharactersValid(mEOFCharacters);
+#endif
+    return mEOFCharacters;
+  }
+
+#ifdef DEBUG
+  static void AssertEOFCharactersValid(uint32_t c);
+#endif
+
 protected:
   int32_t Peek(uint32_t n = 0);
   void Advance(uint32_t n = 1);
@@ -212,6 +252,9 @@ protected:
   bool ScanString(nsCSSToken& aResult);
   bool ScanURange(nsCSSToken& aResult);
 
+  void SetEOFCharacters(uint32_t aEOFCharacters);
+  void AddEOFCharacters(uint32_t aEOFCharacters);
+
   const PRUnichar *mBuffer;
   uint32_t mOffset;
   uint32_t mCount;
@@ -224,6 +267,7 @@ protected:
   uint32_t mTokenOffset;
 
   uint32_t mRecordStartOffset;
+  EOFCharacters mEOFCharacters;
 
   mozilla::css::ErrorReporter *mReporter;