Merge pull request #184 from alexej-v/errInvalidToken_panic_fix

Fix lexer.errInvalidToken panic

jlexer/lexer.go

@@ -240,7 +240,7 @@ func (r *Lexer) fetchNumber() {
 
 // findStringLen tries to scan into the string literal for ending quote char to determine required size.
 // The size will be exact if no escapes are present and may be inexact if there are escaped chars.
-func findStringLen(data []byte) (hasEscapes bool, length int) {
+func findStringLen(data []byte) (isValid, hasEscapes bool, length int) {
 	delta := 0
 
 	for i := 0; i < len(data); i++ {
@@ -252,11 +252,11 @@ func findStringLen(data []byte) (hasEscapes bool, length int) {
 				delta++
 			}
 		case '"':
-			return (delta > 0), (i - delta)
+			return true, (delta > 0), (i - delta)
 		}
 	}
 
-	return false, len(data)
+	return false, false, len(data)
 }
 
 // getu4 decodes \uXXXX from the beginning of s, returning the hex value,
@@ -342,7 +342,12 @@ func (r *Lexer) fetchString() {
 	r.pos++
 	data := r.Data[r.pos:]
 
-	hasEscapes, length := findStringLen(data)
+	isValid, hasEscapes, length := findStringLen(data)
+	if !isValid {
+		r.pos += length
+		r.errParse("unterminated string literal")
+		return
+	}
 	if !hasEscapes {
 		r.token.byteValue = data[:length]
 		r.pos += length + 1

jlexer/lexer_test.go

@@ -312,3 +312,22 @@ func TestJsonNumber(t *testing.T) {
 		}
 	}
 }
+
+func TestFetchStringUnterminatedString(t *testing.T) {
+	for _, test := range []struct {
+		data []byte
+	}{
+		{data: []byte(`"sting without trailing quote`)},
+		{data: []byte(`"\"`)},
+		{data: []byte{'"'}},
+	} {
+		l := Lexer{Data: test.data}
+		l.fetchString()
+		if l.pos > len(l.Data) {
+			t.Errorf("fetchString(%s): pos should not be greater than length of Data", test.data)
+		}
+		if l.Error() == nil {
+			t.Errorf("fetchString(%s): should add parsing error", test.data)
+		}
+	}
+}