diff --git a/jlexer/lexer.go b/jlexer/lexer.go index ddd376b..74b5bda 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -5,6 +5,7 @@ package jlexer import ( + "bytes" "encoding/base64" "encoding/json" "errors" @@ -32,9 +33,10 @@ const ( type token struct { kind tokenKind // Type of a token. - boolValue bool // Value if a boolean literal token. - byteValue []byte // Raw value of a token. - delimValue byte + boolValue bool // Value if a boolean literal token. + byteValueCloned bool // true if byteValue was allocated and does not refer to original json body + byteValue []byte // Raw value of a token. + delimValue byte } // Lexer is a JSON lexer: it iterates over JSON tokens in a byte slice. @@ -240,23 +242,50 @@ func (r *Lexer) fetchNumber() { // findStringLen tries to scan into the string literal for ending quote char to determine required size. // The size will be exact if no escapes are present and may be inexact if there are escaped chars. -func findStringLen(data []byte) (isValid, hasEscapes bool, length int) { - delta := 0 - - for i := 0; i < len(data); i++ { - switch data[i] { - case '\\': - i++ - delta++ - if i < len(data) && data[i] == 'u' { - delta++ - } - case '"': - return true, (delta > 0), (i - delta) +func findStringLen(data []byte) (isValid bool, length int) { + for { + idx := bytes.IndexByte(data, '"') + if idx == -1 { + return false, len(data) } + if idx == 0 || (idx > 0 && data[idx-1] != '\\') { + return true, length + idx + } + length += idx + 1 + data = data[idx+1:] + } +} + +// unescapeStringToken performs unescaping of string token. +// if no escaping is needed, original string is returned, otherwise - a new one allocated +func (r *Lexer) unescapeStringToken() (err error) { + data := r.token.byteValue + var unescapedData []byte + + for { + i := bytes.IndexByte(data, '\\') + if i == -1 { + break + } + + escapedRune, escapedBytes, err := decodeEscape(data[i:]) + if err != nil { + r.errParse(err.Error()) + return err + } + var d [4]byte + s := utf8.EncodeRune(d[:], escapedRune) + unescapedData = append(unescapedData, data[:i]...) + unescapedData = append(unescapedData, d[:s]...) + + data = data[i+escapedBytes:] } - return false, false, len(data) + if len(unescapedData) > 0 { + r.token.byteValue = append(unescapedData, data...) + r.token.byteValueCloned = true + } + return } // getu4 decodes \uXXXX from the beginning of s, returning the hex value, @@ -286,36 +315,30 @@ func getu4(s []byte) rune { return val } -// processEscape processes a single escape sequence and returns number of bytes processed. -func (r *Lexer) processEscape(data []byte) (int, error) { +// decodeEscape processes a single escape sequence and returns number of bytes processed. +func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) { if len(data) < 2 { - return 0, fmt.Errorf("syntax error at %v", string(data)) + return 0, 0, fmt.Errorf("syntax error at %v", string(data)) } c := data[1] switch c { case '"', '/', '\\': - r.token.byteValue = append(r.token.byteValue, c) - return 2, nil + return rune(c), 2, nil case 'b': - r.token.byteValue = append(r.token.byteValue, '\b') - return 2, nil + return '\b', 2, nil case 'f': - r.token.byteValue = append(r.token.byteValue, '\f') - return 2, nil + return '\f', 2, nil case 'n': - r.token.byteValue = append(r.token.byteValue, '\n') - return 2, nil + return '\n', 2, nil case 'r': - r.token.byteValue = append(r.token.byteValue, '\r') - return 2, nil + return '\r', 2, nil case 't': - r.token.byteValue = append(r.token.byteValue, '\t') - return 2, nil + return '\t', 2, nil case 'u': rr := getu4(data) if rr < 0 { - return 0, errors.New("syntax error") + return 0, 0, errors.New("syntax error") } read := 6 @@ -328,13 +351,10 @@ func (r *Lexer) processEscape(data []byte) (int, error) { rr = unicode.ReplacementChar } } - var d [4]byte - s := utf8.EncodeRune(d[:], rr) - r.token.byteValue = append(r.token.byteValue, d[:s]...) - return read, nil + return rr, read, nil } - return 0, errors.New("syntax error") + return 0, 0, errors.New("syntax error") } // fetchString scans a string literal token. @@ -342,43 +362,14 @@ func (r *Lexer) fetchString() { r.pos++ data := r.Data[r.pos:] - isValid, hasEscapes, length := findStringLen(data) + isValid, length := findStringLen(data) if !isValid { r.pos += length r.errParse("unterminated string literal") return } - if !hasEscapes { - r.token.byteValue = data[:length] - r.pos += length + 1 - return - } - - r.token.byteValue = make([]byte, 0, length) - p := 0 - for i := 0; i < len(data); { - switch data[i] { - case '"': - r.pos += i + 1 - r.token.byteValue = append(r.token.byteValue, data[p:i]...) - i++ - return - - case '\\': - r.token.byteValue = append(r.token.byteValue, data[p:i]...) - off, err := r.processEscape(data[i:]) - if err != nil { - r.errParse(err.Error()) - return - } - i += off - p = i - - default: - i++ - } - } - r.errParse("unterminated string literal") + r.token.byteValue = data[:length] + r.pos += length + 1 // skip closing '"' as well } // scanToken scans the next token if no token is currently available in the lexer. @@ -610,6 +601,11 @@ func (r *Lexer) unsafeString() (string, []byte) { r.errInvalidToken("string") return "", nil } + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return "", nil + } + bytes := r.token.byteValue ret := bytesToStr(r.token.byteValue) r.consume() @@ -640,7 +636,16 @@ func (r *Lexer) String() string { r.errInvalidToken("string") return "" } - ret := string(r.token.byteValue) + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return "" + } + var ret string + if r.token.byteValueCloned { + ret = bytesToStr(r.token.byteValue) + } else { + ret = string(r.token.byteValue) + } r.consume() return ret } diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index fcf9780..d3ab65b 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -33,7 +33,7 @@ func TestString(t *testing.T) { got := l.String() if got != test.want { - t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want) + t.Errorf("[%d, %q] String() = '%v'; want '%v'", i, test.toParse, got, test.want) } err := l.Error() if err != nil && !test.wantError { @@ -262,12 +262,12 @@ func TestJsonNumber(t *testing.T) { {toParse: `10`, want: json.Number("10"), wantValue: int64(10)}, {toParse: `0`, want: json.Number("0"), wantValue: int64(0)}, {toParse: `0.12`, want: json.Number("0.12"), wantValue: 0.12}, - {toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25E-4}, + {toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25e-4}, {toParse: `"10"`, want: json.Number("10"), wantValue: int64(10)}, {toParse: `"0"`, want: json.Number("0"), wantValue: int64(0)}, {toParse: `"0.12"`, want: json.Number("0.12"), wantValue: 0.12}, - {toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25E-4}, + {toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25e-4}, {toParse: `"foo"`, want: json.Number("foo"), wantValueError: true}, {toParse: `null`, want: json.Number(""), wantValueError: true}, @@ -324,7 +324,7 @@ func TestFetchStringUnterminatedString(t *testing.T) { l := Lexer{Data: test.data} l.fetchString() if l.pos > len(l.Data) { - t.Errorf("fetchString(%s): pos should not be greater than length of Data", test.data) + t.Errorf("fetchString(%s): pos=%v should not be greater than length of Data = %v", test.data, l.pos, len(l.Data)) } if l.Error() == nil { t.Errorf("fetchString(%s): should add parsing error", test.data)