From efc4b46d03d0d8e1c895bf8f712d6265dfc4d315 Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Mon, 3 Feb 2020 16:52:55 +0300 Subject: [PATCH] Optimisations: use bytes.IndexByte() + delay unescaping There are 2 issues with current implementation: 1. It performs a plain byte to byte loop to find string boundaries and perform unescaping. Replace this with bytes.IndexByte() implementation. 2. It performs unescaping of string values even when this is not really needed, e.g. for members which are absent in target data structure. This patch fixes both issues and results in ~12% faster BenchmarkEJ_Unmarshal_M-8, plus number of allocations goes down from 128 to 52 (!): benchmark old MB/s new MB/s speedup BenchmarkEJ_Unmarshal_M-8 317.99 356.27 1.12x BenchmarkEJ_Unmarshal_S-8 142.19 139.77 0.98x benchmark old allocs new allocs delta BenchmarkEJ_Unmarshal_M-8 128 52 -59.38% BenchmarkEJ_Unmarshal_S-8 3 3 +0.00% The rest of benchmarks are w/o changes. NOTE: performance can be improved up to 1.24x if unescaping is not performed for member names. Signed-off-by: Kirill Korotaev --- jlexer/lexer.go | 147 ++++++++++++++++++++++--------------------- jlexer/lexer_test.go | 8 +-- 2 files changed, 80 insertions(+), 75 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index ddd376b..74b5bda 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -5,6 +5,7 @@ package jlexer import ( + "bytes" "encoding/base64" "encoding/json" "errors" @@ -32,9 +33,10 @@ const ( type token struct { kind tokenKind // Type of a token. - boolValue bool // Value if a boolean literal token. - byteValue []byte // Raw value of a token. - delimValue byte + boolValue bool // Value if a boolean literal token. + byteValueCloned bool // true if byteValue was allocated and does not refer to original json body + byteValue []byte // Raw value of a token. + delimValue byte } // Lexer is a JSON lexer: it iterates over JSON tokens in a byte slice. @@ -240,23 +242,50 @@ func (r *Lexer) fetchNumber() { // findStringLen tries to scan into the string literal for ending quote char to determine required size. // The size will be exact if no escapes are present and may be inexact if there are escaped chars. -func findStringLen(data []byte) (isValid, hasEscapes bool, length int) { - delta := 0 - - for i := 0; i < len(data); i++ { - switch data[i] { - case '\\': - i++ - delta++ - if i < len(data) && data[i] == 'u' { - delta++ - } - case '"': - return true, (delta > 0), (i - delta) +func findStringLen(data []byte) (isValid bool, length int) { + for { + idx := bytes.IndexByte(data, '"') + if idx == -1 { + return false, len(data) } + if idx == 0 || (idx > 0 && data[idx-1] != '\\') { + return true, length + idx + } + length += idx + 1 + data = data[idx+1:] + } +} + +// unescapeStringToken performs unescaping of string token. +// if no escaping is needed, original string is returned, otherwise - a new one allocated +func (r *Lexer) unescapeStringToken() (err error) { + data := r.token.byteValue + var unescapedData []byte + + for { + i := bytes.IndexByte(data, '\\') + if i == -1 { + break + } + + escapedRune, escapedBytes, err := decodeEscape(data[i:]) + if err != nil { + r.errParse(err.Error()) + return err + } + var d [4]byte + s := utf8.EncodeRune(d[:], escapedRune) + unescapedData = append(unescapedData, data[:i]...) + unescapedData = append(unescapedData, d[:s]...) + + data = data[i+escapedBytes:] } - return false, false, len(data) + if len(unescapedData) > 0 { + r.token.byteValue = append(unescapedData, data...) + r.token.byteValueCloned = true + } + return } // getu4 decodes \uXXXX from the beginning of s, returning the hex value, @@ -286,36 +315,30 @@ func getu4(s []byte) rune { return val } -// processEscape processes a single escape sequence and returns number of bytes processed. -func (r *Lexer) processEscape(data []byte) (int, error) { +// decodeEscape processes a single escape sequence and returns number of bytes processed. +func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) { if len(data) < 2 { - return 0, fmt.Errorf("syntax error at %v", string(data)) + return 0, 0, fmt.Errorf("syntax error at %v", string(data)) } c := data[1] switch c { case '"', '/', '\\': - r.token.byteValue = append(r.token.byteValue, c) - return 2, nil + return rune(c), 2, nil case 'b': - r.token.byteValue = append(r.token.byteValue, '\b') - return 2, nil + return '\b', 2, nil case 'f': - r.token.byteValue = append(r.token.byteValue, '\f') - return 2, nil + return '\f', 2, nil case 'n': - r.token.byteValue = append(r.token.byteValue, '\n') - return 2, nil + return '\n', 2, nil case 'r': - r.token.byteValue = append(r.token.byteValue, '\r') - return 2, nil + return '\r', 2, nil case 't': - r.token.byteValue = append(r.token.byteValue, '\t') - return 2, nil + return '\t', 2, nil case 'u': rr := getu4(data) if rr < 0 { - return 0, errors.New("syntax error") + return 0, 0, errors.New("syntax error") } read := 6 @@ -328,13 +351,10 @@ func (r *Lexer) processEscape(data []byte) (int, error) { rr = unicode.ReplacementChar } } - var d [4]byte - s := utf8.EncodeRune(d[:], rr) - r.token.byteValue = append(r.token.byteValue, d[:s]...) - return read, nil + return rr, read, nil } - return 0, errors.New("syntax error") + return 0, 0, errors.New("syntax error") } // fetchString scans a string literal token. @@ -342,43 +362,14 @@ func (r *Lexer) fetchString() { r.pos++ data := r.Data[r.pos:] - isValid, hasEscapes, length := findStringLen(data) + isValid, length := findStringLen(data) if !isValid { r.pos += length r.errParse("unterminated string literal") return } - if !hasEscapes { - r.token.byteValue = data[:length] - r.pos += length + 1 - return - } - - r.token.byteValue = make([]byte, 0, length) - p := 0 - for i := 0; i < len(data); { - switch data[i] { - case '"': - r.pos += i + 1 - r.token.byteValue = append(r.token.byteValue, data[p:i]...) - i++ - return - - case '\\': - r.token.byteValue = append(r.token.byteValue, data[p:i]...) - off, err := r.processEscape(data[i:]) - if err != nil { - r.errParse(err.Error()) - return - } - i += off - p = i - - default: - i++ - } - } - r.errParse("unterminated string literal") + r.token.byteValue = data[:length] + r.pos += length + 1 // skip closing '"' as well } // scanToken scans the next token if no token is currently available in the lexer. @@ -610,6 +601,11 @@ func (r *Lexer) unsafeString() (string, []byte) { r.errInvalidToken("string") return "", nil } + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return "", nil + } + bytes := r.token.byteValue ret := bytesToStr(r.token.byteValue) r.consume() @@ -640,7 +636,16 @@ func (r *Lexer) String() string { r.errInvalidToken("string") return "" } - ret := string(r.token.byteValue) + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return "" + } + var ret string + if r.token.byteValueCloned { + ret = bytesToStr(r.token.byteValue) + } else { + ret = string(r.token.byteValue) + } r.consume() return ret } diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index fcf9780..d3ab65b 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -33,7 +33,7 @@ func TestString(t *testing.T) { got := l.String() if got != test.want { - t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want) + t.Errorf("[%d, %q] String() = '%v'; want '%v'", i, test.toParse, got, test.want) } err := l.Error() if err != nil && !test.wantError { @@ -262,12 +262,12 @@ func TestJsonNumber(t *testing.T) { {toParse: `10`, want: json.Number("10"), wantValue: int64(10)}, {toParse: `0`, want: json.Number("0"), wantValue: int64(0)}, {toParse: `0.12`, want: json.Number("0.12"), wantValue: 0.12}, - {toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25E-4}, + {toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25e-4}, {toParse: `"10"`, want: json.Number("10"), wantValue: int64(10)}, {toParse: `"0"`, want: json.Number("0"), wantValue: int64(0)}, {toParse: `"0.12"`, want: json.Number("0.12"), wantValue: 0.12}, - {toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25E-4}, + {toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25e-4}, {toParse: `"foo"`, want: json.Number("foo"), wantValueError: true}, {toParse: `null`, want: json.Number(""), wantValueError: true}, @@ -324,7 +324,7 @@ func TestFetchStringUnterminatedString(t *testing.T) { l := Lexer{Data: test.data} l.fetchString() if l.pos > len(l.Data) { - t.Errorf("fetchString(%s): pos should not be greater than length of Data", test.data) + t.Errorf("fetchString(%s): pos=%v should not be greater than length of Data = %v", test.data, l.pos, len(l.Data)) } if l.Error() == nil { t.Errorf("fetchString(%s): should add parsing error", test.data)