netbird/easyjson
0
0
Fork 0
mirror of https://github.com/netbirdio/easyjson.git synced 2026-05-22 18:44:42 -07:00
Code Issues Packages Projects Releases Wiki Activity

Optimisations: use bytes.IndexByte() + delay unescaping

Browse Source 
There are 2 issues with current implementation:

1. It performs a plain byte to byte loop to find string boundaries and
perform unescaping. Replace this with bytes.IndexByte() implementation.

2. It performs unescaping of string values even when this is not really
needed, e.g. for members which are absent in target data structure.

This patch fixes both issues and results in ~12% faster BenchmarkEJ_Unmarshal_M-8,
plus number of allocations goes down from 128 to 52 (!):

benchmark                                     old MB/s     new MB/s     speedup
BenchmarkEJ_Unmarshal_M-8                     317.99       356.27       1.12x
BenchmarkEJ_Unmarshal_S-8                     142.19       139.77       0.98x

benchmark                                     old allocs     new allocs     delta
BenchmarkEJ_Unmarshal_M-8                     128            52             -59.38%
BenchmarkEJ_Unmarshal_S-8                     3              3              +0.00%

The rest of benchmarks are w/o changes.

NOTE: performance can be improved up to 1.24x if unescaping is not
performed for member names.

Signed-off-by: Kirill Korotaev <kirillx@gmail.com>
This commit is contained in:
Kirill Korotaev
2020-04-11 19:08:38 +03:00
parent 57978476cf
commit efc4b46d03
2 changed files with 80 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jlexer/lexer.go
+76 -71
View File
@@ -5,6 +5,7 @@
package jlexer
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
@@ -32,9 +33,10 @@ const (
type token struct {
kind tokenKind // Type of a token.
boolValue bool // Value if a boolean literal token.
byteValue []byte // Raw value of a token.
delimValue byte
boolValue bool // Value if a boolean literal token.
byteValueCloned bool // true if byteValue was allocated and does not refer to original json body
byteValue []byte // Raw value of a token.
delimValue byte
}
// Lexer is a JSON lexer: it iterates over JSON tokens in a byte slice.
@@ -240,23 +242,50 @@ func (r *Lexer) fetchNumber() {
// findStringLen tries to scan into the string literal for ending quote char to determine required size.
// The size will be exact if no escapes are present and may be inexact if there are escaped chars.
func findStringLen(data []byte) (isValid, hasEscapes bool, length int) {
delta := 0
for i := 0; i < len(data); i++ {
switch data[i] {
case '\\':
i++
delta++
if i < len(data) && data[i] == 'u' {
delta++
}
case '"':
return true, (delta > 0), (i - delta)
func findStringLen(data []byte) (isValid bool, length int) {
for {
idx := bytes.IndexByte(data, '"')
if idx == -1 {
return false, len(data)
}
if idx == 0 || (idx > 0 && data[idx-1] != '\\') {
return true, length + idx
}
length += idx + 1
data = data[idx+1:]
}
}
// unescapeStringToken performs unescaping of string token.
// if no escaping is needed, original string is returned, otherwise - a new one allocated
func (r *Lexer) unescapeStringToken() (err error) {
data := r.token.byteValue
var unescapedData []byte
for {
i := bytes.IndexByte(data, '\\')
if i == -1 {
break
}
escapedRune, escapedBytes, err := decodeEscape(data[i:])
if err != nil {
r.errParse(err.Error())
return err
}
var d [4]byte
s := utf8.EncodeRune(d[:], escapedRune)
unescapedData = append(unescapedData, data[:i]...)
unescapedData = append(unescapedData, d[:s]...)
data = data[i+escapedBytes:]
}
return false, false, len(data)
if len(unescapedData) > 0 {
r.token.byteValue = append(unescapedData, data...)
r.token.byteValueCloned = true
}
return
}
// getu4 decodes \uXXXX from the beginning of s, returning the hex value,
@@ -286,36 +315,30 @@ func getu4(s []byte) rune {
return val
}
// processEscape processes a single escape sequence and returns number of bytes processed.
func (r *Lexer) processEscape(data []byte) (int, error) {
// decodeEscape processes a single escape sequence and returns number of bytes processed.
func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
if len(data) < 2 {
return 0, fmt.Errorf("syntax error at %v", string(data))
return 0, 0, fmt.Errorf("syntax error at %v", string(data))
}
c := data[1]
switch c {
case '"', '/', '\\':
r.token.byteValue = append(r.token.byteValue, c)
return 2, nil
return rune(c), 2, nil
case 'b':
r.token.byteValue = append(r.token.byteValue, '\b')
return 2, nil
return '\b', 2, nil
case 'f':
r.token.byteValue = append(r.token.byteValue, '\f')
return 2, nil
return '\f', 2, nil
case 'n':
r.token.byteValue = append(r.token.byteValue, '\n')
return 2, nil
return '\n', 2, nil
case 'r':
r.token.byteValue = append(r.token.byteValue, '\r')
return 2, nil
return '\r', 2, nil
case 't':
r.token.byteValue = append(r.token.byteValue, '\t')
return 2, nil
return '\t', 2, nil
case 'u':
rr := getu4(data)
if rr < 0 {
return 0, errors.New("syntax error")
return 0, 0, errors.New("syntax error")
}
read := 6
@@ -328,13 +351,10 @@ func (r *Lexer) processEscape(data []byte) (int, error) {
rr = unicode.ReplacementChar
}
}
var d [4]byte
s := utf8.EncodeRune(d[:], rr)
r.token.byteValue = append(r.token.byteValue, d[:s]...)
return read, nil
return rr, read, nil
}
return 0, errors.New("syntax error")
return 0, 0, errors.New("syntax error")
}
// fetchString scans a string literal token.
@@ -342,43 +362,14 @@ func (r *Lexer) fetchString() {
r.pos++
data := r.Data[r.pos:]
isValid, hasEscapes, length := findStringLen(data)
isValid, length := findStringLen(data)
if !isValid {
r.pos += length
r.errParse("unterminated string literal")
return
}
if !hasEscapes {
r.token.byteValue = data[:length]
r.pos += length + 1
return
}
r.token.byteValue = make([]byte, 0, length)
p := 0
for i := 0; i < len(data); {
switch data[i] {
case '"':
r.pos += i + 1
r.token.byteValue = append(r.token.byteValue, data[p:i]...)
i++
return
case '\\':
r.token.byteValue = append(r.token.byteValue, data[p:i]...)
off, err := r.processEscape(data[i:])
if err != nil {
r.errParse(err.Error())
return
}
i += off
p = i
default:
i++
}
}
r.errParse("unterminated string literal")
r.token.byteValue = data[:length]
r.pos += length + 1 // skip closing '"' as well
}
// scanToken scans the next token if no token is currently available in the lexer.
@@ -610,6 +601,11 @@ func (r *Lexer) unsafeString() (string, []byte) {
r.errInvalidToken("string")
return "", nil
}
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return "", nil
}
bytes := r.token.byteValue
ret := bytesToStr(r.token.byteValue)
r.consume()
@@ -640,7 +636,16 @@ func (r *Lexer) String() string {
r.errInvalidToken("string")
return ""
}
ret := string(r.token.byteValue)
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return ""
}
var ret string
if r.token.byteValueCloned {
ret = bytesToStr(r.token.byteValue)
} else {
ret = string(r.token.byteValue)
}
r.consume()
return ret
}
jlexer/lexer_test.go
+4 -4
View File
@@ -33,7 +33,7 @@ func TestString(t *testing.T) {
got := l.String()
if got != test.want {
t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want)
t.Errorf("[%d, %q] String() = '%v'; want '%v'", i, test.toParse, got, test.want)
}
err := l.Error()
if err != nil && !test.wantError {
@@ -262,12 +262,12 @@ func TestJsonNumber(t *testing.T) {
{toParse: `10`, want: json.Number("10"), wantValue: int64(10)},
{toParse: `0`, want: json.Number("0"), wantValue: int64(0)},
{toParse: `0.12`, want: json.Number("0.12"), wantValue: 0.12},
{toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25E-4},
{toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25e-4},
{toParse: `"10"`, want: json.Number("10"), wantValue: int64(10)},
{toParse: `"0"`, want: json.Number("0"), wantValue: int64(0)},
{toParse: `"0.12"`, want: json.Number("0.12"), wantValue: 0.12},
{toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25E-4},
{toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25e-4},
{toParse: `"foo"`, want: json.Number("foo"), wantValueError: true},
{toParse: `null`, want: json.Number(""), wantValueError: true},
@@ -324,7 +324,7 @@ func TestFetchStringUnterminatedString(t *testing.T) {
l := Lexer{Data: test.data}
l.fetchString()
if l.pos > len(l.Data) {
t.Errorf("fetchString(%s): pos should not be greater than length of Data", test.data)
t.Errorf("fetchString(%s): pos=%v should not be greater than length of Data = %v", test.data, l.pos, len(l.Data))
}
if l.Error() == nil {
t.Errorf("fetchString(%s): should add parsing error", test.data)