From efc4b46d03d0d8e1c895bf8f712d6265dfc4d315 Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Mon, 3 Feb 2020 16:52:55 +0300 Subject: [PATCH 1/7] Optimisations: use bytes.IndexByte() + delay unescaping There are 2 issues with current implementation: 1. It performs a plain byte to byte loop to find string boundaries and perform unescaping. Replace this with bytes.IndexByte() implementation. 2. It performs unescaping of string values even when this is not really needed, e.g. for members which are absent in target data structure. This patch fixes both issues and results in ~12% faster BenchmarkEJ_Unmarshal_M-8, plus number of allocations goes down from 128 to 52 (!): benchmark old MB/s new MB/s speedup BenchmarkEJ_Unmarshal_M-8 317.99 356.27 1.12x BenchmarkEJ_Unmarshal_S-8 142.19 139.77 0.98x benchmark old allocs new allocs delta BenchmarkEJ_Unmarshal_M-8 128 52 -59.38% BenchmarkEJ_Unmarshal_S-8 3 3 +0.00% The rest of benchmarks are w/o changes. NOTE: performance can be improved up to 1.24x if unescaping is not performed for member names. Signed-off-by: Kirill Korotaev --- jlexer/lexer.go | 147 ++++++++++++++++++++++--------------------- jlexer/lexer_test.go | 8 +-- 2 files changed, 80 insertions(+), 75 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index ddd376b..74b5bda 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -5,6 +5,7 @@ package jlexer import ( + "bytes" "encoding/base64" "encoding/json" "errors" @@ -32,9 +33,10 @@ const ( type token struct { kind tokenKind // Type of a token. - boolValue bool // Value if a boolean literal token. - byteValue []byte // Raw value of a token. - delimValue byte + boolValue bool // Value if a boolean literal token. + byteValueCloned bool // true if byteValue was allocated and does not refer to original json body + byteValue []byte // Raw value of a token. + delimValue byte } // Lexer is a JSON lexer: it iterates over JSON tokens in a byte slice. @@ -240,23 +242,50 @@ func (r *Lexer) fetchNumber() { // findStringLen tries to scan into the string literal for ending quote char to determine required size. // The size will be exact if no escapes are present and may be inexact if there are escaped chars. -func findStringLen(data []byte) (isValid, hasEscapes bool, length int) { - delta := 0 - - for i := 0; i < len(data); i++ { - switch data[i] { - case '\\': - i++ - delta++ - if i < len(data) && data[i] == 'u' { - delta++ - } - case '"': - return true, (delta > 0), (i - delta) +func findStringLen(data []byte) (isValid bool, length int) { + for { + idx := bytes.IndexByte(data, '"') + if idx == -1 { + return false, len(data) } + if idx == 0 || (idx > 0 && data[idx-1] != '\\') { + return true, length + idx + } + length += idx + 1 + data = data[idx+1:] + } +} + +// unescapeStringToken performs unescaping of string token. +// if no escaping is needed, original string is returned, otherwise - a new one allocated +func (r *Lexer) unescapeStringToken() (err error) { + data := r.token.byteValue + var unescapedData []byte + + for { + i := bytes.IndexByte(data, '\\') + if i == -1 { + break + } + + escapedRune, escapedBytes, err := decodeEscape(data[i:]) + if err != nil { + r.errParse(err.Error()) + return err + } + var d [4]byte + s := utf8.EncodeRune(d[:], escapedRune) + unescapedData = append(unescapedData, data[:i]...) + unescapedData = append(unescapedData, d[:s]...) + + data = data[i+escapedBytes:] } - return false, false, len(data) + if len(unescapedData) > 0 { + r.token.byteValue = append(unescapedData, data...) + r.token.byteValueCloned = true + } + return } // getu4 decodes \uXXXX from the beginning of s, returning the hex value, @@ -286,36 +315,30 @@ func getu4(s []byte) rune { return val } -// processEscape processes a single escape sequence and returns number of bytes processed. -func (r *Lexer) processEscape(data []byte) (int, error) { +// decodeEscape processes a single escape sequence and returns number of bytes processed. +func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) { if len(data) < 2 { - return 0, fmt.Errorf("syntax error at %v", string(data)) + return 0, 0, fmt.Errorf("syntax error at %v", string(data)) } c := data[1] switch c { case '"', '/', '\\': - r.token.byteValue = append(r.token.byteValue, c) - return 2, nil + return rune(c), 2, nil case 'b': - r.token.byteValue = append(r.token.byteValue, '\b') - return 2, nil + return '\b', 2, nil case 'f': - r.token.byteValue = append(r.token.byteValue, '\f') - return 2, nil + return '\f', 2, nil case 'n': - r.token.byteValue = append(r.token.byteValue, '\n') - return 2, nil + return '\n', 2, nil case 'r': - r.token.byteValue = append(r.token.byteValue, '\r') - return 2, nil + return '\r', 2, nil case 't': - r.token.byteValue = append(r.token.byteValue, '\t') - return 2, nil + return '\t', 2, nil case 'u': rr := getu4(data) if rr < 0 { - return 0, errors.New("syntax error") + return 0, 0, errors.New("syntax error") } read := 6 @@ -328,13 +351,10 @@ func (r *Lexer) processEscape(data []byte) (int, error) { rr = unicode.ReplacementChar } } - var d [4]byte - s := utf8.EncodeRune(d[:], rr) - r.token.byteValue = append(r.token.byteValue, d[:s]...) - return read, nil + return rr, read, nil } - return 0, errors.New("syntax error") + return 0, 0, errors.New("syntax error") } // fetchString scans a string literal token. @@ -342,43 +362,14 @@ func (r *Lexer) fetchString() { r.pos++ data := r.Data[r.pos:] - isValid, hasEscapes, length := findStringLen(data) + isValid, length := findStringLen(data) if !isValid { r.pos += length r.errParse("unterminated string literal") return } - if !hasEscapes { - r.token.byteValue = data[:length] - r.pos += length + 1 - return - } - - r.token.byteValue = make([]byte, 0, length) - p := 0 - for i := 0; i < len(data); { - switch data[i] { - case '"': - r.pos += i + 1 - r.token.byteValue = append(r.token.byteValue, data[p:i]...) - i++ - return - - case '\\': - r.token.byteValue = append(r.token.byteValue, data[p:i]...) - off, err := r.processEscape(data[i:]) - if err != nil { - r.errParse(err.Error()) - return - } - i += off - p = i - - default: - i++ - } - } - r.errParse("unterminated string literal") + r.token.byteValue = data[:length] + r.pos += length + 1 // skip closing '"' as well } // scanToken scans the next token if no token is currently available in the lexer. @@ -610,6 +601,11 @@ func (r *Lexer) unsafeString() (string, []byte) { r.errInvalidToken("string") return "", nil } + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return "", nil + } + bytes := r.token.byteValue ret := bytesToStr(r.token.byteValue) r.consume() @@ -640,7 +636,16 @@ func (r *Lexer) String() string { r.errInvalidToken("string") return "" } - ret := string(r.token.byteValue) + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return "" + } + var ret string + if r.token.byteValueCloned { + ret = bytesToStr(r.token.byteValue) + } else { + ret = string(r.token.byteValue) + } r.consume() return ret } diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index fcf9780..d3ab65b 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -33,7 +33,7 @@ func TestString(t *testing.T) { got := l.String() if got != test.want { - t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want) + t.Errorf("[%d, %q] String() = '%v'; want '%v'", i, test.toParse, got, test.want) } err := l.Error() if err != nil && !test.wantError { @@ -262,12 +262,12 @@ func TestJsonNumber(t *testing.T) { {toParse: `10`, want: json.Number("10"), wantValue: int64(10)}, {toParse: `0`, want: json.Number("0"), wantValue: int64(0)}, {toParse: `0.12`, want: json.Number("0.12"), wantValue: 0.12}, - {toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25E-4}, + {toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25e-4}, {toParse: `"10"`, want: json.Number("10"), wantValue: int64(10)}, {toParse: `"0"`, want: json.Number("0"), wantValue: int64(0)}, {toParse: `"0.12"`, want: json.Number("0.12"), wantValue: 0.12}, - {toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25E-4}, + {toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25e-4}, {toParse: `"foo"`, want: json.Number("foo"), wantValueError: true}, {toParse: `null`, want: json.Number(""), wantValueError: true}, @@ -324,7 +324,7 @@ func TestFetchStringUnterminatedString(t *testing.T) { l := Lexer{Data: test.data} l.fetchString() if l.pos > len(l.Data) { - t.Errorf("fetchString(%s): pos should not be greater than length of Data", test.data) + t.Errorf("fetchString(%s): pos=%v should not be greater than length of Data = %v", test.data, l.pos, len(l.Data)) } if l.Error() == nil { t.Errorf("fetchString(%s): should add parsing error", test.data) From 36293b60e3953d04475bfe31a7c5161013fc23a7 Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Mon, 3 Feb 2020 23:18:02 +0300 Subject: [PATCH 2/7] Preallocate string before unescaping This reduces number of allocations in BenchmarkEJ_Unmarshal_M-8 test from 52 down to 46. Signed-off-by: Kirill Korotaev --- jlexer/lexer.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 74b5bda..62fbcf9 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -260,6 +260,7 @@ func findStringLen(data []byte) (isValid bool, length int) { // if no escaping is needed, original string is returned, otherwise - a new one allocated func (r *Lexer) unescapeStringToken() (err error) { data := r.token.byteValue + wasEscaped := false var unescapedData []byte for { @@ -273,6 +274,12 @@ func (r *Lexer) unescapeStringToken() (err error) { r.errParse(err.Error()) return err } + + if !wasEscaped { + unescapedData = make([]byte, 0, len(r.token.byteValue)) + wasEscaped = true + } + var d [4]byte s := utf8.EncodeRune(d[:], escapedRune) unescapedData = append(unescapedData, data[:i]...) @@ -281,7 +288,7 @@ func (r *Lexer) unescapeStringToken() (err error) { data = data[i+escapedBytes:] } - if len(unescapedData) > 0 { + if wasEscaped { r.token.byteValue = append(unescapedData, data...) r.token.byteValueCloned = true } From 3a0ce97e28d4a0c3f3adb53c486130b17794b145 Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Tue, 4 Feb 2020 18:14:50 +0300 Subject: [PATCH 3/7] slightly simplify unescapeStringToken() and remove wasEscaped flag --- jlexer/lexer.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 62fbcf9..0254862 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -260,7 +260,6 @@ func findStringLen(data []byte) (isValid bool, length int) { // if no escaping is needed, original string is returned, otherwise - a new one allocated func (r *Lexer) unescapeStringToken() (err error) { data := r.token.byteValue - wasEscaped := false var unescapedData []byte for { @@ -275,9 +274,8 @@ func (r *Lexer) unescapeStringToken() (err error) { return err } - if !wasEscaped { + if unescapedData == nil { unescapedData = make([]byte, 0, len(r.token.byteValue)) - wasEscaped = true } var d [4]byte @@ -288,7 +286,7 @@ func (r *Lexer) unescapeStringToken() (err error) { data = data[i+escapedBytes:] } - if wasEscaped { + if unescapedData != nil { r.token.byteValue = append(unescapedData, data...) r.token.byteValueCloned = true } From 9a01c9afdbb32d90050fa430a2bc235c3d2783e4 Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Tue, 4 Feb 2020 18:16:09 +0300 Subject: [PATCH 4/7] Add new -disable_members_unescape option to avoid unescaping of member names, ints, ... Improves performance for about 15% on BenchmarkEJ_Unmarshal_M-8 --- Makefile | 2 +- bootstrap/bootstrap.go | 20 ++++++++++++-------- easyjson/main.go | 30 ++++++++++++++++-------------- gen/decoder.go | 3 +++ gen/generator.go | 16 +++++++++++----- jlexer/lexer.go | 9 ++++++--- 6 files changed, 49 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index ce30363..dbe6a2b 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ generate: build bin/easyjson -all ./tests/html.go bin/easyjson -snake_case ./tests/snake.go bin/easyjson -omit_empty ./tests/omitempty.go - bin/easyjson -build_tags=use_easyjson ./benchmark/data.go + bin/easyjson -build_tags=use_easyjson -disable_members_unescape ./benchmark/data.go bin/easyjson ./tests/nested_easy.go bin/easyjson ./tests/named_type.go bin/easyjson ./tests/custom_map_key_type.go diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 9d1c2e3..7e984d9 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -23,18 +23,19 @@ type Generator struct { PkgPath, PkgName string Types []string - NoStdMarshalers bool - SnakeCase bool - LowerCamelCase bool - OmitEmpty bool - DisallowUnknownFields bool + NoStdMarshalers bool + SnakeCase bool + LowerCamelCase bool + OmitEmpty bool + DisallowUnknownFields bool + SkipMemberNameUnescaping bool OutName string BuildTags string - StubsOnly bool - LeaveTemps bool - NoFormat bool + StubsOnly bool + LeaveTemps bool + NoFormat bool SimpleBytes bool } @@ -129,6 +130,9 @@ func (g *Generator) writeMain() (path string, err error) { if g.SimpleBytes { fmt.Fprintln(f, " g.SimpleBytes()") } + if g.SkipMemberNameUnescaping { + fmt.Fprintln(f, " g.SkipMemberNameUnescaping()") + } sort.Strings(g.Types) for _, v := range g.Types { diff --git a/easyjson/main.go b/easyjson/main.go index 26e95cb..7e935fb 100644 --- a/easyjson/main.go +++ b/easyjson/main.go @@ -29,6 +29,7 @@ var noformat = flag.Bool("noformat", false, "do not run 'gofmt -w' on output fil var specifiedName = flag.String("output_filename", "", "specify the filename of the output") var processPkg = flag.Bool("pkg", false, "process the whole package instead of just the given file") var disallowUnknownFields = flag.Bool("disallow_unknown_fields", false, "return error if any unknown field in json appeared") +var skipMemberNameUnescaping = flag.Bool("disable_members_unescape", false, "don't perform unescaping of member names to improve performance") func generate(fname string) (err error) { fInfo, err := os.Stat(fname) @@ -62,20 +63,21 @@ func generate(fname string) (err error) { } g := bootstrap.Generator{ - BuildTags: trimmedBuildTags, - PkgPath: p.PkgPath, - PkgName: p.PkgName, - Types: p.StructNames, - SnakeCase: *snakeCase, - LowerCamelCase: *lowerCamelCase, - NoStdMarshalers: *noStdMarshalers, - DisallowUnknownFields: *disallowUnknownFields, - OmitEmpty: *omitEmpty, - LeaveTemps: *leaveTemps, - OutName: outName, - StubsOnly: *stubs, - NoFormat: *noformat, - SimpleBytes: *simpleBytes, + BuildTags: trimmedBuildTags, + PkgPath: p.PkgPath, + PkgName: p.PkgName, + Types: p.StructNames, + SnakeCase: *snakeCase, + LowerCamelCase: *lowerCamelCase, + NoStdMarshalers: *noStdMarshalers, + DisallowUnknownFields: *disallowUnknownFields, + SkipMemberNameUnescaping: *skipMemberNameUnescaping, + OmitEmpty: *omitEmpty, + LeaveTemps: *leaveTemps, + OutName: outName, + StubsOnly: *stubs, + NoFormat: *noformat, + SimpleBytes: *simpleBytes, } if err := g.Run(); err != nil { diff --git a/gen/decoder.go b/gen/decoder.go index 2a17cc7..5fa6179 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -457,6 +457,9 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { typ := g.getType(t) fmt.Fprintln(g.out, "func "+fname+"(in *jlexer.Lexer, out *"+typ+") {") + if g.skipMemberNameUnescaping { + fmt.Fprintln(g.out, " in.SkipUnescape = true") + } fmt.Fprintln(g.out, " isTopLevel := in.IsStart()") fmt.Fprintln(g.out, " if in.IsNull() {") fmt.Fprintln(g.out, " if isTopLevel {") diff --git a/gen/generator.go b/gen/generator.go index 344e516..79f4d6f 100644 --- a/gen/generator.go +++ b/gen/generator.go @@ -33,11 +33,12 @@ type Generator struct { varCounter int - noStdMarshalers bool - omitEmpty bool - disallowUnknownFields bool - fieldNamer FieldNamer - simpleBytes bool + noStdMarshalers bool + omitEmpty bool + disallowUnknownFields bool + fieldNamer FieldNamer + simpleBytes bool + skipMemberNameUnescaping bool // package path to local alias map for tracking imports imports map[string]string @@ -117,6 +118,11 @@ func (g *Generator) DisallowUnknownFields() { g.disallowUnknownFields = true } +// SkipMemberNameUnescaping instructs to skip member names unescaping to improve performance +func (g *Generator) SkipMemberNameUnescaping() { + g.skipMemberNameUnescaping = true +} + // OmitEmpty triggers `json=",omitempty"` behaviour by default. func (g *Generator) OmitEmpty() { g.omitEmpty = true diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 0254862..53ff2a8 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -49,6 +49,7 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. + SkipUnescape bool // Skip unescaping on all but user returned strings (i.e. member names, numbers, bytes etc.) UseMultipleErrors bool // If we want to use multiple errors. fatalError error // Fatal error occurred during lexing. It is usually a syntax error. @@ -606,9 +607,11 @@ func (r *Lexer) unsafeString() (string, []byte) { r.errInvalidToken("string") return "", nil } - if err := r.unescapeStringToken(); err != nil { - r.errInvalidToken("string") - return "", nil + if !r.SkipUnescape { + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return "", nil + } } bytes := r.token.byteValue From e0f67b396dd4995ba0515b5718863c9e3c2a880f Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Sat, 11 Apr 2020 19:04:13 +0300 Subject: [PATCH 5/7] Add test cases for non-escaped mode of member names --- Makefile | 6 +++- README.md | 10 +++++++ tests/members_escaped.go | 6 ++++ tests/members_escaping_test.go | 51 ++++++++++++++++++++++++++++++++++ tests/members_unescaped.go | 6 ++++ 5 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 tests/members_escaped.go create mode 100644 tests/members_escaping_test.go create mode 100644 tests/members_unescaped.go diff --git a/Makefile b/Makefile index dbe6a2b..57427dd 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,9 @@ generate: build ./tests/reference_to_pointer.go \ ./tests/html.go \ ./tests/unknown_fields.go \ - ./tests/type_declaration.go + ./tests/type_declaration.go \ + ./tests/members_escaped.go \ + ./tests/members_unescaped.go \ bin/easyjson -all ./tests/data.go bin/easyjson -all ./tests/nothing.go @@ -38,6 +40,8 @@ generate: build bin/easyjson -disallow_unknown_fields ./tests/disallow_unknown.go bin/easyjson ./tests/unknown_fields.go bin/easyjson ./tests/type_declaration.go + bin/easyjson ./tests/members_escaped.go + bin/easyjson -disable_members_unescape ./tests/members_unescaped.go test: generate go test \ diff --git a/README.md b/README.md index c88848d..d5d44bf 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ Usage of easyjson: only generate stubs for marshaler/unmarshaler funcs -disallow_unknown_fields return error if some unknown field in json appeared + -disable_members_unescape + disable unescaping of \uXXXX string sequences in member names ``` Using `-all` will generate marshalers/unmarshalers for all Go structs in the @@ -78,6 +80,14 @@ Additional option notes: * `-build_tags` will add the specified build tags to generated Go sources. +## Structure json tag options + +Besides standart json tag options like 'omitempty' the following are supported: + +* 'noclone' - disables allocation and copying of string values, making them + refer to original json buffer memory. This works great for short lived + objects which are not hold in memory after decoding and immediate usage. + ## Generated Marshaler/Unmarshaler Funcs For Go struct types, easyjson generates the funcs `MarshalEasyJSON` / diff --git a/tests/members_escaped.go b/tests/members_escaped.go new file mode 100644 index 0000000..0fd68e8 --- /dev/null +++ b/tests/members_escaped.go @@ -0,0 +1,6 @@ +package tests + +//easyjson:json +type MembersEscaped struct { + A string `json:"漢語"` +} diff --git a/tests/members_escaping_test.go b/tests/members_escaping_test.go new file mode 100644 index 0000000..e385648 --- /dev/null +++ b/tests/members_escaping_test.go @@ -0,0 +1,51 @@ +package tests + +import ( + "reflect" + "testing" + + "github.com/mailru/easyjson" +) + +func TestMembersEscaping(t *testing.T) { + cases := []struct { + data string + esc MembersEscaped + unesc MembersUnescaped + }{ + { + data: `{"漢語": "中国"}`, + esc: MembersEscaped{A: "中国"}, + unesc: MembersUnescaped{A: "中国"}, + }, + { + data: `{"漢語": "\u4e2D\u56fD"}`, + esc: MembersEscaped{A: "中国"}, + unesc: MembersUnescaped{A: "中国"}, + }, + { + data: `{"\u6f22\u8a9E": "中国"}`, + esc: MembersEscaped{A: "中国"}, + unesc: MembersUnescaped{A: ""}, + }, + { + data: `{"\u6f22\u8a9E": "\u4e2D\u56fD"}`, + esc: MembersEscaped{A: "中国"}, + unesc: MembersUnescaped{A: ""}, + }, + } + + for i, c := range cases { + var esc MembersEscaped + easyjson.Unmarshal([]byte(c.data), &esc) + if !reflect.DeepEqual(esc, c.esc) { + t.Errorf("[%d] TestMembersEscaping(): got=%+v, exp=%+v", i, esc, c.esc) + } + + var unesc MembersUnescaped + easyjson.Unmarshal([]byte(c.data), &unesc) + if !reflect.DeepEqual(unesc, c.unesc) { + t.Errorf("[%d] TestMembersEscaping(): no-unescaping case: got=%+v, exp=%+v", i, esc, c.esc) + } + } +} diff --git a/tests/members_unescaped.go b/tests/members_unescaped.go new file mode 100644 index 0000000..1b95721 --- /dev/null +++ b/tests/members_unescaped.go @@ -0,0 +1,6 @@ +package tests + +//easyjson:json +type MembersUnescaped struct { + A string `json:"漢語"` +} From 7a2b99443eaf7a8fae6a74bbf71d7a446af65baf Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Sat, 11 Apr 2020 19:20:33 +0300 Subject: [PATCH 6/7] remove README.md comment from another PR --- README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.md b/README.md index d5d44bf..92d9f1f 100644 --- a/README.md +++ b/README.md @@ -80,14 +80,6 @@ Additional option notes: * `-build_tags` will add the specified build tags to generated Go sources. -## Structure json tag options - -Besides standart json tag options like 'omitempty' the following are supported: - -* 'noclone' - disables allocation and copying of string values, making them - refer to original json buffer memory. This works great for short lived - objects which are not hold in memory after decoding and immediate usage. - ## Generated Marshaler/Unmarshaler Funcs For Go struct types, easyjson generates the funcs `MarshalEasyJSON` / From 33d35f870a46108cebc6ff10c4e8fa976563fd69 Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Sat, 11 Apr 2020 20:20:50 +0300 Subject: [PATCH 7/7] unescaping: should process string numbers in unescaped member names mode There was a small glitch: numbers as strings where not unescaped in -disable_members_unescape mode, though this mode was implied to affect field names only. --- gen/decoder.go | 5 +---- jlexer/lexer.go | 35 ++++++++++++++++++++--------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index 5fa6179..2127d0d 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -457,9 +457,6 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { typ := g.getType(t) fmt.Fprintln(g.out, "func "+fname+"(in *jlexer.Lexer, out *"+typ+") {") - if g.skipMemberNameUnescaping { - fmt.Fprintln(g.out, " in.SkipUnescape = true") - } fmt.Fprintln(g.out, " isTopLevel := in.IsStart()") fmt.Fprintln(g.out, " if in.IsNull() {") fmt.Fprintln(g.out, " if isTopLevel {") @@ -489,7 +486,7 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { fmt.Fprintln(g.out, " in.Delim('{')") fmt.Fprintln(g.out, " for !in.IsDelim('}') {") - fmt.Fprintln(g.out, " key := in.UnsafeString()") + fmt.Fprintf(g.out, " key := in.UnsafeFieldName(%v)\n", g.skipMemberNameUnescaping) fmt.Fprintln(g.out, " in.WantColon()") fmt.Fprintln(g.out, " if in.IsNull() {") fmt.Fprintln(g.out, " in.Skip()") diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 53ff2a8..d6a7f4d 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -49,7 +49,6 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. - SkipUnescape bool // Skip unescaping on all but user returned strings (i.e. member names, numbers, bytes etc.) UseMultipleErrors bool // If we want to use multiple errors. fatalError error // Fatal error occurred during lexing. It is usually a syntax error. @@ -599,7 +598,7 @@ func (r *Lexer) Consumed() { } } -func (r *Lexer) unsafeString() (string, []byte) { +func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) { if r.token.kind == tokenUndef && r.Ok() { r.FetchToken() } @@ -607,7 +606,7 @@ func (r *Lexer) unsafeString() (string, []byte) { r.errInvalidToken("string") return "", nil } - if !r.SkipUnescape { + if !skipUnescape { if err := r.unescapeStringToken(); err != nil { r.errInvalidToken("string") return "", nil @@ -625,13 +624,19 @@ func (r *Lexer) unsafeString() (string, []byte) { // Warning: returned string may point to the input buffer, so the string should not outlive // the input buffer. Intended pattern of usage is as an argument to a switch statement. func (r *Lexer) UnsafeString() string { - ret, _ := r.unsafeString() + ret, _ := r.unsafeString(false) return ret } // UnsafeBytes returns the byte slice if the token is a string literal. func (r *Lexer) UnsafeBytes() []byte { - _, ret := r.unsafeString() + _, ret := r.unsafeString(false) + return ret +} + +// UnsafeFieldName returns current member name string token +func (r *Lexer) UnsafeFieldName(skipUnescape bool) string { + ret, _ := r.unsafeString(skipUnescape) return ret } @@ -852,7 +857,7 @@ func (r *Lexer) Int() int { } func (r *Lexer) Uint8Str() uint8 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -869,7 +874,7 @@ func (r *Lexer) Uint8Str() uint8 { } func (r *Lexer) Uint16Str() uint16 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -886,7 +891,7 @@ func (r *Lexer) Uint16Str() uint16 { } func (r *Lexer) Uint32Str() uint32 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -903,7 +908,7 @@ func (r *Lexer) Uint32Str() uint32 { } func (r *Lexer) Uint64Str() uint64 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -928,7 +933,7 @@ func (r *Lexer) UintptrStr() uintptr { } func (r *Lexer) Int8Str() int8 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -945,7 +950,7 @@ func (r *Lexer) Int8Str() int8 { } func (r *Lexer) Int16Str() int16 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -962,7 +967,7 @@ func (r *Lexer) Int16Str() int16 { } func (r *Lexer) Int32Str() int32 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -979,7 +984,7 @@ func (r *Lexer) Int32Str() int32 { } func (r *Lexer) Int64Str() int64 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -1017,7 +1022,7 @@ func (r *Lexer) Float32() float32 { } func (r *Lexer) Float32Str() float32 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 } @@ -1050,7 +1055,7 @@ func (r *Lexer) Float64() float64 { } func (r *Lexer) Float64Str() float64 { - s, b := r.unsafeString() + s, b := r.unsafeString(false) if !r.Ok() { return 0 }