diff --git a/Makefile b/Makefile index 57427dd..f1c386b 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ generate: build ./tests/type_declaration.go \ ./tests/members_escaped.go \ ./tests/members_unescaped.go \ + ./tests/intern.go bin/easyjson -all ./tests/data.go bin/easyjson -all ./tests/nothing.go @@ -42,6 +43,7 @@ generate: build bin/easyjson ./tests/type_declaration.go bin/easyjson ./tests/members_escaped.go bin/easyjson -disable_members_unescape ./tests/members_unescaped.go + bin/easyjson ./tests/intern.go test: generate go test \ diff --git a/README.md b/README.md index 92d9f1f..91494c9 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,27 @@ through a call to `buffer.Init()` prior to any marshaling or unmarshaling. Please see the [GoDoc listing](https://godoc.org/github.com/mailru/easyjson/buffer) for more information. +## String interning + +During unmarshaling, `string` field values can be optionally +[interned](https://en.wikipedia.org/wiki/String_interning) to reduce memory +allocations and usage by deduplicating strings in memory, at the expense of slightly +increased CPU usage. + +This will work effectively only for `string` fields being decoded that have frequently +the same value (e.g. if you have a string field that can only assume a small number +of possible values). + +To enable string interning, add the `intern` keyword tag to your `json` tag on `string` +fields, e.g.: + +```go +type Foo struct { + UUID string `json:"uuid"` // will not be interned during unmarshaling + State string `json:"state,intern"` // will be interned during unmarshaling +} +``` + ## Issues, Notes, and Limitations * easyjson is still early in its development. As such, there are likely to be diff --git a/benchmark/go.sum b/benchmark/go.sum index 24d8af5..f72d630 100644 --- a/benchmark/go.sum +++ b/benchmark/go.sum @@ -2,6 +2,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= diff --git a/gen/decoder.go b/gen/decoder.go index cb6ddd4..f5d5305 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -112,9 +112,15 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+out+" = "+dec) return nil } else if dec := primitiveStringDecoders[t.Kind()]; dec != "" && tags.asString { + if tags.intern && t.Kind() == reflect.String { + dec = "in.StringIntern()" + } fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")") return nil } else if dec := primitiveDecoders[t.Kind()]; dec != "" { + if tags.intern && t.Kind() == reflect.String { + dec = "in.StringIntern()" + } fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")") return nil } @@ -385,7 +391,6 @@ func getStructFields(t reflect.Type) ([]reflect.StructField, error) { t1 = t1.Elem() } - if t1.Kind() == reflect.Struct { fs, err := getStructFields(t1) if err != nil { @@ -399,7 +404,6 @@ func getStructFields(t reflect.Type) ([]reflect.StructField, error) { } } - for i := 0; i < t.NumField(); i++ { f := t.Field(i) tags := parseFieldTags(f) diff --git a/gen/encoder.go b/gen/encoder.go index c31f11b..62fd4d3 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -58,6 +58,7 @@ type fieldTags struct { noOmitEmpty bool asString bool required bool + intern bool } // parseFieldTags parses the json field tag into a structure. @@ -78,6 +79,8 @@ func parseFieldTags(f reflect.StructField) fieldTags { ret.asString = true case s == "required": ret.required = true + case s == "intern": + ret.intern = true } } diff --git a/go.mod b/go.mod index 7bc4a65..f4945d3 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/mailru/easyjson go 1.12 + +require github.com/josharian/intern v1.0.0 diff --git a/jlexer/lexer.go b/jlexer/lexer.go index d6a7f4d..153d4ed 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -15,6 +15,8 @@ import ( "unicode" "unicode/utf16" "unicode/utf8" + + "github.com/josharian/intern" ) // tokenKind determines type of a token. @@ -663,6 +665,20 @@ func (r *Lexer) String() string { return ret } +// StringIntern reads a string literal, and performs string interning on it. +func (r *Lexer) StringIntern() string { + if r.token.kind == tokenUndef && r.Ok() { + r.FetchToken() + } + if !r.Ok() || r.token.kind != tokenString { + r.errInvalidToken("string") + return "" + } + ret := intern.Bytes(r.token.byteValue) + r.consume() + return ret +} + // Bytes reads a string literal and base64 decodes it into a byte slice. func (r *Lexer) Bytes() []byte { if r.token.kind == tokenUndef && r.Ok() { diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index d3ab65b..981caa5 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -29,21 +29,58 @@ func TestString(t *testing.T) { {toParse: `"\x"`, wantError: true}, // invalid escape {toParse: `"\ud800"`, want: "�"}, // invalid utf-8 char; return replacement char } { - l := Lexer{Data: []byte(test.toParse)} + { + l := Lexer{Data: []byte(test.toParse)} - got := l.String() - if got != test.want { - t.Errorf("[%d, %q] String() = '%v'; want '%v'", i, test.toParse, got, test.want) + got := l.String() + if got != test.want { + t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want) + } + err := l.Error() + if err != nil && !test.wantError { + t.Errorf("[%d, %q] String() error: %v", i, test.toParse, err) + } else if err == nil && test.wantError { + t.Errorf("[%d, %q] String() ok; want error", i, test.toParse) + } } - err := l.Error() - if err != nil && !test.wantError { - t.Errorf("[%d, %q] String() error: %v", i, test.toParse, err) - } else if err == nil && test.wantError { - t.Errorf("[%d, %q] String() ok; want error", i, test.toParse) + { + l := Lexer{Data: []byte(test.toParse)} + + got := l.StringIntern() + if got != test.want { + t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want) + } + err := l.Error() + if err != nil && !test.wantError { + t.Errorf("[%d, %q] String() error: %v", i, test.toParse, err) + } else if err == nil && test.wantError { + t.Errorf("[%d, %q] String() ok; want error", i, test.toParse) + } } } } +func TestStringIntern(t *testing.T) { + data := []byte(`"string interning test"`) + var l Lexer + + allocsPerRun := testing.AllocsPerRun(1000, func() { + l = Lexer{Data: data} + _ = l.StringIntern() + }) + if allocsPerRun != 0 { + t.Fatalf("expected 0 allocs, got %f", allocsPerRun) + } + + allocsPerRun = testing.AllocsPerRun(1000, func() { + l = Lexer{Data: data} + _ = l.String() + }) + if allocsPerRun != 1 { + t.Fatalf("expected 1 allocs, got %f", allocsPerRun) + } +} + func TestBytes(t *testing.T) { for i, test := range []struct { toParse string diff --git a/tests/basic_test.go b/tests/basic_test.go index 7fee7cc..edf9bf2 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -57,6 +57,7 @@ var testCases = []struct { {&myGenDeclaredValue, myGenDeclaredString}, {&myGenDeclaredWithCommentValue, myGenDeclaredWithCommentString}, {&myTypeDeclaredValue, myTypeDeclaredString}, + {&intern, internString}, } func TestMarshal(t *testing.T) { diff --git a/tests/intern.go b/tests/intern.go new file mode 100644 index 0000000..3bb6925 --- /dev/null +++ b/tests/intern.go @@ -0,0 +1,14 @@ +package tests + +//easyjson:json +type NoIntern struct { + Field string `json:"field"` +} + +//easyjson:json +type Intern struct { + Field string `json:"field,intern"` +} + +var intern = Intern{Field: "interned"} +var internString = `{"field":"interned"}` diff --git a/tests/intern_test.go b/tests/intern_test.go new file mode 100644 index 0000000..f61b400 --- /dev/null +++ b/tests/intern_test.go @@ -0,0 +1,35 @@ +package tests + +import ( + "testing" + + "github.com/mailru/easyjson" +) + +func TestStringIntern(t *testing.T) { + data := []byte(`{"field": "string interning test"}`) + + var i Intern + allocsPerRun := testing.AllocsPerRun(1000, func() { + i = Intern{} + easyjson.Unmarshal(data, &i) + if i.Field != "string interning test" { + t.Fatalf("wrong value: %q", i.Field) + } + }) + if allocsPerRun != 1 { + t.Fatalf("expected 1 allocs, got %f", allocsPerRun) + } + + var n NoIntern + allocsPerRun = testing.AllocsPerRun(1000, func() { + n = NoIntern{} + easyjson.Unmarshal(data, &n) + if n.Field != "string interning test" { + t.Fatalf("wrong value: %q", n.Field) + } + }) + if allocsPerRun != 2 { + t.Fatalf("expected 2 allocs, got %f", allocsPerRun) + } +}