Merge branch 'cafxx-intern' of git://github.com/CAFxX/easyjson into CAFxX-cafxx-intern

Makefile

@@ -23,6 +23,7 @@ generate: build
 		./tests/type_declaration.go \
 		./tests/members_escaped.go \
 		./tests/members_unescaped.go \
+		./tests/intern.go
 
 	bin/easyjson -all ./tests/data.go
 	bin/easyjson -all ./tests/nothing.go
@@ -42,6 +43,7 @@ generate: build
 	bin/easyjson ./tests/type_declaration.go
 	bin/easyjson ./tests/members_escaped.go
 	bin/easyjson -disable_members_unescape ./tests/members_unescaped.go
+	bin/easyjson ./tests/intern.go
 
 test: generate
 	go test \

README.md

@@ -137,6 +137,27 @@ through a call to `buffer.Init()` prior to any marshaling or unmarshaling.
 Please see the [GoDoc listing](https://godoc.org/github.com/mailru/easyjson/buffer)
 for more information.
 
+## String interning
+
+During unmarshaling, `string` field values can be optionally
+[interned](https://en.wikipedia.org/wiki/String_interning) to reduce memory
+allocations and usage by deduplicating strings in memory, at the expense of slightly
+increased CPU usage.
+
+This will work effectively only for `string` fields being decoded that have frequently
+the same value (e.g. if you have a string field that can only assume a small number
+of possible values).
+
+To enable string interning, add the `intern` keyword tag to your `json` tag on `string`
+fields, e.g.:
+
+```go
+type Foo struct {
+  UUID  string `json:"uuid"`         // will not be interned during unmarshaling
+  State string `json:"state,intern"` // will be interned during unmarshaling
+}
+```
+
 ## Issues, Notes, and Limitations
 
 * easyjson is still early in its development. As such, there are likely to be

benchmark/go.sum

@@ -2,6 +2,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=

gen/decoder.go

@@ -112,9 +112,15 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
 		fmt.Fprintln(g.out, ws+out+" = "+dec)
 		return nil
 	} else if dec := primitiveStringDecoders[t.Kind()]; dec != "" && tags.asString {
+		if tags.intern && t.Kind() == reflect.String {
+			dec = "in.StringIntern()"
+		}
 		fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")")
 		return nil
 	} else if dec := primitiveDecoders[t.Kind()]; dec != "" {
+		if tags.intern && t.Kind() == reflect.String {
+			dec = "in.StringIntern()"
+		}
 		fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")")
 		return nil
 	}
@@ -385,7 +391,6 @@ func getStructFields(t reflect.Type) ([]reflect.StructField, error) {
 			t1 = t1.Elem()
 		}
 
-
 		if t1.Kind() == reflect.Struct {
 			fs, err := getStructFields(t1)
 			if err != nil {
@@ -399,7 +404,6 @@ func getStructFields(t reflect.Type) ([]reflect.StructField, error) {
 		}
 	}
 
-
 	for i := 0; i < t.NumField(); i++ {
 		f := t.Field(i)
 		tags := parseFieldTags(f)

gen/encoder.go

@@ -58,6 +58,7 @@ type fieldTags struct {
 	noOmitEmpty bool
 	asString    bool
 	required    bool
+	intern      bool
 }
 
 // parseFieldTags parses the json field tag into a structure.
@@ -78,6 +79,8 @@ func parseFieldTags(f reflect.StructField) fieldTags {
 			ret.asString = true
 		case s == "required":
 			ret.required = true
+		case s == "intern":
+			ret.intern = true
 		}
 	}
 

go.mod

@@ -1,3 +1,5 @@
 module github.com/mailru/easyjson
 
 go 1.12
+
+require github.com/josharian/intern v1.0.0

jlexer/lexer.go

@@ -15,6 +15,8 @@ import (
 	"unicode"
 	"unicode/utf16"
 	"unicode/utf8"
+
+	"github.com/josharian/intern"
 )
 
 // tokenKind determines type of a token.
@@ -663,6 +665,20 @@ func (r *Lexer) String() string {
 	return ret
 }
 
+// StringIntern reads a string literal, and performs string interning on it.
+func (r *Lexer) StringIntern() string {
+	if r.token.kind == tokenUndef && r.Ok() {
+		r.FetchToken()
+	}
+	if !r.Ok() || r.token.kind != tokenString {
+		r.errInvalidToken("string")
+		return ""
+	}
+	ret := intern.Bytes(r.token.byteValue)
+	r.consume()
+	return ret
+}
+
 // Bytes reads a string literal and base64 decodes it into a byte slice.
 func (r *Lexer) Bytes() []byte {
 	if r.token.kind == tokenUndef && r.Ok() {

jlexer/lexer_test.go

@@ -29,21 +29,58 @@ func TestString(t *testing.T) {
 		{toParse: `"\x"`, wantError: true}, // invalid escape
 		{toParse: `"\ud800"`, want: "�"},   // invalid utf-8 char; return replacement char
 	} {
-		l := Lexer{Data: []byte(test.toParse)}
+		{
+			l := Lexer{Data: []byte(test.toParse)}
 
-		got := l.String()
-		if got != test.want {
-			t.Errorf("[%d, %q] String() = '%v'; want '%v'", i, test.toParse, got, test.want)
+			got := l.String()
+			if got != test.want {
+				t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want)
+			}
+			err := l.Error()
+			if err != nil && !test.wantError {
+				t.Errorf("[%d, %q] String() error: %v", i, test.toParse, err)
+			} else if err == nil && test.wantError {
+				t.Errorf("[%d, %q] String() ok; want error", i, test.toParse)
+			}
 		}
-		err := l.Error()
-		if err != nil && !test.wantError {
-			t.Errorf("[%d, %q] String() error: %v", i, test.toParse, err)
-		} else if err == nil && test.wantError {
-			t.Errorf("[%d, %q] String() ok; want error", i, test.toParse)
+		{
+			l := Lexer{Data: []byte(test.toParse)}
+
+			got := l.StringIntern()
+			if got != test.want {
+				t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want)
+			}
+			err := l.Error()
+			if err != nil && !test.wantError {
+				t.Errorf("[%d, %q] String() error: %v", i, test.toParse, err)
+			} else if err == nil && test.wantError {
+				t.Errorf("[%d, %q] String() ok; want error", i, test.toParse)
+			}
 		}
 	}
 }
 
+func TestStringIntern(t *testing.T) {
+	data := []byte(`"string interning test"`)
+	var l Lexer
+
+	allocsPerRun := testing.AllocsPerRun(1000, func() {
+		l = Lexer{Data: data}
+		_ = l.StringIntern()
+	})
+	if allocsPerRun != 0 {
+		t.Fatalf("expected 0 allocs, got %f", allocsPerRun)
+	}
+
+	allocsPerRun = testing.AllocsPerRun(1000, func() {
+		l = Lexer{Data: data}
+		_ = l.String()
+	})
+	if allocsPerRun != 1 {
+		t.Fatalf("expected 1 allocs, got %f", allocsPerRun)
+	}
+}
+
 func TestBytes(t *testing.T) {
 	for i, test := range []struct {
 		toParse   string

tests/basic_test.go

@@ -57,6 +57,7 @@ var testCases = []struct {
 	{&myGenDeclaredValue, myGenDeclaredString},
 	{&myGenDeclaredWithCommentValue, myGenDeclaredWithCommentString},
 	{&myTypeDeclaredValue, myTypeDeclaredString},
+	{&intern, internString},
 }
 
 func TestMarshal(t *testing.T) {

tests/intern.go

@@ -0,0 +1,14 @@
+package tests
+
+//easyjson:json
+type NoIntern struct {
+	Field string `json:"field"`
+}
+
+//easyjson:json
+type Intern struct {
+	Field string `json:"field,intern"`
+}
+
+var intern = Intern{Field: "interned"}
+var internString = `{"field":"interned"}`

tests/intern_test.go

@@ -0,0 +1,35 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/mailru/easyjson"
+)
+
+func TestStringIntern(t *testing.T) {
+	data := []byte(`{"field": "string interning test"}`)
+
+	var i Intern
+	allocsPerRun := testing.AllocsPerRun(1000, func() {
+		i = Intern{}
+		easyjson.Unmarshal(data, &i)
+		if i.Field != "string interning test" {
+			t.Fatalf("wrong value: %q", i.Field)
+		}
+	})
+	if allocsPerRun != 1 {
+		t.Fatalf("expected 1 allocs, got %f", allocsPerRun)
+	}
+
+	var n NoIntern
+	allocsPerRun = testing.AllocsPerRun(1000, func() {
+		n = NoIntern{}
+		easyjson.Unmarshal(data, &n)
+		if n.Field != "string interning test" {
+			t.Fatalf("wrong value: %q", n.Field)
+		}
+	})
+	if allocsPerRun != 2 {
+		t.Fatalf("expected 2 allocs, got %f", allocsPerRun)
+	}
+}