Add "nocopy" json tag to return unsafe string reference

When large amounts of data are unmarshaled for immediate non long live use than
copying of strings from original buffer can be avoided and instead a
reference to the original buffer returned in the field value.

Note, if the value requires unescaping, than it will be processed as
normally done with copying.

Signed-off-by: Kirill Korotaev <kirillx@gmail.com>
This commit is contained in:
Kirill Korotaev
2020-04-12 20:48:07 +03:00
parent 0d5e07263b
commit 244a66fa8f
6 changed files with 69 additions and 1 deletions
+3 -1
View File
@@ -23,7 +23,8 @@ generate: build
./tests/type_declaration.go \
./tests/members_escaped.go \
./tests/members_unescaped.go \
./tests/intern.go
./tests/intern.go \
./tests/nocopy.go \
bin/easyjson -all ./tests/data.go
bin/easyjson -all ./tests/nothing.go
@@ -44,6 +45,7 @@ generate: build
bin/easyjson ./tests/members_escaped.go
bin/easyjson -disable_members_unescape ./tests/members_unescaped.go
bin/easyjson ./tests/intern.go
bin/easyjson ./tests/nocopy.go
test: generate
go test \
+12
View File
@@ -80,6 +80,18 @@ Additional option notes:
* `-build_tags` will add the specified build tags to generated Go sources.
## Structure json tag options
Besides standart json tag options like 'omitempty' the following are supported:
* 'nocopy' - disables allocation and copying of string values, making them
refer to original json buffer memory. This works great for short lived
objects which are not hold in memory after decoding and immediate usage.
Note if string requires unescaping it will be processed as normally.
* 'intern' - string "interning" (deduplication) to save memory when the very
same string dictionory values are often met all over the structure.
See below for more details.
## Generated Marshaler/Unmarshaler Funcs
For Go struct types, easyjson generates the funcs `MarshalEasyJSON` /
+3
View File
@@ -121,6 +121,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
if tags.intern && t.Kind() == reflect.String {
dec = "in.StringIntern()"
}
if tags.noCopy && t.Kind() == reflect.String {
dec = "in.UnsafeString()"
}
fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")")
return nil
}
+3
View File
@@ -59,6 +59,7 @@ type fieldTags struct {
asString bool
required bool
intern bool
noCopy bool
}
// parseFieldTags parses the json field tag into a structure.
@@ -81,6 +82,8 @@ func parseFieldTags(f reflect.StructField) fieldTags {
ret.required = true
case s == "intern":
ret.intern = true
case s == "nocopy":
ret.noCopy = true
}
}
+7
View File
@@ -0,0 +1,7 @@
package tests
//easyjson:json
type NocopyStruct struct {
A string `json:"a"`
B string `json:"b,nocopy"`
}
+41
View File
@@ -0,0 +1,41 @@
package tests
import (
"reflect"
"testing"
"unsafe"
"github.com/mailru/easyjson"
)
// verifies if string pointer belongs to the given buffer or outside of it
func strBelongsTo(s string, buf []byte) bool {
sPtr := (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
bufPtr := (*reflect.SliceHeader)(unsafe.Pointer(&buf)).Data
if bufPtr <= sPtr && sPtr < bufPtr+uintptr(len(buf)) {
return true
}
return false
}
func TestNocopy(t *testing.T) {
data := []byte(`{"a": "valueA", "b": "valueB"}`)
exp := NocopyStruct{
A: "valueA",
B: "valueB",
}
res := NocopyStruct{}
easyjson.Unmarshal(data, &res)
if !reflect.DeepEqual(exp, res) {
t.Errorf("TestNocopy(): got=%+v, exp=%+v", res, exp)
}
if strBelongsTo(res.A, data) {
t.Error("TestNocopy(): field A was not copied and refers to buffer")
}
if !strBelongsTo(res.B, data) {
t.Error("TestNocopy(): field B was copied rather than refer to bufferr")
}
}