From 01d6457c01d1f088b6b483dd8961babc87e4604f Mon Sep 17 00:00:00 2001 From: Dan Field Date: Wed, 26 Jul 2017 11:08:51 -0400 Subject: [PATCH 1/7] Update parser_windows.go Use filepath.IsAbs instead of path.IsAbs for Windows; convert paths to lowercase when normalizing them --- parser/parser_windows.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/parser/parser_windows.go b/parser/parser_windows.go index 64974aa..a0b2611 100644 --- a/parser/parser_windows.go +++ b/parser/parser_windows.go @@ -4,15 +4,18 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" ) func normalizePath(path string) string { - return strings.Replace(path, "\\", "/", -1) + // use lower case, as Windows file systems will almost always be case insensitive + return strings.ToLower(strings.Replace(path, "\\", "/", -1)) } func getPkgPath(fname string, isDir bool) (string, error) { - if !path.IsAbs(fname) { + // path.IsAbs doesn't work properly on Windows; use filepath.IsAbs instead + if !filepath.IsAbs(fname) { pwd, err := os.Getwd() if err != nil { return "", err From 1a9a11a1d1791bb923db17933b2d2880d4c824af Mon Sep 17 00:00:00 2001 From: James Nugent Date: Fri, 28 Jul 2017 14:54:39 -0500 Subject: [PATCH 2/7] Trim whitespace around build tags This commit trims whitespace around the build tags specified on the command line in order that invocations such as: easyjson -build_tags linux generate the correct "// +build linux" rather than adding additional whitespace to give "// +build linux". --- easyjson/main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easyjson/main.go b/easyjson/main.go index 1c39497..1cd30bb 100644 --- a/easyjson/main.go +++ b/easyjson/main.go @@ -54,8 +54,13 @@ func generate(fname string) (err error) { outName = *specifiedName } + var trimmedBuildTags string + if *buildTags != "" { + trimmedBuildTags = strings.TrimSpace(*buildTags) + } + g := bootstrap.Generator{ - BuildTags: *buildTags, + BuildTags: trimmedBuildTags, PkgPath: p.PkgPath, PkgName: p.PkgName, Types: p.StructNames, From 56b59a977c074f5565a974fc6e1fb0d243286092 Mon Sep 17 00:00:00 2001 From: JUNJIE NAN Date: Wed, 16 Aug 2017 23:15:17 +0800 Subject: [PATCH 3/7] Avoid pkg alias naming conflict with vars Say the import pkg is `pkg/v1`, and one var name may be `v1` too. The conflict will fail the build. In the fix we add `_` before the alias name if it starts with character `v`. --- gen/generator.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gen/generator.go b/gen/generator.go index f4312f6..eb0d70b 100644 --- a/gen/generator.go +++ b/gen/generator.go @@ -224,6 +224,10 @@ func fixAliasName(alias string) string { "_", -1, ) + + if alias[0] == 'v' { // to void conflicting with var names, say v1 + alias = "_" + alias + } return alias } @@ -380,7 +384,7 @@ func (DefaultFieldNamer) GetJSONFieldName(t reflect.Type, f reflect.StructField) } // LowerCamelCaseFieldNamer -type LowerCamelCaseFieldNamer struct {} +type LowerCamelCaseFieldNamer struct{} func isLower(b byte) bool { return b <= 122 && b >= 97 @@ -407,7 +411,7 @@ func lowerFirst(s string) string { If the following char is upper OR numeric, LOWER it If is the end of string, LEAVE it Else lowercase - */ + */ foundLower := false for i := range s { From c90651112a7fc1782690b0a652137a4eab45f444 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Fri, 1 Sep 2017 09:43:11 +0100 Subject: [PATCH 4/7] correct spelling mistake --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d19751e..9366e3f 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Usage of easyjson: ``` Using `-all` will generate marshalers/unmarshalers for all Go structs in the -file. If `-all` is not provided, then only those structs whose preceeding +file. If `-all` is not provided, then only those structs whose preceding comment starts with `easyjson:json` will have marshalers/unmarshalers generated. For example: From bb658fd1b2e7634f1739359a44d1003e7f6ca448 Mon Sep 17 00:00:00 2001 From: Vladimir Varankin Date: Sat, 2 Sep 2017 16:41:07 +0300 Subject: [PATCH 5/7] add jsoniter to benchmarks --- .travis.yml | 9 +-- Makefile | 3 +- benchmark/default_test.go | 2 +- benchmark/jsoniter_test.go | 119 +++++++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 benchmark/jsoniter_test.go diff --git a/.travis.yml b/.travis.yml index 3e5ac13..884f8bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: go go: - - tip + - tip install: - - go get github.com/ugorji/go/codec - - go get github.com/pquerna/ffjson/fflib/v1 - - go get github.com/golang/lint/golint + - go get github.com/ugorji/go/codec + - go get github.com/pquerna/ffjson/fflib/v1 + - go get github.com/json-iterator/go + - go get github.com/golang/lint/golint diff --git a/Makefile b/Makefile index 8e720a0..ea591b0 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ export GOPATH all: test -.root/src/$(PKG): +.root/src/$(PKG): mkdir -p $@ for i in $$PWD/* ; do ln -s $$i $@/`basename $$i` ; done @@ -45,6 +45,7 @@ test: generate root bench-other: generate root @go test -benchmem -bench . $(PKG)/benchmark @go test -benchmem -tags use_ffjson -bench . $(PKG)/benchmark + @go test -benchmem -tags use_jsoniter -bench . $(PKG)/benchmark @go test -benchmem -tags use_codec -bench . $(PKG)/benchmark bench-python: diff --git a/benchmark/default_test.go b/benchmark/default_test.go index b647bef..68b3791 100644 --- a/benchmark/default_test.go +++ b/benchmark/default_test.go @@ -1,4 +1,4 @@ -// +build !use_easyjson,!use_ffjson,!use_codec +// +build !use_easyjson,!use_ffjson,!use_codec,!use_jsoniter package benchmark diff --git a/benchmark/jsoniter_test.go b/benchmark/jsoniter_test.go new file mode 100644 index 0000000..004f891 --- /dev/null +++ b/benchmark/jsoniter_test.go @@ -0,0 +1,119 @@ +// +build use_jsoniter + +package benchmark + +import ( + "testing" + + jsoniter "github.com/json-iterator/go" +) + +func BenchmarkJI_Unmarshal_M(b *testing.B) { + b.SetBytes(int64(len(largeStructText))) + for i := 0; i < b.N; i++ { + var s LargeStruct + err := jsoniter.Unmarshal(largeStructText, &s) + if err != nil { + b.Error(err) + } + } +} + +func BenchmarkJI_Unmarshal_S(b *testing.B) { + for i := 0; i < b.N; i++ { + var s Entities + err := jsoniter.Unmarshal(smallStructText, &s) + if err != nil { + b.Error(err) + } + } + b.SetBytes(int64(len(smallStructText))) +} + +func BenchmarkJI_Marshal_M(b *testing.B) { + var l int64 + for i := 0; i < b.N; i++ { + data, err := jsoniter.Marshal(&largeStructData) + if err != nil { + b.Error(err) + } + l = int64(len(data)) + } + b.SetBytes(l) +} + +func BenchmarkJI_Marshal_L(b *testing.B) { + var l int64 + for i := 0; i < b.N; i++ { + data, err := jsoniter.Marshal(&xlStructData) + if err != nil { + b.Error(err) + } + l = int64(len(data)) + } + b.SetBytes(l) +} + +func BenchmarkJI_Marshal_M_Parallel(b *testing.B) { + var l int64 + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + data, err := jsoniter.Marshal(&largeStructData) + if err != nil { + b.Error(err) + } + l = int64(len(data)) + } + }) + b.SetBytes(l) +} + +func BenchmarkJI_Marshal_L_Parallel(b *testing.B) { + var l int64 + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + data, err := jsoniter.Marshal(&xlStructData) + if err != nil { + b.Error(err) + } + l = int64(len(data)) + } + }) + b.SetBytes(l) +} + +func BenchmarkJI_Marshal_S(b *testing.B) { + var l int64 + for i := 0; i < b.N; i++ { + data, err := jsoniter.Marshal(&smallStructData) + if err != nil { + b.Error(err) + } + l = int64(len(data)) + } + b.SetBytes(l) +} + +func BenchmarkJI_Marshal_S_Parallel(b *testing.B) { + var l int64 + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + data, err := jsoniter.Marshal(&smallStructData) + if err != nil { + b.Error(err) + } + l = int64(len(data)) + } + }) + b.SetBytes(l) +} + +func BenchmarkJI_Marshal_M_ToWriter(b *testing.B) { + enc := jsoniter.NewEncoder(&DummyWriter{}) + for i := 0; i < b.N; i++ { + err := enc.Encode(&largeStructData) + if err != nil { + b.Error(err) + } + } +} From 0172647770beb4c561ab78aedb623d13fc205b63 Mon Sep 17 00:00:00 2001 From: Chuntao Lu Date: Mon, 16 Oct 2017 18:33:24 -0700 Subject: [PATCH 6/7] Handle errors when generating encoder/decoder for nested structures --- gen/decoder.go | 16 ++++++++--- gen/encoder.go | 16 ++++++++--- tests/non_string_keyed_map_test.go | 44 ++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 tests/non_string_keyed_map_test.go diff --git a/gen/decoder.go b/gen/decoder.go index 2fece5d..600bb32 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -127,7 +127,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" for !in.IsDelim(']') {") fmt.Fprintln(g.out, ws+" var "+tmpVar+" "+g.getType(elem)) - g.genTypeDecoder(elem, tmpVar, tags, indent+2) + if err := g.genTypeDecoder(elem, tmpVar, tags, indent+2); err != nil { + return err + } fmt.Fprintln(g.out, ws+" "+out+" = append("+out+", "+tmpVar+")") fmt.Fprintln(g.out, ws+" in.WantComma()") @@ -159,7 +161,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" for !in.IsDelim(']') {") fmt.Fprintln(g.out, ws+" if "+iterVar+" < "+fmt.Sprint(length)+" {") - g.genTypeDecoder(elem, out+"["+iterVar+"]", tags, indent+3) + if err := g.genTypeDecoder(elem, out+"["+iterVar+"]", tags, indent+3); err != nil { + return err + } fmt.Fprintln(g.out, ws+" "+iterVar+"++") fmt.Fprintln(g.out, ws+" } else {") @@ -186,7 +190,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" "+out+" = new("+g.getType(t.Elem())+")") fmt.Fprintln(g.out, ws+" }") - g.genTypeDecoder(t.Elem(), "*"+out, tags, indent+1) + if err := g.genTypeDecoder(t.Elem(), "*"+out, tags, indent+1); err != nil { + return err + } fmt.Fprintln(g.out, ws+"}") @@ -213,7 +219,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" in.WantColon()") fmt.Fprintln(g.out, ws+" var "+tmpVar+" "+g.getType(elem)) - g.genTypeDecoder(elem, tmpVar, tags, indent+2) + if err := g.genTypeDecoder(elem, tmpVar, tags, indent+2); err != nil { + return err + } fmt.Fprintln(g.out, ws+" ("+out+")[key] = "+tmpVar) fmt.Fprintln(g.out, ws+" in.WantComma()") diff --git a/gen/encoder.go b/gen/encoder.go index a54f6e2..e3f7c40 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -137,7 +137,9 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT fmt.Fprintln(g.out, ws+" out.RawByte(',')") fmt.Fprintln(g.out, ws+" }") - g.genTypeEncoder(elem, vVar, tags, indent+2) + if err := g.genTypeEncoder(elem, vVar, tags, indent+2); err != nil { + return err + } fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+" out.RawByte(']')") @@ -157,7 +159,9 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT fmt.Fprintln(g.out, ws+" out.RawByte(',')") fmt.Fprintln(g.out, ws+" }") - g.genTypeEncoder(elem, in+"["+iVar+"]", tags, indent+1) + if err := g.genTypeEncoder(elem, in+"["+iVar+"]", tags, indent+1); err != nil { + return err + } fmt.Fprintln(g.out, ws+"}") fmt.Fprintln(g.out, ws+"out.RawByte(']')") @@ -174,7 +178,9 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT fmt.Fprintln(g.out, ws+` out.RawString("null")`) fmt.Fprintln(g.out, ws+"} else {") - g.genTypeEncoder(t.Elem(), "*"+in, tags, indent+1) + if err := g.genTypeEncoder(t.Elem(), "*"+in, tags, indent+1); err != nil { + return err + } fmt.Fprintln(g.out, ws+"}") @@ -196,7 +202,9 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT fmt.Fprintln(g.out, ws+" out.String(string("+tmpVar+"Name))") fmt.Fprintln(g.out, ws+" out.RawByte(':')") - g.genTypeEncoder(t.Elem(), tmpVar+"Value", tags, indent+2) + if err := g.genTypeEncoder(t.Elem(), tmpVar+"Value", tags, indent+2); err != nil { + return err + } fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+" out.RawByte('}')") diff --git a/tests/non_string_keyed_map_test.go b/tests/non_string_keyed_map_test.go new file mode 100644 index 0000000..392b8f3 --- /dev/null +++ b/tests/non_string_keyed_map_test.go @@ -0,0 +1,44 @@ +package tests + +import ( + "os" + "testing" + + "github.com/mailru/easyjson/gen" +) + +type IntMap map[int]string +type IntMapSlice []IntMap +type IntMapArray [2]IntMap +type IntMapPtr *IntMap +type IntMapMap map[string]IntMap + +func TestNonStringKeyedtMapEncoder(t *testing.T) { + f := "non_string_keyed_map_easyjson.go" + for _, test := range []struct { + Data interface{} + }{ + { + Data: IntMap{}, + }, + { + Data: IntMapSlice{}, + }, + { + Data: IntMapArray{}, + }, + { + Data: IntMapPtr(nil), + }, + { + Data: IntMapMap{}, + }, + } { + g := gen.NewGenerator(f) + g.Add(test.Data) + e := g.Run(os.Stdout) + if e == nil { + t.Errorf("generation for %#v should have errored", test.Data) + } + } +} From 67fbe08504b4b75bf7121d5c5eaf8c5241d5dafa Mon Sep 17 00:00:00 2001 From: lispad Date: Wed, 18 Oct 2017 00:11:05 +0300 Subject: [PATCH 7/7] Feature: allow to unmarshal json.Number Both go native encoding/json json.Unmarshal and ffjson's UnmarshalJSON methods could unmarshal both json-encoded strings: `{"number": 10}` `{"number": "20"}` to struct { Number json.Number `json:"number"` } But easyjson got error on unmarshaling second string. --- gen/decoder.go | 9 ++++++- jlexer/lexer.go | 23 +++++++++++++++++ jlexer/lexer_test.go | 60 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/gen/decoder.go b/gen/decoder.go index 600bb32..f8f43e5 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -48,6 +48,10 @@ var primitiveStringDecoders = map[reflect.Kind]string{ reflect.Uint64: "in.Uint64Str()", } +var customDecoders = map[string]string{ + "json.Number": "in.JsonNumber()", +} + // genTypeDecoder generates decoding code for the type t, but uses unmarshaler interface if implemented by t. func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, indent int) error { ws := strings.Repeat(" ", indent) @@ -82,7 +86,10 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags fieldTags, indent int) error { ws := strings.Repeat(" ", indent) // Check whether type is primitive, needs to be done after interface check. - if dec := primitiveStringDecoders[t.Kind()]; dec != "" && tags.asString { + if dec := customDecoders[t.String()]; dec != "" { + fmt.Fprintln(g.out, ws+out+" = "+dec) + return nil + } else if dec := primitiveStringDecoders[t.Kind()]; dec != "" && tags.asString { fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")") return nil } else if dec := primitiveDecoders[t.Kind()]; dec != "" { diff --git a/jlexer/lexer.go b/jlexer/lexer.go index e81f103..563ca06 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -6,6 +6,7 @@ package jlexer import ( "encoding/base64" + "encoding/json" "errors" "fmt" "io" @@ -1043,6 +1044,28 @@ func (r *Lexer) GetNonFatalErrors() []*LexerError { return r.multipleErrors } +// JsonNumber fetches and json.Number from 'encoding/json' package. +// Both int, float or string, contains them are valid values +func (r *Lexer) JsonNumber() json.Number { + if r.token.kind == tokenUndef && r.Ok() { + r.FetchToken() + } + if !r.Ok() { + r.errInvalidToken("json.Number") + return json.Number("0") + } + + switch r.token.kind { + case tokenString: + return json.Number(r.String()) + case tokenNumber: + return json.Number(r.Raw()) + default: + r.errSyntax() + return json.Number("0") + } +} + // Interface fetches an interface{} analogous to the 'encoding/json' package. func (r *Lexer) Interface() interface{} { if r.token.kind == tokenUndef && r.Ok() { diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index b8a6498..4ce4abe 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -2,6 +2,7 @@ package jlexer import ( "bytes" + "encoding/json" "reflect" "testing" ) @@ -249,3 +250,62 @@ func TestConsumed(t *testing.T) { } } } + +func TestJsonNumber(t *testing.T) { + for i, test := range []struct { + toParse string + want json.Number + wantLexerError bool + wantValue interface{} + wantValueError bool + }{ + {toParse: `10`, want: json.Number("10"), wantValue: int64(10)}, + {toParse: `0`, want: json.Number("0"), wantValue: int64(0)}, + {toParse: `0.12`, want: json.Number("0.12"), wantValue: 0.12}, + {toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25E-4}, + + {toParse: `"10"`, want: json.Number("10"), wantValue: int64(10)}, + {toParse: `"0"`, want: json.Number("0"), wantValue: int64(0)}, + {toParse: `"0.12"`, want: json.Number("0.12"), wantValue: 0.12}, + {toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25E-4}, + + {toParse: `"a""`, wantValueError: true}, + + {toParse: `[1]`, wantLexerError: true}, + {toParse: `{}`, wantLexerError: true}, + {toParse: `a`, wantLexerError: true}, + } { + l := Lexer{Data: []byte(test.toParse)} + + got := l.JsonNumber() + if got != test.want && !test.wantLexerError && !test.wantValueError { + t.Errorf("[%d, %q] JsonNumber() = %v; want %v", i, test.toParse, got, test.want) + } + + err := l.Error() + if err != nil && !test.wantLexerError { + t.Errorf("[%d, %q] JsonNumber() lexer error: %v", i, test.toParse, err) + } else if err == nil && test.wantLexerError { + t.Errorf("[%d, %q] JsonNumber() ok; want lexer error", i, test.toParse) + } + + var valueErr error + var gotValue interface{} + switch test.wantValue.(type) { + case float64: + gotValue, valueErr = got.Float64() + default: + gotValue, valueErr = got.Int64() + } + + if !reflect.DeepEqual(gotValue, test.wantValue) && !test.wantLexerError && !test.wantValueError { + t.Errorf("[%d, %q] JsonNumber() = %v; want %v", i, test.toParse, gotValue, test.wantValue) + } + + if valueErr != nil && !test.wantValueError { + t.Errorf("[%d, %q] JsonNumber() value error: %v", i, test.toParse, err) + } else if valueErr == nil && test.wantValueError { + t.Errorf("[%d, %q] JsonNumber() ok; want value error", i, test.toParse) + } + } +}