From ff14d442c9645c3ae955d2d433ba894d7668785c Mon Sep 17 00:00:00 2001 From: hjusforgues Date: Wed, 13 Jul 2016 13:50:58 +0700 Subject: [PATCH 1/6] Remove pointer-receivers for Marshalling func in bootstrap file --- bootstrap/bootstrap.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index e9a1d21..3a83e94 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -59,11 +59,11 @@ func (g *Generator) writeStub() error { for _, t := range g.Types { fmt.Fprintln(f) if !g.NoStdMarshalers { - fmt.Fprintln(f, "func (*", t, ") MarshalJSON() ([]byte, error) { return nil, nil }") + fmt.Fprintln(f, "func (", t, ") MarshalJSON() ([]byte, error) { return nil, nil }") fmt.Fprintln(f, "func (*", t, ") UnmarshalJSON([]byte) error { return nil }") } - fmt.Fprintln(f, "func (*", t, ") MarshalEasyJSON(w *jwriter.Writer) {}") + fmt.Fprintln(f, "func (", t, ") MarshalEasyJSON(w *jwriter.Writer) {}") fmt.Fprintln(f, "func (*", t, ") UnmarshalEasyJSON(l *jlexer.Lexer) {}") fmt.Fprintln(f) fmt.Fprintln(f, "type EasyJSON_exporter_"+t+" *"+t) From d4d111339231b98aee2a3945b4e8f0571e831622 Mon Sep 17 00:00:00 2001 From: Victor Starodub Date: Wed, 13 Jul 2016 18:14:04 +0300 Subject: [PATCH 2/6] Fix flaky test. --- tests/data.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/data.go b/tests/data.go index d0e1e7a..1c4ae78 100644 --- a/tests/data.go +++ b/tests/data.go @@ -437,8 +437,9 @@ var mapsString = `{` + `}` type DeepNest struct { - SliceMap map[Str][]Str - MapSlice []map[Str]Str + SliceMap map[Str][]Str + SliceMap1 map[Str][]Str + MapSlice []map[Str]Str } var deepNestValue = DeepNest{ @@ -447,6 +448,8 @@ var deepNestValue = DeepNest{ "0", "1", }, + }, + SliceMap1: map[Str][]Str{ "testSliceMap2": nil, }, MapSlice: []map[Str]Str{ @@ -458,7 +461,9 @@ var deepNestValue = DeepNest{ var deepNestString = `{` + `"SliceMap":{` + - `"testSliceMap1":["0","1"],` + + `"testSliceMap1":["0","1"]` + + `},` + + `"SliceMap1":{` + `"testSliceMap2":[]` + `},` + `"MapSlice":[` + From 2b7d5bf36878e5f31ff9bbd098e3ae443f536e39 Mon Sep 17 00:00:00 2001 From: Vadim Petrov Date: Wed, 13 Jul 2016 18:17:55 +0300 Subject: [PATCH 3/6] Fix generating valid files with no types to marshal/unmarshal in file --- Makefile | 4 +++- bootstrap/bootstrap.go | 19 ++++++++++++------- gen/generator.go | 7 ++++++- tests/nothing.go | 3 +++ 4 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 tests/nothing.go diff --git a/Makefile b/Makefile index 420f306..39d52e3 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,11 @@ generate: root build .root/bin/easyjson -stubs \ .root/src/$(PKG)/tests/snake.go \ .root/src/$(PKG)/tests/data.go \ - .root/src/$(PKG)/tests/omitempty.go + .root/src/$(PKG)/tests/omitempty.go \ + .root/src/$(PKG)/tests/nothing.go .root/bin/easyjson -all .root/src/$(PKG)/tests/data.go + .root/bin/easyjson -all .root/src/$(PKG)/tests/nothing.go .root/bin/easyjson -snake_case .root/src/$(PKG)/tests/snake.go .root/bin/easyjson -omit_empty .root/src/$(PKG)/tests/omitempty.go .root/bin/easyjson -build_tags=use_easyjson .root/src/$(PKG)/benchmark/data.go diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 3a83e94..04aaaad 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -50,11 +50,14 @@ func (g *Generator) writeStub() error { fmt.Fprintln(f, "// compilable during generation.") fmt.Fprintln(f) fmt.Fprintln(f, "package ", g.PkgName) - fmt.Fprintln(f) - fmt.Fprintln(f, "import (") - fmt.Fprintln(f, ` "`+pkgWriter+`"`) - fmt.Fprintln(f, ` "`+pkgLexer+`"`) - fmt.Fprintln(f, ")") + + if len(g.Types) > 0 { + fmt.Fprintln(f) + fmt.Fprintln(f, "import (") + fmt.Fprintln(f, ` "`+pkgWriter+`"`) + fmt.Fprintln(f, ` "`+pkgLexer+`"`) + fmt.Fprintln(f, ")") + } for _, t := range g.Types { fmt.Fprintln(f) @@ -90,8 +93,10 @@ func (g *Generator) writeMain() (path string, err error) { fmt.Fprintln(f, ` "os"`) fmt.Fprintln(f) fmt.Fprintf(f, " %q\n", genPackage) - fmt.Fprintln(f) - fmt.Fprintf(f, " pkg %q\n", g.PkgPath) + if len(g.Types) > 0 { + fmt.Fprintln(f) + fmt.Fprintf(f, " pkg %q\n", g.PkgPath) + } fmt.Fprintln(f, ")") fmt.Fprintln(f) fmt.Fprintln(f, "func main() {") diff --git a/gen/generator.go b/gen/generator.go index b6dc08e..8881063 100644 --- a/gen/generator.go +++ b/gen/generator.go @@ -156,7 +156,12 @@ func (g *Generator) printHeader() { fmt.Println(")") fmt.Println("") - fmt.Println("var _ = json.RawMessage{} // suppress unused package warning") + fmt.Println("// suppress unused package warning") + fmt.Println("var (") + fmt.Println(" _ = json.RawMessage{}") + fmt.Println(" _ = jlexer.Lexer{}") + fmt.Println(" _ = jwriter.Writer{}") + fmt.Println(")") fmt.Println() } diff --git a/tests/nothing.go b/tests/nothing.go new file mode 100644 index 0000000..35334f5 --- /dev/null +++ b/tests/nothing.go @@ -0,0 +1,3 @@ +package tests + +// No structs in this file From 98e04f6e05afc8c98c51f39a9743e13f27b568b1 Mon Sep 17 00:00:00 2001 From: Vadim Petrov Date: Wed, 13 Jul 2016 19:44:31 +0300 Subject: [PATCH 4/6] Fix generating nested structs with named types for slices and maps --- gen/decoder.go | 4 +-- gen/generator.go | 16 +++++++----- tests/data.go | 68 +++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 70 insertions(+), 18 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index a1291fc..4030b14 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -86,7 +86,7 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i fmt.Fprintln(g.out, ws+"in.Delim('[')") fmt.Fprintln(g.out, ws+"if !in.IsDelim(']') {") - fmt.Fprintln(g.out, ws+" "+out+" = make([]"+g.getType(elem)+", 0, "+fmt.Sprint(capacity)+")") + fmt.Fprintln(g.out, ws+" "+out+" = make("+g.getType(t)+", 0, "+fmt.Sprint(capacity)+")") fmt.Fprintln(g.out, ws+"} else {") fmt.Fprintln(g.out, ws+" "+out+" = nil") fmt.Fprintln(g.out, ws+"}") @@ -130,7 +130,7 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i fmt.Fprintln(g.out, ws+"} else {") fmt.Fprintln(g.out, ws+" in.Delim('{')") fmt.Fprintln(g.out, ws+" if !in.IsDelim('}') {") - fmt.Fprintln(g.out, ws+" "+out+" = make(map["+g.getType(t.Key())+"]"+g.getType(t.Elem())+")") + fmt.Fprintln(g.out, ws+" "+out+" = make("+g.getType(t)+")") fmt.Fprintln(g.out, ws+" } else {") fmt.Fprintln(g.out, ws+" "+out+" = nil") fmt.Fprintln(g.out, ws+" }") diff --git a/gen/generator.go b/gen/generator.go index b6dc08e..3128594 100644 --- a/gen/generator.go +++ b/gen/generator.go @@ -222,13 +222,15 @@ func (g *Generator) pkgAlias(pkgPath string) string { // getType return the textual type name of given type that can be used in generated code. func (g *Generator) getType(t reflect.Type) string { - switch t.Kind() { - case reflect.Ptr: - return "*" + g.getType(t.Elem()) - case reflect.Slice: - return "[]" + g.getType(t.Elem()) - case reflect.Map: - return "map[" + g.getType(t.Key()) + "]" + g.getType(t.Elem()) + if t.Name() == "" { + switch t.Kind() { + case reflect.Ptr: + return "*" + g.getType(t.Elem()) + case reflect.Slice: + return "[]" + g.getType(t.Elem()) + case reflect.Map: + return "map[" + g.getType(t.Key()) + "]" + g.getType(t.Elem()) + } } if t.Name() == "" || t.PkgPath() == "" { diff --git a/tests/data.go b/tests/data.go index 1c4ae78..39bc9c8 100644 --- a/tests/data.go +++ b/tests/data.go @@ -436,38 +436,88 @@ var mapsString = `{` + `"CustomMap":{"c":"d"}` + `}` +type NamedSlice []Str +type NamedMap map[Str]Str + type DeepNest struct { - SliceMap map[Str][]Str - SliceMap1 map[Str][]Str - MapSlice []map[Str]Str + SliceMap map[Str][]Str + SliceMap1 map[Str][]Str + NamedSliceMap map[Str]NamedSlice + NamedMapMap map[Str]NamedMap + MapSlice []map[Str]Str + NamedSliceSlice []NamedSlice + NamedMapSlice []NamedMap } var deepNestValue = DeepNest{ SliceMap: map[Str][]Str{ - "testSliceMap1": []Str{ + "testSliceMap": []Str{ "0", "1", }, }, SliceMap1: map[Str][]Str{ - "testSliceMap2": nil, + "testSliceMap1": nil, + }, + NamedSliceMap: map[Str]NamedSlice{ + "testNamedSliceMap": NamedSlice{ + "2", + "3", + }, + }, + NamedMapMap: map[Str]NamedMap{ + "testNamedMapMap": NamedMap{ + "key1": "value1", + }, }, MapSlice: []map[Str]Str{ map[Str]Str{ - "testMapSlice1": "someValue", + "testMapSlice": "someValue", + }, + }, + NamedSliceSlice: []NamedSlice{ + NamedSlice{ + "someValue1", + "someValue2", + }, + NamedSlice{ + "someValue3", + "someValue4", + }, + }, + NamedMapSlice: []NamedMap{ + NamedMap{ + "key2": "value2", + }, + NamedMap{ + "key3": "value3", }, }, } var deepNestString = `{` + `"SliceMap":{` + - `"testSliceMap1":["0","1"]` + + `"testSliceMap":["0","1"]` + `},` + `"SliceMap1":{` + - `"testSliceMap2":[]` + + `"testSliceMap1":[]` + + `},` + + `"NamedSliceMap":{` + + `"testNamedSliceMap":["2","3"]` + + `},` + + `"NamedMapMap":{` + + `"testNamedMapMap":{"key1":"value1"}` + `},` + `"MapSlice":[` + - `{"testMapSlice1":"someValue"}` + + `{"testMapSlice":"someValue"}` + + `],` + + `"NamedSliceSlice":[` + + `["someValue1","someValue2"],` + + `["someValue3","someValue4"]` + + `],` + + `"NamedMapSlice":[` + + `{"key2":"value2"},` + + `{"key3":"value3"}` + `]` + `}` From 0032ae6d741f4b1f9f0844edf94c0f54c4caf975 Mon Sep 17 00:00:00 2001 From: Vadim Petrov Date: Wed, 13 Jul 2016 22:29:35 +0300 Subject: [PATCH 5/6] Add slice of NamedString to test --- tests/data.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/data.go b/tests/data.go index 39bc9c8..6775096 100644 --- a/tests/data.go +++ b/tests/data.go @@ -440,13 +440,14 @@ type NamedSlice []Str type NamedMap map[Str]Str type DeepNest struct { - SliceMap map[Str][]Str - SliceMap1 map[Str][]Str - NamedSliceMap map[Str]NamedSlice - NamedMapMap map[Str]NamedMap - MapSlice []map[Str]Str - NamedSliceSlice []NamedSlice - NamedMapSlice []NamedMap + SliceMap map[Str][]Str + SliceMap1 map[Str][]Str + NamedSliceMap map[Str]NamedSlice + NamedMapMap map[Str]NamedMap + MapSlice []map[Str]Str + NamedSliceSlice []NamedSlice + NamedMapSlice []NamedMap + NamedStringSlice []NamedString } var deepNestValue = DeepNest{ @@ -493,6 +494,9 @@ var deepNestValue = DeepNest{ "key3": "value3", }, }, + NamedStringSlice: []NamedString{ + "value4", "value5", + }, } var deepNestString = `{` + @@ -518,7 +522,8 @@ var deepNestString = `{` + `"NamedMapSlice":[` + `{"key2":"value2"},` + `{"key3":"value3"}` + - `]` + + `],` + + `"NamedStringSlice":["value4","value5"]` + `}` type RequiredOptionalStruct struct { From 45c84278be3852f1220c5703ce8d25aa32070ec1 Mon Sep 17 00:00:00 2001 From: Vasily Romanov Date: Thu, 14 Jul 2016 13:47:41 +0300 Subject: [PATCH 6/6] correct encode broken utf --- jwriter/writer.go | 95 +++++++++++++++++++++++++++------------------ tests/basic_test.go | 36 +++++++++-------- tests/data.go | 4 -- 3 files changed, 78 insertions(+), 57 deletions(-) diff --git a/jwriter/writer.go b/jwriter/writer.go index 85fa440..907675f 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -4,15 +4,15 @@ package jwriter import ( "io" "strconv" + "unicode/utf8" "github.com/mailru/easyjson/buffer" ) // Writer is a JSON writer. type Writer struct { - EscapeLtGt bool - Error error - Buffer buffer.Buffer + Error error + Buffer buffer.Buffer } // Size returns the size of the data that was written out. @@ -198,10 +198,7 @@ func (w *Writer) Bool(v bool) { } } -func hex(c byte) byte { - const chars = "0123456789abcdef" - return chars[c&0xf] -} +const chars = "0123456789abcdef" func (w *Writer) String(s string) { w.Buffer.AppendByte('"') @@ -211,41 +208,65 @@ func (w *Writer) String(s string) { p := 0 // last non-escape symbol - for i := 0; i < len(s); i++ { - c := s[i] - var escape byte - switch c { - case '\t': - escape = 't' - case '\r': - escape = 'r' - case '\n': - escape = 'n' - case '\\': - escape = '\\' - case '"': - escape = '"' - case '<', '>': - if !w.EscapeLtGt { - continue + for i := 0; i < len(s); { + // single-with character + if c := s[i]; c < utf8.RuneSelf { + var escape byte + switch c { + case '\t': + escape = 't' + case '\r': + escape = 'r' + case '\n': + escape = 'n' + case '\\': + escape = '\\' + case '"': + escape = '"' + case '<', '>': + // do nothing + default: + if c >= 0x20 { + // no escaping is required + i++ + continue + } } - default: - if c >= 0x20 { - // no escaping is required - continue + if escape != 0 { + w.Buffer.AppendString(s[p:i]) + w.Buffer.AppendByte('\\') + w.Buffer.AppendByte(escape) + } else { + w.Buffer.AppendString(s[p:i]) + w.Buffer.AppendString(`\u00`) + w.Buffer.AppendByte(chars[c>>4]) + w.Buffer.AppendByte(chars[c&0xf]) } + i++ + p = i + continue } - if escape != 0 { + + // broken utf + runeValue, runeWidth := utf8.DecodeRuneInString(s[i:]) + if runeValue == utf8.RuneError && runeWidth == 1 { w.Buffer.AppendString(s[p:i]) - w.Buffer.AppendByte('\\') - w.Buffer.AppendByte(escape) - } else { - w.Buffer.AppendString(s[p:i]) - w.Buffer.AppendString(`\u00`) - w.Buffer.AppendByte(hex(c >> 4)) - w.Buffer.AppendByte(hex(c)) + w.Buffer.AppendString(`\ufffd`) + i++ + p = i + continue } - p = i + 1 + + // jsonp stuff - tab separator and line separator + if runeValue == '\u2028' || runeValue == '\u2029' { + w.Buffer.AppendString(s[p:i]) + w.Buffer.AppendString(`\u202`) + w.Buffer.AppendByte(chars[runeValue&0xf]) + i += runeWidth + p = i + continue + } + i += runeWidth } w.Buffer.AppendString(s[p:]) w.Buffer.AppendByte('"') diff --git a/tests/basic_test.go b/tests/basic_test.go index f6fdc92..8525240 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -105,24 +105,28 @@ func TestParseNull(t *testing.T) { } } -var testCasesEncodeLtGt = []struct { - Writer *jwriter.Writer - Encoded string +var testSpecialCases = []struct { + EncodedString string + Value string }{ - {&jwriter.Writer{ - EscapeLtGt: false, - }, encodeLtGtFalseWantString}, - {&jwriter.Writer{ - EscapeLtGt: true, - }, encodeLtGtTrueWantString}, + {`"Username \u003cuser@example.com\u003e"`, `Username `}, + {`"Username\ufffd"`, "Username\xc5"}, + {`"тестzтест"`, "тестzтест"}, + {`"тест\ufffdтест"`, "тест\xc5тест"}, + {`"绿茶"`, "绿茶"}, + {`"绿\ufffd茶"`, "绿\xc5茶"}, + {`"тест\u2028"`, "тест\xE2\x80\xA8"}, + {`"\\\r\n\t\""`, "\\\r\n\t\""}, + {`"ü"`, "ü"}, } -func TestEncodeLtGt(t *testing.T) { - for i, test := range testCasesEncodeLtGt { - test.Writer.String(encodeLtGtString) - got := string(test.Writer.Buffer.BuildBytes()) - if got != test.Encoded { - t.Errorf("[%d] Encoded() = %+v; want %+v", i, got, test.Encoded) +func TestSpecialCases(t *testing.T) { + for i, test := range testSpecialCases { + w := jwriter.Writer{} + w.String(test.Value) + got := string(w.Buffer.BuildBytes()) + if got != test.EncodedString { + t.Errorf("[%d] Encoded() = %+v; want %+v", i, got, test.EncodedString) } } -} +} \ No newline at end of file diff --git a/tests/data.go b/tests/data.go index 6775096..1378f89 100644 --- a/tests/data.go +++ b/tests/data.go @@ -530,7 +530,3 @@ type RequiredOptionalStruct struct { FirstName string `json:"first_name,required"` Lastname string `json:"last_name"` } - -var encodeLtGtString = `Username ` -var encodeLtGtFalseWantString = `"Username "` -var encodeLtGtTrueWantString = `"Username \u003cuser@example.com\u003e"`