diff --git a/gen/decoder.go b/gen/decoder.go index 706f77b..94c1a9a 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -13,7 +13,7 @@ import ( // Target this byte size for initial slice allocation to reduce garbage collection. const minSliceBytes = 64 -func (g *Generator) getStructDecoderName(t reflect.Type) string { +func (g *Generator) getDecoderName(t reflect.Type) string { return g.functionName("decode", t) } @@ -47,7 +47,7 @@ var primitiveStringDecoders = map[reflect.Kind]string{ reflect.Uint64: "in.Uint64Str()", } -// genTypeDecoder generates decoding code for the type t. +// genTypeDecoder generates decoding code for the type t, but uses unmarshaler interface if implemented by t. func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, indent int) error { ws := strings.Repeat(" ", indent) @@ -65,6 +65,13 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i return nil } + err := g.genTypeDecoderNoCheck(t, out, tags, indent) + return err +} + +// genTypeDecoderNoCheck generates decoding code for the type t. +func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags fieldTags, indent int) error { + ws := strings.Repeat(" ", indent) // Check whether type is primitive, needs to be done after interface check. if dec := primitiveStringDecoders[t.Kind()]; dec != "" && tags.asString { fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")") @@ -86,7 +93,7 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i fmt.Fprintln(g.out, ws+"in.Delim('[')") fmt.Fprintln(g.out, ws+"if !in.IsDelim(']') {") - fmt.Fprintln(g.out, ws+" "+out+" = make([]"+g.getType(elem)+", 0, "+fmt.Sprint(capacity)+")") + fmt.Fprintln(g.out, ws+" "+out+" = make("+g.getType(t)+", 0, "+fmt.Sprint(capacity)+")") fmt.Fprintln(g.out, ws+"} else {") fmt.Fprintln(g.out, ws+" "+out+" = nil") fmt.Fprintln(g.out, ws+"}") @@ -101,7 +108,7 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i fmt.Fprintln(g.out, ws+"in.Delim(']')") case reflect.Struct: - dec := g.getStructDecoderName(t) + dec := g.getDecoderName(t) g.addType(t) fmt.Fprintln(g.out, ws+dec+"(in, &"+out+")") @@ -130,7 +137,7 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i fmt.Fprintln(g.out, ws+"} else {") fmt.Fprintln(g.out, ws+" in.Delim('{')") fmt.Fprintln(g.out, ws+" if !in.IsDelim('}') {") - fmt.Fprintln(g.out, ws+" "+out+" = make(map["+g.getType(t.Key())+"]"+g.getType(t.Elem())+")") + fmt.Fprintln(g.out, ws+" "+out+" = make("+g.getType(t)+")") fmt.Fprintln(g.out, ws+" } else {") fmt.Fprintln(g.out, ws+" "+out+" = nil") fmt.Fprintln(g.out, ws+" }") @@ -260,12 +267,39 @@ func getStructFields(t reflect.Type) ([]reflect.StructField, error) { return mergeStructFields(efields, fields), nil } +func (g *Generator) genDecoder(t reflect.Type) error { + switch t.Kind() { + case reflect.Slice: + return g.genSliceDecoder(t) + default: + return g.genStructDecoder(t) + } +} + +func (g *Generator) genSliceDecoder(t reflect.Type) error { + if t.Kind() != reflect.Slice { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a slice type", t) + } + + fname := g.getDecoderName(t) + typ := g.getType(t) + + fmt.Fprintln(g.out, "func "+fname+"(in *jlexer.Lexer, out *"+typ+") {") + err := g.genTypeDecoderNoCheck(t, "*out", fieldTags{}, 1) + if err != nil { + return err + } + fmt.Fprintln(g.out, "}") + + return nil +} + func (g *Generator) genStructDecoder(t reflect.Type) error { if t.Kind() != reflect.Struct { return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct type", t) } - fname := g.getStructDecoderName(t) + fname := g.getDecoderName(t) typ := g.getType(t) fmt.Fprintln(g.out, "func "+fname+"(in *jlexer.Lexer, out *"+typ+") {") @@ -326,11 +360,11 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { } func (g *Generator) genStructUnmarshaller(t reflect.Type) error { - if t.Kind() != reflect.Struct { - return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct type", t) + if t.Kind() != reflect.Struct && t.Kind() != reflect.Slice { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct/slice type", t) } - fname := g.getStructDecoderName(t) + fname := g.getDecoderName(t) typ := g.getType(t) if !g.noStdMarshalers { diff --git a/gen/encoder.go b/gen/encoder.go index f860e79..e5b6ab3 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -10,7 +10,7 @@ import ( "github.com/mailru/easyjson" ) -func (g *Generator) getStructEncoderName(t reflect.Type) string { +func (g *Generator) getEncoderName(t reflect.Type) string { return g.functionName("encode", t) } @@ -79,7 +79,7 @@ func parseFieldTags(f reflect.StructField) fieldTags { return ret } -// genTypeEncoder generates code that encodes in of type t into the writer. +// genTypeEncoder generates code that encodes in of type t into the writer, but uses marshaler interface if implemented by t. func (g *Generator) genTypeEncoder(t reflect.Type, in string, tags fieldTags, indent int) error { ws := strings.Repeat(" ", indent) @@ -95,6 +95,14 @@ func (g *Generator) genTypeEncoder(t reflect.Type, in string, tags fieldTags, in return nil } + err := g.genTypeEncoderNoCheck(t, in, tags, indent) + return err +} + +// genTypeEncoderNoCheck generates code that encodes in of type t into the writer. +func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldTags, indent int) error { + ws := strings.Repeat(" ", indent) + // Check whether type is primitive, needs to be done after interface check. if enc := primitiveStringEncoders[t.Kind()]; enc != "" && tags.asString { fmt.Fprintf(g.out, ws+enc+"\n", in) @@ -122,7 +130,7 @@ func (g *Generator) genTypeEncoder(t reflect.Type, in string, tags fieldTags, in fmt.Fprintln(g.out, ws+"out.RawByte(']')") case reflect.Struct: - enc := g.getStructEncoderName(t) + enc := g.getEncoderName(t) g.addType(t) fmt.Fprintln(g.out, ws+enc+"(out, "+in+")") @@ -224,12 +232,38 @@ func (g *Generator) genStructFieldEncoder(t reflect.Type, f reflect.StructField) return nil } -func (g *Generator) genStructEncoder(t reflect.Type) error { - if t.Kind() != reflect.Struct { - return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct type", t) +func (g *Generator) genEncoder(t reflect.Type) error { + switch t.Kind() { + case reflect.Slice: + return g.genSliceEncoder(t) + default: + return g.genStructEncoder(t) + } +} + +func (g *Generator) genSliceEncoder(t reflect.Type) error { + if t.Kind() != reflect.Slice { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a slice type", t) } - fname := g.getStructEncoderName(t) + fname := g.getEncoderName(t) + typ := g.getType(t) + + fmt.Fprintln(g.out, "func "+fname+"(out *jwriter.Writer, in "+typ+") {") + err := g.genTypeEncoderNoCheck(t, "in", fieldTags{}, 1) + if err != nil { + return err + } + fmt.Fprintln(g.out, "}") + return nil +} + +func (g *Generator) genStructEncoder(t reflect.Type) error { + if t.Kind() != reflect.Struct { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct type") + } + + fname := g.getEncoderName(t) typ := g.getType(t) fmt.Fprintln(g.out, "func "+fname+"(out *jwriter.Writer, in "+typ+") {") @@ -254,11 +288,11 @@ func (g *Generator) genStructEncoder(t reflect.Type) error { } func (g *Generator) genStructMarshaller(t reflect.Type) error { - if t.Kind() != reflect.Struct { - return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct type", t) + if t.Kind() != reflect.Struct && t.Kind() != reflect.Slice { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct/slice type", t) } - fname := g.getStructEncoderName(t) + fname := g.getEncoderName(t) typ := g.getType(t) if !g.noStdMarshalers { diff --git a/gen/generator.go b/gen/generator.go index fe8af99..fe7d4e2 100644 --- a/gen/generator.go +++ b/gen/generator.go @@ -193,10 +193,10 @@ func (g *Generator) Run(out io.Writer) error { g.typesUnseen = g.typesUnseen[:len(g.typesUnseen)-1] g.typesSeen[t] = true - if err := g.genStructDecoder(t); err != nil { + if err := g.genDecoder(t); err != nil { return err } - if err := g.genStructEncoder(t); err != nil { + if err := g.genEncoder(t); err != nil { return err } @@ -245,13 +245,15 @@ func (g *Generator) pkgAlias(pkgPath string) string { // getType return the textual type name of given type that can be used in generated code. func (g *Generator) getType(t reflect.Type) string { - switch t.Kind() { - case reflect.Ptr: - return "*" + g.getType(t.Elem()) - case reflect.Slice: - return "[]" + g.getType(t.Elem()) - case reflect.Map: - return "map[" + g.getType(t.Key()) + "]" + g.getType(t.Elem()) + if t.Name() == "" { + switch t.Kind() { + case reflect.Ptr: + return "*" + g.getType(t.Elem()) + case reflect.Slice: + return "[]" + g.getType(t.Elem()) + case reflect.Map: + return "map[" + g.getType(t.Key()) + "]" + g.getType(t.Elem()) + } } if t.Name() == "" || t.PkgPath() == "" { diff --git a/jwriter/writer.go b/jwriter/writer.go index 85fa440..907675f 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -4,15 +4,15 @@ package jwriter import ( "io" "strconv" + "unicode/utf8" "github.com/mailru/easyjson/buffer" ) // Writer is a JSON writer. type Writer struct { - EscapeLtGt bool - Error error - Buffer buffer.Buffer + Error error + Buffer buffer.Buffer } // Size returns the size of the data that was written out. @@ -198,10 +198,7 @@ func (w *Writer) Bool(v bool) { } } -func hex(c byte) byte { - const chars = "0123456789abcdef" - return chars[c&0xf] -} +const chars = "0123456789abcdef" func (w *Writer) String(s string) { w.Buffer.AppendByte('"') @@ -211,41 +208,65 @@ func (w *Writer) String(s string) { p := 0 // last non-escape symbol - for i := 0; i < len(s); i++ { - c := s[i] - var escape byte - switch c { - case '\t': - escape = 't' - case '\r': - escape = 'r' - case '\n': - escape = 'n' - case '\\': - escape = '\\' - case '"': - escape = '"' - case '<', '>': - if !w.EscapeLtGt { - continue + for i := 0; i < len(s); { + // single-with character + if c := s[i]; c < utf8.RuneSelf { + var escape byte + switch c { + case '\t': + escape = 't' + case '\r': + escape = 'r' + case '\n': + escape = 'n' + case '\\': + escape = '\\' + case '"': + escape = '"' + case '<', '>': + // do nothing + default: + if c >= 0x20 { + // no escaping is required + i++ + continue + } } - default: - if c >= 0x20 { - // no escaping is required - continue + if escape != 0 { + w.Buffer.AppendString(s[p:i]) + w.Buffer.AppendByte('\\') + w.Buffer.AppendByte(escape) + } else { + w.Buffer.AppendString(s[p:i]) + w.Buffer.AppendString(`\u00`) + w.Buffer.AppendByte(chars[c>>4]) + w.Buffer.AppendByte(chars[c&0xf]) } + i++ + p = i + continue } - if escape != 0 { + + // broken utf + runeValue, runeWidth := utf8.DecodeRuneInString(s[i:]) + if runeValue == utf8.RuneError && runeWidth == 1 { w.Buffer.AppendString(s[p:i]) - w.Buffer.AppendByte('\\') - w.Buffer.AppendByte(escape) - } else { - w.Buffer.AppendString(s[p:i]) - w.Buffer.AppendString(`\u00`) - w.Buffer.AppendByte(hex(c >> 4)) - w.Buffer.AppendByte(hex(c)) + w.Buffer.AppendString(`\ufffd`) + i++ + p = i + continue } - p = i + 1 + + // jsonp stuff - tab separator and line separator + if runeValue == '\u2028' || runeValue == '\u2029' { + w.Buffer.AppendString(s[p:i]) + w.Buffer.AppendString(`\u202`) + w.Buffer.AppendByte(chars[runeValue&0xf]) + i += runeWidth + p = i + continue + } + i += runeWidth } w.Buffer.AppendString(s[p:]) w.Buffer.AppendByte('"') diff --git a/tests/basic_test.go b/tests/basic_test.go index ce291b9..0450c67 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -32,6 +32,7 @@ var testCases = []struct { {&excludedFieldValue, excludedFieldString}, {&mapsValue, mapsString}, {&deepNestValue, deepNestString}, + {&IntsValue, IntsString}, {&camelCasesFunctionsValue, camelCasesFunctionsString}, } @@ -106,24 +107,28 @@ func TestParseNull(t *testing.T) { } } -var testCasesEncodeLtGt = []struct { - Writer *jwriter.Writer - Encoded string +var testSpecialCases = []struct { + EncodedString string + Value string }{ - {&jwriter.Writer{ - EscapeLtGt: false, - }, encodeLtGtFalseWantString}, - {&jwriter.Writer{ - EscapeLtGt: true, - }, encodeLtGtTrueWantString}, + {`"Username \u003cuser@example.com\u003e"`, `Username `}, + {`"Username\ufffd"`, "Username\xc5"}, + {`"тестzтест"`, "тестzтест"}, + {`"тест\ufffdтест"`, "тест\xc5тест"}, + {`"绿茶"`, "绿茶"}, + {`"绿\ufffd茶"`, "绿\xc5茶"}, + {`"тест\u2028"`, "тест\xE2\x80\xA8"}, + {`"\\\r\n\t\""`, "\\\r\n\t\""}, + {`"ü"`, "ü"}, } -func TestEncodeLtGt(t *testing.T) { - for i, test := range testCasesEncodeLtGt { - test.Writer.String(encodeLtGtString) - got := string(test.Writer.Buffer.BuildBytes()) - if got != test.Encoded { - t.Errorf("[%d] Encoded() = %+v; want %+v", i, got, test.Encoded) +func TestSpecialCases(t *testing.T) { + for i, test := range testSpecialCases { + w := jwriter.Writer{} + w.String(test.Value) + got := string(w.Buffer.BuildBytes()) + if got != test.EncodedString { + t.Errorf("[%d] Encoded() = %+v; want %+v", i, got, test.EncodedString) } } } diff --git a/tests/data.go b/tests/data.go index 1c4ae78..faa3093 100644 --- a/tests/data.go +++ b/tests/data.go @@ -246,6 +246,8 @@ type Structs struct { AnonymousSlice []struct{ V int } AnonymousPtrSlice []*struct{ V int } + Slice []string + unexported bool } @@ -282,6 +284,8 @@ var structsValue = Structs{ AnonymousSlice: []struct{ V int }{{1}, {2}}, AnonymousPtrSlice: []*struct{ V int }{{3}, {4}}, + + Slice: []string{"test5", "test6"}, } var structsString = "{" + @@ -306,6 +310,8 @@ var structsString = "{" + `"AnonymousSlice":[{"V":1},{"V":2}],` + `"AnonymousPtrSlice":[{"V":3},{"V":4}],` + + `"Slice":["test5","test6"],` + + // Embedded fields go last. `"V":"subp",` + `"Value":"test"` + @@ -436,46 +442,104 @@ var mapsString = `{` + `"CustomMap":{"c":"d"}` + `}` +type NamedSlice []Str +type NamedMap map[Str]Str + type DeepNest struct { - SliceMap map[Str][]Str - SliceMap1 map[Str][]Str - MapSlice []map[Str]Str + SliceMap map[Str][]Str + SliceMap1 map[Str][]Str + NamedSliceMap map[Str]NamedSlice + NamedMapMap map[Str]NamedMap + MapSlice []map[Str]Str + NamedSliceSlice []NamedSlice + NamedMapSlice []NamedMap + NamedStringSlice []NamedString } var deepNestValue = DeepNest{ SliceMap: map[Str][]Str{ - "testSliceMap1": []Str{ + "testSliceMap": []Str{ "0", "1", }, }, SliceMap1: map[Str][]Str{ - "testSliceMap2": nil, + "testSliceMap1": nil, + }, + NamedSliceMap: map[Str]NamedSlice{ + "testNamedSliceMap": NamedSlice{ + "2", + "3", + }, + }, + NamedMapMap: map[Str]NamedMap{ + "testNamedMapMap": NamedMap{ + "key1": "value1", + }, }, MapSlice: []map[Str]Str{ map[Str]Str{ - "testMapSlice1": "someValue", + "testMapSlice": "someValue", }, }, + NamedSliceSlice: []NamedSlice{ + NamedSlice{ + "someValue1", + "someValue2", + }, + NamedSlice{ + "someValue3", + "someValue4", + }, + }, + NamedMapSlice: []NamedMap{ + NamedMap{ + "key2": "value2", + }, + NamedMap{ + "key3": "value3", + }, + }, + NamedStringSlice: []NamedString{ + "value4", "value5", + }, } var deepNestString = `{` + `"SliceMap":{` + - `"testSliceMap1":["0","1"]` + + `"testSliceMap":["0","1"]` + `},` + `"SliceMap1":{` + - `"testSliceMap2":[]` + + `"testSliceMap1":[]` + + `},` + + `"NamedSliceMap":{` + + `"testNamedSliceMap":["2","3"]` + + `},` + + `"NamedMapMap":{` + + `"testNamedMapMap":{"key1":"value1"}` + `},` + `"MapSlice":[` + - `{"testMapSlice1":"someValue"}` + - `]` + + `{"testMapSlice":"someValue"}` + + `],` + + `"NamedSliceSlice":[` + + `["someValue1","someValue2"],` + + `["someValue3","someValue4"]` + + `],` + + `"NamedMapSlice":[` + + `{"key2":"value2"},` + + `{"key3":"value3"}` + + `],` + + `"NamedStringSlice":["value4","value5"]` + `}` +//easyjson:json +type Ints []int + +var IntsValue = Ints{1, 2, 3, 4, 5} + +var IntsString = `[1,2,3,4,5]` + type RequiredOptionalStruct struct { FirstName string `json:"first_name,required"` Lastname string `json:"last_name"` } - -var encodeLtGtString = `Username ` -var encodeLtGtFalseWantString = `"Username "` -var encodeLtGtTrueWantString = `"Username \u003cuser@example.com\u003e"`