From b0ee67a33d9178466bb099d9dedffe846ac59ae3 Mon Sep 17 00:00:00 2001 From: jz Date: Thu, 18 Aug 2016 13:23:52 -0700 Subject: [PATCH] Follow encoding/json spec when dealing with slices - nil slices are encoded as JSON nulls - empty slices are encoded as empty JSON arrays - byte slices are encoded as base 64 encoded JSON strings Demo: https://play.golang.org/p/Z_YE5PHS3g --- gen/decoder.go | 54 ++++++++++++++++++++++++++++---------------- gen/encoder.go | 24 +++++++++++++------- jlexer/lexer.go | 23 +++++++++++++++++++ jlexer/lexer_test.go | 29 ++++++++++++++++++++++++ jwriter/writer.go | 14 ++++++++++++ tests/basic_test.go | 1 + tests/data.go | 42 ++++++++++++++++++++++++++++++---- 7 files changed, 156 insertions(+), 31 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index cc4756f..d52b504 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -86,27 +86,43 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field tmpVar := g.uniqueVarName() elem := t.Elem() - capacity := minSliceBytes / elem.Size() - if capacity == 0 { - capacity = 1 + if elem.Kind() == reflect.Uint8 { + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+" "+out+" = nil") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" "+out+" = in.Bytes()") + fmt.Fprintln(g.out, ws+"}") + + } else { + + capacity := minSliceBytes / elem.Size() + if capacity == 0 { + capacity = 1 + } + + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+" "+out+" = nil") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" in.Delim('[')") + fmt.Fprintln(g.out, ws+" if !in.IsDelim(']') {") + fmt.Fprintln(g.out, ws+" "+out+" = make("+g.getType(t)+", 0, "+fmt.Sprint(capacity)+")") + fmt.Fprintln(g.out, ws+" } else {") + fmt.Fprintln(g.out, ws+" "+out+" = "+g.getType(t)+"{}") + fmt.Fprintln(g.out, ws+" }") + fmt.Fprintln(g.out, ws+" for !in.IsDelim(']') {") + fmt.Fprintln(g.out, ws+" var "+tmpVar+" "+g.getType(elem)) + + g.genTypeDecoder(elem, tmpVar, tags, indent+2) + + fmt.Fprintln(g.out, ws+" "+out+" = append("+out+", "+tmpVar+")") + fmt.Fprintln(g.out, ws+" in.WantComma()") + fmt.Fprintln(g.out, ws+" }") + fmt.Fprintln(g.out, ws+" in.Delim(']')") + fmt.Fprintln(g.out, ws+"}") } - fmt.Fprintln(g.out, ws+"in.Delim('[')") - fmt.Fprintln(g.out, ws+"if !in.IsDelim(']') {") - fmt.Fprintln(g.out, ws+" "+out+" = make("+g.getType(t)+", 0, "+fmt.Sprint(capacity)+")") - fmt.Fprintln(g.out, ws+"} else {") - fmt.Fprintln(g.out, ws+" "+out+" = nil") - fmt.Fprintln(g.out, ws+"}") - fmt.Fprintln(g.out, ws+"for !in.IsDelim(']') {") - fmt.Fprintln(g.out, ws+" var "+tmpVar+" "+g.getType(elem)) - - g.genTypeDecoder(elem, tmpVar, tags, indent+1) - - fmt.Fprintln(g.out, ws+" "+out+" = append("+out+", "+tmpVar+")") - fmt.Fprintln(g.out, ws+" in.WantComma()") - fmt.Fprintln(g.out, ws+"}") - fmt.Fprintln(g.out, ws+"in.Delim(']')") - case reflect.Struct: dec := g.getDecoderName(t) g.addType(t) diff --git a/gen/encoder.go b/gen/encoder.go index e5b6ab3..4bac8e8 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -118,16 +118,24 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT iVar := g.uniqueVarName() vVar := g.uniqueVarName() - fmt.Fprintln(g.out, ws+"out.RawByte('[')") - fmt.Fprintln(g.out, ws+"for "+iVar+", "+vVar+" := range "+in+" {") - fmt.Fprintln(g.out, ws+" if "+iVar+" > 0 {") - fmt.Fprintln(g.out, ws+" out.RawByte(',')") - fmt.Fprintln(g.out, ws+" }") + if t.Elem().Kind() == reflect.Uint8 { + fmt.Fprintln(g.out, ws+"out.Base64Bytes("+in+")") + } else { + fmt.Fprintln(g.out, ws+"if "+in+" == nil {") + fmt.Fprintln(g.out, ws+` out.RawString("null")`) + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" out.RawByte('[')") + fmt.Fprintln(g.out, ws+" for "+iVar+", "+vVar+" := range "+in+" {") + fmt.Fprintln(g.out, ws+" if "+iVar+" > 0 {") + fmt.Fprintln(g.out, ws+" out.RawByte(',')") + fmt.Fprintln(g.out, ws+" }") - g.genTypeEncoder(elem, vVar, tags, indent+1) + g.genTypeEncoder(elem, vVar, tags, indent+2) - fmt.Fprintln(g.out, ws+"}") - fmt.Fprintln(g.out, ws+"out.RawByte(']')") + fmt.Fprintln(g.out, ws+" }") + fmt.Fprintln(g.out, ws+" out.RawByte(']')") + fmt.Fprintln(g.out, ws+"}") + } case reflect.Struct: enc := g.getEncoderName(t) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index d700c0a..36cd820 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -5,6 +5,7 @@ package jlexer import ( + "encoding/base64" "fmt" "io" "reflect" @@ -560,6 +561,28 @@ func (r *Lexer) String() string { return ret } +// Bytes reads a string literal and base64 decodes it into a byte slice. +func (r *Lexer) Bytes() []byte { + if r.token.kind == tokenUndef && r.Ok() { + r.fetchToken() + } + if !r.Ok() || r.token.kind != tokenString { + r.errInvalidToken("string") + return nil + } + ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue))) + len, err := base64.StdEncoding.Decode(ret, r.token.byteValue) + if err != nil { + r.err = &LexerError{ + Reason: err.Error(), + } + return nil + } + + r.consume() + return ret[:len] +} + // Bool reads a true or false boolean keyword. func (r *Lexer) Bool() bool { if r.token.kind == tokenUndef && r.Ok() { diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index 900ab61..b8e70f0 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -1,6 +1,7 @@ package jlexer import ( + "bytes" "reflect" "testing" ) @@ -39,6 +40,34 @@ func TestString(t *testing.T) { } } +func TestBytes(t *testing.T) { + for i, test := range []struct { + toParse string + want string + wantError bool + }{ + {toParse: `"c2ltcGxlIHN0cmluZw=="`, want: "simple string"}, + {toParse: " \r\r\n\t " + `"dGVzdA=="`, want: "test"}, + + {toParse: `5`, wantError: true}, // not a JSON string + {toParse: `"foobar"`, wantError: true}, // not base64 encoded + {toParse: `"c2ltcGxlIHN0cmluZw="`, wantError: true}, // invalid base64 padding + } { + l := Lexer{Data: []byte(test.toParse)} + + got := l.Bytes() + if bytes.Compare(got, []byte(test.want)) != 0 { + t.Errorf("[%d, %q] Bytes() = %v; want: %v", i, test.toParse, got, []byte(test.want)) + } + err := l.Error() + if err != nil && !test.wantError { + t.Errorf("[%d, %q] Bytes() error: %v", i, test.toParse, err) + } else if err == nil && test.wantError { + t.Errorf("[%d, %q] Bytes() ok; want error", i, test.toParse) + } + } +} + func TestNumber(t *testing.T) { for i, test := range []struct { toParse string diff --git a/jwriter/writer.go b/jwriter/writer.go index d24176d..acd670f 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -2,6 +2,7 @@ package jwriter import ( + "encoding/base64" "io" "strconv" "unicode/utf8" @@ -59,6 +60,19 @@ func (w *Writer) Raw(data []byte, err error) { } } +// Base64Bytes appends data to the buffer after base64 encoding it +func (w *Writer) Base64Bytes(data []byte) { + if data == nil { + w.Buffer.AppendString("null") + return + } + w.Buffer.AppendByte('"') + dst := make([]byte, base64.StdEncoding.EncodedLen(len(data))) + base64.StdEncoding.Encode(dst, data) + w.Buffer.AppendBytes(dst) + w.Buffer.AppendByte('"') +} + func (w *Writer) Uint8(n uint8) { w.Buffer.EnsureSpace(3) w.Buffer.Buf = strconv.AppendUint(w.Buffer.Buf, uint64(n), 10) diff --git a/tests/basic_test.go b/tests/basic_test.go index d2eac28..a15253a 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -30,6 +30,7 @@ var testCases = []struct { {&stdMarshalerValue, stdMarshalerString}, {&unexportedStructValue, unexportedStructString}, {&excludedFieldValue, excludedFieldString}, + {&sliceValue, sliceString}, {&mapsValue, mapsString}, {&deepNestValue, deepNestString}, {&IntsValue, IntsString}, diff --git a/tests/data.go b/tests/data.go index faa3093..06298c2 100644 --- a/tests/data.go +++ b/tests/data.go @@ -296,10 +296,10 @@ var structsString = "{" + `"SubNil":null,` + `"SubSlice":[{"Value":"s1","Value2":""},{"Value":"s2","Value2":""}],` + - `"SubSliceNil":[],` + + `"SubSliceNil":null,` + `"SubPtrSlice":[{"Value":"p1","Value2":""},{"Value":"p2","Value2":""}],` + - `"SubPtrSliceNil":[],` + + `"SubPtrSliceNil":null,` + `"SubA1":{"Value":"test3","Value2":"v3"},` + `"SubA2":{"Value":"test4","Value2":"v4"},` + @@ -418,6 +418,33 @@ var excludedFieldValue = ExcludedField{ } var excludedFieldString = `{"process":true}` +type Slices struct { + ByteSlice []byte + EmptyByteSlice []byte + NilByteSlice []byte + IntSlice []int + EmptyIntSlice []int + NilIntSlice []int +} + +var sliceValue = Slices{ + ByteSlice: []byte("abc"), + EmptyByteSlice: []byte{}, + NilByteSlice: []byte(nil), + IntSlice: []int{1, 2, 3, 4, 5}, + EmptyIntSlice: []int{}, + NilIntSlice: []int(nil), +} + +var sliceString = `{` + + `"ByteSlice":"YWJj",` + + `"EmptyByteSlice":"",` + + `"NilByteSlice":null,` + + `"IntSlice":[1,2,3,4,5],` + + `"EmptyIntSlice":[],` + + `"NilIntSlice":null` + + `}` + type Str string type Maps struct { @@ -448,6 +475,7 @@ type NamedMap map[Str]Str type DeepNest struct { SliceMap map[Str][]Str SliceMap1 map[Str][]Str + SliceMap2 map[Str][]Str NamedSliceMap map[Str]NamedSlice NamedMapMap map[Str]NamedMap MapSlice []map[Str]Str @@ -464,7 +492,10 @@ var deepNestValue = DeepNest{ }, }, SliceMap1: map[Str][]Str{ - "testSliceMap1": nil, + "testSliceMap1": []Str(nil), + }, + SliceMap2: map[Str][]Str{ + "testSliceMap2": []Str{}, }, NamedSliceMap: map[Str]NamedSlice{ "testNamedSliceMap": NamedSlice{ @@ -510,7 +541,10 @@ var deepNestString = `{` + `"testSliceMap":["0","1"]` + `},` + `"SliceMap1":{` + - `"testSliceMap1":[]` + + `"testSliceMap1":null` + + `},` + + `"SliceMap2":{` + + `"testSliceMap2":[]` + `},` + `"NamedSliceMap":{` + `"testNamedSliceMap":["2","3"]` +