From bc1638cf78d077deaae64dbc40b40f0a8cc0ae09 Mon Sep 17 00:00:00 2001 From: "Nicolas S. Dade" Date: Thu, 22 Sep 2016 01:13:38 -0700 Subject: [PATCH 1/4] Add support for array types Arrays are useful to decode into to keep the allocations down and thus the gc pressure low. The code allows the json to contain truncated arrays without raising an error, same as standard encoding/json. However excess elements in the json is an error. --- gen/decoder.go | 48 ++++++++++++++++++++++++++++++++++++++------- gen/encoder.go | 36 ++++++++++++++++++++++++++-------- gen/generator.go | 3 +++ tests/basic_test.go | 1 + tests/data.go | 21 ++++++++++++++++++++ 5 files changed, 94 insertions(+), 15 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index d52b504..424195d 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -123,6 +123,40 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+"}") } + case reflect.Array: + tmpVar := g.uniqueVarName() + iterVar := g.uniqueVarName() + elem := t.Elem() + + if elem.Kind() == reflect.Uint8 { + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" copy("+out+"[:], in.Bytes())") + fmt.Fprintln(g.out, ws+"}") + + } else { + + length := t.Len() + + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" in.Delim('[')") + fmt.Fprintln(g.out, ws+" "+iterVar+" := 0") + fmt.Fprintln(g.out, ws+" for !in.IsDelim(']') && "+iterVar+" < "+fmt.Sprint(length)+" {") + fmt.Fprintln(g.out, ws+" var "+tmpVar+" "+g.getType(elem)) + + g.genTypeDecoder(elem, tmpVar, tags, indent+2) + + fmt.Fprintln(g.out, ws+" "+out+"["+iterVar+"] = "+tmpVar) + fmt.Fprintln(g.out, ws+" "+iterVar+"++") + fmt.Fprintln(g.out, ws+" in.WantComma()") + fmt.Fprintln(g.out, ws+" }") + fmt.Fprintln(g.out, ws+" in.Delim(']')") + fmt.Fprintln(g.out, ws+"}") + } + case reflect.Struct: dec := g.getDecoderName(t) g.addType(t) @@ -287,16 +321,16 @@ func getStructFields(t reflect.Type) ([]reflect.StructField, error) { func (g *Generator) genDecoder(t reflect.Type) error { switch t.Kind() { - case reflect.Slice: - return g.genSliceDecoder(t) + case reflect.Slice, reflect.Array: + return g.genSliceArrayDecoder(t) default: return g.genStructDecoder(t) } } -func (g *Generator) genSliceDecoder(t reflect.Type) error { - if t.Kind() != reflect.Slice { - return fmt.Errorf("cannot generate encoder/decoder for %v, not a slice type", t) +func (g *Generator) genSliceArrayDecoder(t reflect.Type) error { + if t.Kind() != reflect.Slice && t.Kind() != reflect.Array { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a slice or array type", t) } fname := g.getDecoderName(t) @@ -378,8 +412,8 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { } func (g *Generator) genStructUnmarshaller(t reflect.Type) error { - if t.Kind() != reflect.Struct && t.Kind() != reflect.Slice { - return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct/slice type", t) + if t.Kind() != reflect.Struct && t.Kind() != reflect.Slice && t.Kind() != reflect.Array { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct/slice/array type", t) } fname := g.getDecoderName(t) diff --git a/gen/encoder.go b/gen/encoder.go index 4bac8e8..4765317 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -137,6 +137,26 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT fmt.Fprintln(g.out, ws+"}") } + case reflect.Array: + elem := t.Elem() + iVar := g.uniqueVarName() + vVar := g.uniqueVarName() + + if t.Elem().Kind() == reflect.Uint8 { + fmt.Fprintln(g.out, ws+"out.Base64Bytes("+in+"[:])") + } else { + fmt.Fprintln(g.out, ws+"out.RawByte('[')") + fmt.Fprintln(g.out, ws+"for "+iVar+", "+vVar+" := range "+in+" {") + fmt.Fprintln(g.out, ws+" if "+iVar+" > 0 {") + fmt.Fprintln(g.out, ws+" out.RawByte(',')") + fmt.Fprintln(g.out, ws+" }") + + g.genTypeEncoder(elem, vVar, tags, indent+1) + + fmt.Fprintln(g.out, ws+"}") + fmt.Fprintln(g.out, ws+"out.RawByte(']')") + } + case reflect.Struct: enc := g.getEncoderName(t) g.addType(t) @@ -195,7 +215,7 @@ func (g *Generator) notEmptyCheck(t reflect.Type, v string) string { } switch t.Kind() { - case reflect.Slice, reflect.Map: + case reflect.Slice, reflect.Map: // note: Array types don't have a useful empty value return "len(" + v + ") != 0" case reflect.Interface, reflect.Ptr: return v + " != nil" @@ -242,16 +262,16 @@ func (g *Generator) genStructFieldEncoder(t reflect.Type, f reflect.StructField) func (g *Generator) genEncoder(t reflect.Type) error { switch t.Kind() { - case reflect.Slice: - return g.genSliceEncoder(t) + case reflect.Slice, reflect.Array: + return g.genSliceArrayEncoder(t) default: return g.genStructEncoder(t) } } -func (g *Generator) genSliceEncoder(t reflect.Type) error { - if t.Kind() != reflect.Slice { - return fmt.Errorf("cannot generate encoder/decoder for %v, not a slice type", t) +func (g *Generator) genSliceArrayEncoder(t reflect.Type) error { + if t.Kind() != reflect.Slice && t.Kind() != reflect.Array { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a slice or array type", t) } fname := g.getEncoderName(t) @@ -296,8 +316,8 @@ func (g *Generator) genStructEncoder(t reflect.Type) error { } func (g *Generator) genStructMarshaller(t reflect.Type) error { - if t.Kind() != reflect.Struct && t.Kind() != reflect.Slice { - return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct/slice type", t) + if t.Kind() != reflect.Struct && t.Kind() != reflect.Slice && t.Kind() != reflect.Array { + return fmt.Errorf("cannot generate encoder/decoder for %v, not a struct/slice/array type", t) } fname := g.getEncoderName(t) diff --git a/gen/generator.go b/gen/generator.go index 2a3ef9c..1c527d5 100644 --- a/gen/generator.go +++ b/gen/generator.go @@ -8,6 +8,7 @@ import ( "path" "reflect" "sort" + "strconv" "strings" "unicode" ) @@ -233,6 +234,8 @@ func (g *Generator) getType(t reflect.Type) string { return "*" + g.getType(t.Elem()) case reflect.Slice: return "[]" + g.getType(t.Elem()) + case reflect.Array: + return "[" + strconv.Itoa(t.Len()) + "]" + g.getType(t.Elem()) case reflect.Map: return "map[" + g.getType(t.Key()) + "]" + g.getType(t.Elem()) } diff --git a/tests/basic_test.go b/tests/basic_test.go index a15253a..7e454ae 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -31,6 +31,7 @@ var testCases = []struct { {&unexportedStructValue, unexportedStructString}, {&excludedFieldValue, excludedFieldString}, {&sliceValue, sliceString}, + {&arrayValue, arrayString}, {&mapsValue, mapsString}, {&deepNestValue, deepNestString}, {&IntsValue, IntsString}, diff --git a/tests/data.go b/tests/data.go index 06298c2..1694a55 100644 --- a/tests/data.go +++ b/tests/data.go @@ -445,6 +445,27 @@ var sliceString = `{` + `"NilIntSlice":null` + `}` +type Arrays struct { + ByteArray [3]byte + EmptyByteArray [0]byte + IntArray [5]int + EmptyIntArray [0]int +} + +var arrayValue = Arrays{ + ByteArray: [3]byte{'a', 'b', 'c'}, + EmptyByteArray: [0]byte{}, + IntArray: [5]int{1, 2, 3, 4, 5}, + EmptyIntArray: [0]int{}, +} + +var arrayString = `{` + + `"ByteArray":"YWJj",` + + `"EmptyByteArray":"",` + + `"IntArray":[1,2,3,4,5],` + + `"EmptyIntArray":[]` + + `}` + type Str string type Maps struct { From 0f22c066ac70e8f14525192acf03bb10d46037f0 Mon Sep 17 00:00:00 2001 From: "Nicolas S. Dade" Date: Fri, 23 Sep 2016 01:35:27 -0700 Subject: [PATCH 2/4] marshal and unmarshal in place in arrays that avoids a copy of each item. --- gen/decoder.go | 5 +---- gen/encoder.go | 5 ++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index 424195d..21fea2f 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -124,7 +124,6 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field } case reflect.Array: - tmpVar := g.uniqueVarName() iterVar := g.uniqueVarName() elem := t.Elem() @@ -145,11 +144,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" in.Delim('[')") fmt.Fprintln(g.out, ws+" "+iterVar+" := 0") fmt.Fprintln(g.out, ws+" for !in.IsDelim(']') && "+iterVar+" < "+fmt.Sprint(length)+" {") - fmt.Fprintln(g.out, ws+" var "+tmpVar+" "+g.getType(elem)) - g.genTypeDecoder(elem, tmpVar, tags, indent+2) + g.genTypeDecoder(elem, out+"["+iterVar+"]", tags, indent+2) - fmt.Fprintln(g.out, ws+" "+out+"["+iterVar+"] = "+tmpVar) fmt.Fprintln(g.out, ws+" "+iterVar+"++") fmt.Fprintln(g.out, ws+" in.WantComma()") fmt.Fprintln(g.out, ws+" }") diff --git a/gen/encoder.go b/gen/encoder.go index 4765317..8775b0c 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -140,18 +140,17 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT case reflect.Array: elem := t.Elem() iVar := g.uniqueVarName() - vVar := g.uniqueVarName() if t.Elem().Kind() == reflect.Uint8 { fmt.Fprintln(g.out, ws+"out.Base64Bytes("+in+"[:])") } else { fmt.Fprintln(g.out, ws+"out.RawByte('[')") - fmt.Fprintln(g.out, ws+"for "+iVar+", "+vVar+" := range "+in+" {") + fmt.Fprintln(g.out, ws+"for "+iVar+" := range "+in+" {") fmt.Fprintln(g.out, ws+" if "+iVar+" > 0 {") fmt.Fprintln(g.out, ws+" out.RawByte(',')") fmt.Fprintln(g.out, ws+" }") - g.genTypeEncoder(elem, vVar, tags, indent+1) + g.genTypeEncoder(elem, in+"["+iVar+"]", tags, indent+1) fmt.Fprintln(g.out, ws+"}") fmt.Fprintln(g.out, ws+"out.RawByte(']')") From 32cc087149a528b13873298390487677f7522bd0 Mon Sep 17 00:00:00 2001 From: "Nicolas S. Dade" Date: Fri, 23 Sep 2016 16:53:34 -0700 Subject: [PATCH 3/4] match behavior of encoding/json when unmarshaling into an array excess elements are ignored silently --- gen/decoder.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index 21fea2f..0688e5d 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -143,11 +143,15 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+"} else {") fmt.Fprintln(g.out, ws+" in.Delim('[')") fmt.Fprintln(g.out, ws+" "+iterVar+" := 0") - fmt.Fprintln(g.out, ws+" for !in.IsDelim(']') && "+iterVar+" < "+fmt.Sprint(length)+" {") + fmt.Fprintln(g.out, ws+" for !in.IsDelim(']') {") + fmt.Fprintln(g.out, ws+" if "+iterVar+" < "+fmt.Sprint(length)+" {") - g.genTypeDecoder(elem, out+"["+iterVar+"]", tags, indent+2) + g.genTypeDecoder(elem, out+"["+iterVar+"]", tags, indent+3) - fmt.Fprintln(g.out, ws+" "+iterVar+"++") + fmt.Fprintln(g.out, ws+" "+iterVar+"++") + fmt.Fprintln(g.out, ws+" } else {") + fmt.Fprintln(g.out, ws+" in.SkipRecursive()") + fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+" in.WantComma()") fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+" in.Delim(']')") From 4b3499f325f9466e2ec3eec588be333912f5579a Mon Sep 17 00:00:00 2001 From: "Nicolas S. Dade" Date: Fri, 23 Sep 2016 17:24:21 -0700 Subject: [PATCH 4/4] move comment about empty-value of Arrays to the code path the array case takes --- gen/encoder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gen/encoder.go b/gen/encoder.go index 8775b0c..66e2f5b 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -214,7 +214,7 @@ func (g *Generator) notEmptyCheck(t reflect.Type, v string) string { } switch t.Kind() { - case reflect.Slice, reflect.Map: // note: Array types don't have a useful empty value + case reflect.Slice, reflect.Map: return "len(" + v + ") != 0" case reflect.Interface, reflect.Ptr: return v + " != nil" @@ -229,6 +229,7 @@ func (g *Generator) notEmptyCheck(t reflect.Type, v string) string { return v + " != 0" default: + // note: Array types don't have a useful empty value return "true" } }