Follow encoding/json spec when dealing with slices

- nil slices are encoded as JSON nulls
- empty slices are encoded as empty JSON arrays
- byte slices are encoded as base 64 encoded JSON strings

Demo: https://play.golang.org/p/Z_YE5PHS3g

gen/decoder.go

@@ -86,27 +86,43 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
 		tmpVar := g.uniqueVarName()
 		elem := t.Elem()
 
-		capacity := minSliceBytes / elem.Size()
-		if capacity == 0 {
-			capacity = 1
+		if elem.Kind() == reflect.Uint8 {
+			fmt.Fprintln(g.out, ws+"if in.IsNull() {")
+			fmt.Fprintln(g.out, ws+"  in.Skip()")
+			fmt.Fprintln(g.out, ws+"  "+out+" = nil")
+			fmt.Fprintln(g.out, ws+"} else {")
+			fmt.Fprintln(g.out, ws+"  "+out+" = in.Bytes()")
+			fmt.Fprintln(g.out, ws+"}")
+
+		} else {
+
+			capacity := minSliceBytes / elem.Size()
+			if capacity == 0 {
+				capacity = 1
+			}
+
+			fmt.Fprintln(g.out, ws+"if in.IsNull() {")
+			fmt.Fprintln(g.out, ws+"  in.Skip()")
+			fmt.Fprintln(g.out, ws+"  "+out+" = nil")
+			fmt.Fprintln(g.out, ws+"} else {")
+			fmt.Fprintln(g.out, ws+"  in.Delim('[')")
+			fmt.Fprintln(g.out, ws+"  if !in.IsDelim(']') {")
+			fmt.Fprintln(g.out, ws+"    "+out+" = make("+g.getType(t)+", 0, "+fmt.Sprint(capacity)+")")
+			fmt.Fprintln(g.out, ws+"  } else {")
+			fmt.Fprintln(g.out, ws+"    "+out+" = "+g.getType(t)+"{}")
+			fmt.Fprintln(g.out, ws+"  }")
+			fmt.Fprintln(g.out, ws+"  for !in.IsDelim(']') {")
+			fmt.Fprintln(g.out, ws+"    var "+tmpVar+" "+g.getType(elem))
+
+			g.genTypeDecoder(elem, tmpVar, tags, indent+2)
+
+			fmt.Fprintln(g.out, ws+"    "+out+" = append("+out+", "+tmpVar+")")
+			fmt.Fprintln(g.out, ws+"    in.WantComma()")
+			fmt.Fprintln(g.out, ws+"  }")
+			fmt.Fprintln(g.out, ws+"  in.Delim(']')")
+			fmt.Fprintln(g.out, ws+"}")
 		}
 
-		fmt.Fprintln(g.out, ws+"in.Delim('[')")
-		fmt.Fprintln(g.out, ws+"if !in.IsDelim(']') {")
-		fmt.Fprintln(g.out, ws+"  "+out+" = make("+g.getType(t)+", 0, "+fmt.Sprint(capacity)+")")
-		fmt.Fprintln(g.out, ws+"} else {")
-		fmt.Fprintln(g.out, ws+"  "+out+" = nil")
-		fmt.Fprintln(g.out, ws+"}")
-		fmt.Fprintln(g.out, ws+"for !in.IsDelim(']') {")
-		fmt.Fprintln(g.out, ws+"  var "+tmpVar+" "+g.getType(elem))
-
-		g.genTypeDecoder(elem, tmpVar, tags, indent+1)
-
-		fmt.Fprintln(g.out, ws+"  "+out+" = append("+out+", "+tmpVar+")")
-		fmt.Fprintln(g.out, ws+"  in.WantComma()")
-		fmt.Fprintln(g.out, ws+"}")
-		fmt.Fprintln(g.out, ws+"in.Delim(']')")
-
 	case reflect.Struct:
 		dec := g.getDecoderName(t)
 		g.addType(t)

gen/encoder.go

@@ -118,16 +118,24 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT
 		iVar := g.uniqueVarName()
 		vVar := g.uniqueVarName()
 
-		fmt.Fprintln(g.out, ws+"out.RawByte('[')")
-		fmt.Fprintln(g.out, ws+"for "+iVar+", "+vVar+" := range "+in+" {")
-		fmt.Fprintln(g.out, ws+"  if "+iVar+" > 0 {")
-		fmt.Fprintln(g.out, ws+"    out.RawByte(',')")
-		fmt.Fprintln(g.out, ws+"  }")
+		if t.Elem().Kind() == reflect.Uint8 {
+			fmt.Fprintln(g.out, ws+"out.Base64Bytes("+in+")")
+		} else {
+			fmt.Fprintln(g.out, ws+"if "+in+" == nil {")
+			fmt.Fprintln(g.out, ws+`  out.RawString("null")`)
+			fmt.Fprintln(g.out, ws+"} else {")
+			fmt.Fprintln(g.out, ws+"  out.RawByte('[')")
+			fmt.Fprintln(g.out, ws+"  for "+iVar+", "+vVar+" := range "+in+" {")
+			fmt.Fprintln(g.out, ws+"    if "+iVar+" > 0 {")
+			fmt.Fprintln(g.out, ws+"      out.RawByte(',')")
+			fmt.Fprintln(g.out, ws+"    }")
 
-		g.genTypeEncoder(elem, vVar, tags, indent+1)
+			g.genTypeEncoder(elem, vVar, tags, indent+2)
 
-		fmt.Fprintln(g.out, ws+"}")
-		fmt.Fprintln(g.out, ws+"out.RawByte(']')")
+			fmt.Fprintln(g.out, ws+"  }")
+			fmt.Fprintln(g.out, ws+"  out.RawByte(']')")
+			fmt.Fprintln(g.out, ws+"}")
+		}
 
 	case reflect.Struct:
 		enc := g.getEncoderName(t)

jlexer/lexer.go

@@ -5,6 +5,7 @@
 package jlexer
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"reflect"
@@ -560,6 +561,28 @@ func (r *Lexer) String() string {
 	return ret
 }
 
+// Bytes reads a string literal and base64 decodes it into a byte slice.
+func (r *Lexer) Bytes() []byte {
+	if r.token.kind == tokenUndef && r.Ok() {
+		r.fetchToken()
+	}
+	if !r.Ok() || r.token.kind != tokenString {
+		r.errInvalidToken("string")
+		return nil
+	}
+	ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue)))
+	len, err := base64.StdEncoding.Decode(ret, r.token.byteValue)
+	if err != nil {
+		r.err = &LexerError{
+			Reason: err.Error(),
+		}
+		return nil
+	}
+
+	r.consume()
+	return ret[:len]
+}
+
 // Bool reads a true or false boolean keyword.
 func (r *Lexer) Bool() bool {
 	if r.token.kind == tokenUndef && r.Ok() {

jlexer/lexer_test.go

@@ -1,6 +1,7 @@
 package jlexer
 
 import (
+	"bytes"
 	"reflect"
 	"testing"
 )
@@ -39,6 +40,34 @@ func TestString(t *testing.T) {
 	}
 }
 
+func TestBytes(t *testing.T) {
+	for i, test := range []struct {
+		toParse   string
+		want      string
+		wantError bool
+	}{
+		{toParse: `"c2ltcGxlIHN0cmluZw=="`, want: "simple string"},
+		{toParse: " \r\r\n\t  " + `"dGVzdA=="`, want: "test"},
+
+		{toParse: `5`, wantError: true},                     // not a JSON string
+		{toParse: `"foobar"`, wantError: true},              // not base64 encoded
+		{toParse: `"c2ltcGxlIHN0cmluZw="`, wantError: true}, // invalid base64 padding
+	} {
+		l := Lexer{Data: []byte(test.toParse)}
+
+		got := l.Bytes()
+		if bytes.Compare(got, []byte(test.want)) != 0 {
+			t.Errorf("[%d, %q] Bytes() = %v; want: %v", i, test.toParse, got, []byte(test.want))
+		}
+		err := l.Error()
+		if err != nil && !test.wantError {
+			t.Errorf("[%d, %q] Bytes() error: %v", i, test.toParse, err)
+		} else if err == nil && test.wantError {
+			t.Errorf("[%d, %q] Bytes() ok; want error", i, test.toParse)
+		}
+	}
+}
+
 func TestNumber(t *testing.T) {
 	for i, test := range []struct {
 		toParse   string

jwriter/writer.go

@@ -2,6 +2,7 @@
 package jwriter
 
 import (
+	"encoding/base64"
 	"io"
 	"strconv"
 	"unicode/utf8"
@@ -59,6 +60,19 @@ func (w *Writer) Raw(data []byte, err error) {
 	}
 }
 
+// Base64Bytes appends data to the buffer after base64 encoding it
+func (w *Writer) Base64Bytes(data []byte) {
+	if data == nil {
+		w.Buffer.AppendString("null")
+		return
+	}
+	w.Buffer.AppendByte('"')
+	dst := make([]byte, base64.StdEncoding.EncodedLen(len(data)))
+	base64.StdEncoding.Encode(dst, data)
+	w.Buffer.AppendBytes(dst)
+	w.Buffer.AppendByte('"')
+}
+
 func (w *Writer) Uint8(n uint8) {
 	w.Buffer.EnsureSpace(3)
 	w.Buffer.Buf = strconv.AppendUint(w.Buffer.Buf, uint64(n), 10)

tests/basic_test.go

@@ -30,6 +30,7 @@ var testCases = []struct {
 	{&stdMarshalerValue, stdMarshalerString},
 	{&unexportedStructValue, unexportedStructString},
 	{&excludedFieldValue, excludedFieldString},
+	{&sliceValue, sliceString},
 	{&mapsValue, mapsString},
 	{&deepNestValue, deepNestString},
 	{&IntsValue, IntsString},

tests/data.go

@@ -296,10 +296,10 @@ var structsString = "{" +
 	`"SubNil":null,` +
 
 	`"SubSlice":[{"Value":"s1","Value2":""},{"Value":"s2","Value2":""}],` +
-	`"SubSliceNil":[],` +
+	`"SubSliceNil":null,` +
 
 	`"SubPtrSlice":[{"Value":"p1","Value2":""},{"Value":"p2","Value2":""}],` +
-	`"SubPtrSliceNil":[],` +
+	`"SubPtrSliceNil":null,` +
 
 	`"SubA1":{"Value":"test3","Value2":"v3"},` +
 	`"SubA2":{"Value":"test4","Value2":"v4"},` +
@@ -418,6 +418,33 @@ var excludedFieldValue = ExcludedField{
 }
 var excludedFieldString = `{"process":true}`
 
+type Slices struct {
+	ByteSlice      []byte
+	EmptyByteSlice []byte
+	NilByteSlice   []byte
+	IntSlice       []int
+	EmptyIntSlice  []int
+	NilIntSlice    []int
+}
+
+var sliceValue = Slices{
+	ByteSlice:      []byte("abc"),
+	EmptyByteSlice: []byte{},
+	NilByteSlice:   []byte(nil),
+	IntSlice:       []int{1, 2, 3, 4, 5},
+	EmptyIntSlice:  []int{},
+	NilIntSlice:    []int(nil),
+}
+
+var sliceString = `{` +
+	`"ByteSlice":"YWJj",` +
+	`"EmptyByteSlice":"",` +
+	`"NilByteSlice":null,` +
+	`"IntSlice":[1,2,3,4,5],` +
+	`"EmptyIntSlice":[],` +
+	`"NilIntSlice":null` +
+	`}`
+
 type Str string
 
 type Maps struct {
@@ -448,6 +475,7 @@ type NamedMap map[Str]Str
 type DeepNest struct {
 	SliceMap         map[Str][]Str
 	SliceMap1        map[Str][]Str
+	SliceMap2        map[Str][]Str
 	NamedSliceMap    map[Str]NamedSlice
 	NamedMapMap      map[Str]NamedMap
 	MapSlice         []map[Str]Str
@@ -464,7 +492,10 @@ var deepNestValue = DeepNest{
 		},
 	},
 	SliceMap1: map[Str][]Str{
-		"testSliceMap1": nil,
+		"testSliceMap1": []Str(nil),
+	},
+	SliceMap2: map[Str][]Str{
+		"testSliceMap2": []Str{},
 	},
 	NamedSliceMap: map[Str]NamedSlice{
 		"testNamedSliceMap": NamedSlice{
@@ -510,7 +541,10 @@ var deepNestString = `{` +
 	`"testSliceMap":["0","1"]` +
 	`},` +
 	`"SliceMap1":{` +
-	`"testSliceMap1":[]` +
+	`"testSliceMap1":null` +
+	`},` +
+	`"SliceMap2":{` +
+	`"testSliceMap2":[]` +
 	`},` +
 	`"NamedSliceMap":{` +
 	`"testNamedSliceMap":["2","3"]` +