Merge branch 'master' into opts

.travis.yml

@@ -1,8 +1,9 @@
 language: go
 
 go:
-        - tip
+  - tip
 install:
-        - go get github.com/ugorji/go/codec
-        - go get github.com/pquerna/ffjson/fflib/v1
-        - go get github.com/golang/lint/golint
+  - go get github.com/ugorji/go/codec
+  - go get github.com/pquerna/ffjson/fflib/v1
+  - go get github.com/json-iterator/go
+  - go get github.com/golang/lint/golint

Makefile

@@ -4,7 +4,7 @@ export GOPATH
 
 all: test
 
-.root/src/$(PKG): 	
+.root/src/$(PKG):
 	mkdir -p $@
 	for i in $$PWD/* ; do ln -s $$i $@/`basename $$i` ; done 
 
@@ -45,6 +45,7 @@ test: generate root
 bench-other: generate root
 	@go test -benchmem -bench . $(PKG)/benchmark
 	@go test -benchmem -tags use_ffjson -bench . $(PKG)/benchmark
+	@go test -benchmem -tags use_jsoniter -bench . $(PKG)/benchmark
 	@go test -benchmem -tags use_codec -bench . $(PKG)/benchmark
 
 bench-python:

README.md

@@ -56,7 +56,7 @@ Usage of easyjson:
 ```
 
 Using `-all` will generate marshalers/unmarshalers for all Go structs in the
-file. If `-all` is not provided, then only those structs whose preceeding
+file. If `-all` is not provided, then only those structs whose preceding
 comment starts with `easyjson:json` will have marshalers/unmarshalers
 generated. For example:
 

benchmark/default_test.go

@@ -1,4 +1,4 @@
-// +build !use_easyjson,!use_ffjson,!use_codec
+// +build !use_easyjson,!use_ffjson,!use_codec,!use_jsoniter
 
 package benchmark
 

benchmark/jsoniter_test.go

@@ -0,0 +1,119 @@
+// +build use_jsoniter
+
+package benchmark
+
+import (
+	"testing"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+func BenchmarkJI_Unmarshal_M(b *testing.B) {
+	b.SetBytes(int64(len(largeStructText)))
+	for i := 0; i < b.N; i++ {
+		var s LargeStruct
+		err := jsoniter.Unmarshal(largeStructText, &s)
+		if err != nil {
+			b.Error(err)
+		}
+	}
+}
+
+func BenchmarkJI_Unmarshal_S(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var s Entities
+		err := jsoniter.Unmarshal(smallStructText, &s)
+		if err != nil {
+			b.Error(err)
+		}
+	}
+	b.SetBytes(int64(len(smallStructText)))
+}
+
+func BenchmarkJI_Marshal_M(b *testing.B) {
+	var l int64
+	for i := 0; i < b.N; i++ {
+		data, err := jsoniter.Marshal(&largeStructData)
+		if err != nil {
+			b.Error(err)
+		}
+		l = int64(len(data))
+	}
+	b.SetBytes(l)
+}
+
+func BenchmarkJI_Marshal_L(b *testing.B) {
+	var l int64
+	for i := 0; i < b.N; i++ {
+		data, err := jsoniter.Marshal(&xlStructData)
+		if err != nil {
+			b.Error(err)
+		}
+		l = int64(len(data))
+	}
+	b.SetBytes(l)
+}
+
+func BenchmarkJI_Marshal_M_Parallel(b *testing.B) {
+	var l int64
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			data, err := jsoniter.Marshal(&largeStructData)
+			if err != nil {
+				b.Error(err)
+			}
+			l = int64(len(data))
+		}
+	})
+	b.SetBytes(l)
+}
+
+func BenchmarkJI_Marshal_L_Parallel(b *testing.B) {
+	var l int64
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			data, err := jsoniter.Marshal(&xlStructData)
+			if err != nil {
+				b.Error(err)
+			}
+			l = int64(len(data))
+		}
+	})
+	b.SetBytes(l)
+}
+
+func BenchmarkJI_Marshal_S(b *testing.B) {
+	var l int64
+	for i := 0; i < b.N; i++ {
+		data, err := jsoniter.Marshal(&smallStructData)
+		if err != nil {
+			b.Error(err)
+		}
+		l = int64(len(data))
+	}
+	b.SetBytes(l)
+}
+
+func BenchmarkJI_Marshal_S_Parallel(b *testing.B) {
+	var l int64
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			data, err := jsoniter.Marshal(&smallStructData)
+			if err != nil {
+				b.Error(err)
+			}
+			l = int64(len(data))
+		}
+	})
+	b.SetBytes(l)
+}
+
+func BenchmarkJI_Marshal_M_ToWriter(b *testing.B) {
+	enc := jsoniter.NewEncoder(&DummyWriter{})
+	for i := 0; i < b.N; i++ {
+		err := enc.Encode(&largeStructData)
+		if err != nil {
+			b.Error(err)
+		}
+	}
+}

easyjson/main.go

@@ -54,8 +54,13 @@ func generate(fname string) (err error) {
 		outName = *specifiedName
 	}
 
+	var trimmedBuildTags string
+	if *buildTags != "" {
+		trimmedBuildTags = strings.TrimSpace(*buildTags)
+	}
+
 	g := bootstrap.Generator{
-		BuildTags:       *buildTags,
+		BuildTags:       trimmedBuildTags,
 		PkgPath:         p.PkgPath,
 		PkgName:         p.PkgName,
 		Types:           p.StructNames,

gen/decoder.go

@@ -48,6 +48,10 @@ var primitiveStringDecoders = map[reflect.Kind]string{
 	reflect.Uint64: "in.Uint64Str()",
 }
 
+var customDecoders = map[string]string{
+	"json.Number":    "in.JsonNumber()",
+}
+
 // genTypeDecoder generates decoding code for the type t, but uses unmarshaler interface if implemented by t.
 func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, indent int) error {
 	ws := strings.Repeat("  ", indent)
@@ -82,7 +86,10 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i
 func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags fieldTags, indent int) error {
 	ws := strings.Repeat("  ", indent)
 	// Check whether type is primitive, needs to be done after interface check.
-	if dec := primitiveStringDecoders[t.Kind()]; dec != "" && tags.asString {
+	if dec := customDecoders[t.String()]; dec != ""  {
+		fmt.Fprintln(g.out, ws+out+" = "+dec)
+		return nil
+	} else if dec := primitiveStringDecoders[t.Kind()]; dec != "" && tags.asString {
 		fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")")
 		return nil
 	} else if dec := primitiveDecoders[t.Kind()]; dec != "" {
@@ -127,7 +134,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
 			fmt.Fprintln(g.out, ws+"  for !in.IsDelim(']') {")
 			fmt.Fprintln(g.out, ws+"    var "+tmpVar+" "+g.getType(elem))
 
-			g.genTypeDecoder(elem, tmpVar, tags, indent+2)
+			if err := g.genTypeDecoder(elem, tmpVar, tags, indent+2); err != nil {
+				return err
+			}
 
 			fmt.Fprintln(g.out, ws+"    "+out+" = append("+out+", "+tmpVar+")")
 			fmt.Fprintln(g.out, ws+"    in.WantComma()")
@@ -159,7 +168,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
 			fmt.Fprintln(g.out, ws+"  for !in.IsDelim(']') {")
 			fmt.Fprintln(g.out, ws+"    if "+iterVar+" < "+fmt.Sprint(length)+" {")
 
-			g.genTypeDecoder(elem, out+"["+iterVar+"]", tags, indent+3)
+			if err := g.genTypeDecoder(elem, out+"["+iterVar+"]", tags, indent+3); err != nil {
+				return err
+			}
 
 			fmt.Fprintln(g.out, ws+"      "+iterVar+"++")
 			fmt.Fprintln(g.out, ws+"    } else {")
@@ -186,7 +197,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
 		fmt.Fprintln(g.out, ws+"    "+out+" = new("+g.getType(t.Elem())+")")
 		fmt.Fprintln(g.out, ws+"  }")
 
-		g.genTypeDecoder(t.Elem(), "*"+out, tags, indent+1)
+		if err := g.genTypeDecoder(t.Elem(), "*"+out, tags, indent+1); err != nil {
+			return err
+		}
 
 		fmt.Fprintln(g.out, ws+"}")
 
@@ -213,7 +226,9 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
 		fmt.Fprintln(g.out, ws+"    in.WantColon()")
 		fmt.Fprintln(g.out, ws+"    var "+tmpVar+" "+g.getType(elem))
 
-		g.genTypeDecoder(elem, tmpVar, tags, indent+2)
+		if err := g.genTypeDecoder(elem, tmpVar, tags, indent+2); err != nil {
+			return err
+		}
 
 		fmt.Fprintln(g.out, ws+"    ("+out+")[key] = "+tmpVar)
 		fmt.Fprintln(g.out, ws+"    in.WantComma()")

gen/encoder.go

@@ -141,7 +141,9 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT
 			fmt.Fprintln(g.out, ws+"      out.RawByte(',')")
 			fmt.Fprintln(g.out, ws+"    }")
 
-			g.genTypeEncoder(elem, vVar, tags, indent+2, false)
+			if err := g.genTypeEncoder(elem, vVar, tags, indent+2, false); err != nil {
+				return err
+			}
 
 			fmt.Fprintln(g.out, ws+"  }")
 			fmt.Fprintln(g.out, ws+"  out.RawByte(']')")
@@ -161,7 +163,9 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT
 			fmt.Fprintln(g.out, ws+"    out.RawByte(',')")
 			fmt.Fprintln(g.out, ws+"  }")
 
-			g.genTypeEncoder(elem, in+"["+iVar+"]", tags, indent+1, false)
+			if err := g.genTypeEncoder(elem, in+"["+iVar+"]", tags, indent+1, false); err != nil {
+				return err
+			}
 
 			fmt.Fprintln(g.out, ws+"}")
 			fmt.Fprintln(g.out, ws+"out.RawByte(']')")
@@ -180,7 +184,9 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT
 			fmt.Fprintln(g.out, ws+"} else {")
 		}
 
-		g.genTypeEncoder(t.Elem(), "*"+in, tags, indent+1, false)
+		if err := g.genTypeEncoder(t.Elem(), "*"+in, tags, indent+1, false); err != nil {
+			return err
+		}
 
 		if !assumeNonEmpty {
 			fmt.Fprintln(g.out, ws+"}")
@@ -207,7 +213,9 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT
 		fmt.Fprintln(g.out, ws+"    out.String(string("+tmpVar+"Name))")
 		fmt.Fprintln(g.out, ws+"    out.RawByte(':')")
 
-		g.genTypeEncoder(t.Elem(), tmpVar+"Value", tags, indent+2, false)
+		if err := g.genTypeEncoder(t.Elem(), tmpVar+"Value", tags, indent+2, false); err != nil {
+			return err
+		}
 
 		fmt.Fprintln(g.out, ws+"  }")
 		fmt.Fprintln(g.out, ws+"  out.RawByte('}')")

gen/generator.go

@@ -224,6 +224,10 @@ func fixAliasName(alias string) string {
 		"_",
 		-1,
 	)
+
+	if alias[0] == 'v' { // to void conflicting with var names, say v1
+		alias = "_" + alias
+	}
 	return alias
 }
 
@@ -380,7 +384,7 @@ func (DefaultFieldNamer) GetJSONFieldName(t reflect.Type, f reflect.StructField)
 }
 
 // LowerCamelCaseFieldNamer
-type LowerCamelCaseFieldNamer struct {}
+type LowerCamelCaseFieldNamer struct{}
 
 func isLower(b byte) bool {
 	return b <= 122 && b >= 97
@@ -407,7 +411,7 @@ func lowerFirst(s string) string {
 	    If the following char is upper OR numeric, LOWER it
 	    If is the end of string, LEAVE it
 	  Else lowercase
-	 */
+	*/
 
 	foundLower := false
 	for i := range s {

jlexer/lexer.go

@@ -6,6 +6,7 @@ package jlexer
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -1043,6 +1044,28 @@ func (r *Lexer) GetNonFatalErrors() []*LexerError {
 	return r.multipleErrors
 }
 
+// JsonNumber fetches and json.Number from 'encoding/json' package.
+// Both int, float or string, contains them are valid values
+func (r *Lexer) JsonNumber() json.Number {
+	if r.token.kind == tokenUndef && r.Ok() {
+		r.FetchToken()
+	}
+	if !r.Ok() {
+		r.errInvalidToken("json.Number")
+		return json.Number("0")
+	}
+
+	switch r.token.kind {
+	case tokenString:
+		return json.Number(r.String())
+	case tokenNumber:
+		return json.Number(r.Raw())
+	default:
+		r.errSyntax()
+		return json.Number("0")
+	}
+}
+
 // Interface fetches an interface{} analogous to the 'encoding/json' package.
 func (r *Lexer) Interface() interface{} {
 	if r.token.kind == tokenUndef && r.Ok() {

jlexer/lexer_test.go

@@ -2,6 +2,7 @@ package jlexer
 
 import (
 	"bytes"
+	"encoding/json"
 	"reflect"
 	"testing"
 )
@@ -249,3 +250,62 @@ func TestConsumed(t *testing.T) {
 		}
 	}
 }
+
+func TestJsonNumber(t *testing.T) {
+	for i, test := range []struct {
+		toParse        string
+		want           json.Number
+		wantLexerError bool
+		wantValue      interface{}
+		wantValueError bool
+	}{
+		{toParse: `10`, want: json.Number("10"), wantValue: int64(10)},
+		{toParse: `0`, want: json.Number("0"), wantValue: int64(0)},
+		{toParse: `0.12`, want: json.Number("0.12"), wantValue: 0.12},
+		{toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25E-4},
+
+		{toParse: `"10"`, want: json.Number("10"), wantValue: int64(10)},
+		{toParse: `"0"`, want: json.Number("0"), wantValue: int64(0)},
+		{toParse: `"0.12"`, want: json.Number("0.12"), wantValue: 0.12},
+		{toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25E-4},
+
+		{toParse: `"a""`, wantValueError: true},
+
+		{toParse: `[1]`, wantLexerError: true},
+		{toParse: `{}`, wantLexerError: true},
+		{toParse: `a`, wantLexerError: true},
+	} {
+		l := Lexer{Data: []byte(test.toParse)}
+
+		got := l.JsonNumber()
+		if got != test.want && !test.wantLexerError && !test.wantValueError {
+			t.Errorf("[%d, %q] JsonNumber() = %v; want %v", i, test.toParse, got, test.want)
+		}
+
+		err := l.Error()
+		if err != nil && !test.wantLexerError {
+			t.Errorf("[%d, %q] JsonNumber() lexer error: %v", i, test.toParse, err)
+		} else if err == nil && test.wantLexerError {
+			t.Errorf("[%d, %q] JsonNumber() ok; want lexer error", i, test.toParse)
+		}
+
+		var valueErr error
+		var gotValue interface{}
+		switch test.wantValue.(type) {
+		case float64:
+			gotValue, valueErr = got.Float64()
+		default:
+			gotValue, valueErr = got.Int64()
+		}
+
+		if !reflect.DeepEqual(gotValue, test.wantValue) && !test.wantLexerError && !test.wantValueError {
+			t.Errorf("[%d, %q] JsonNumber() = %v; want %v", i, test.toParse, gotValue, test.wantValue)
+		}
+
+		if valueErr != nil && !test.wantValueError {
+			t.Errorf("[%d, %q] JsonNumber() value error: %v", i, test.toParse, err)
+		} else if valueErr == nil && test.wantValueError {
+			t.Errorf("[%d, %q] JsonNumber() ok; want value error", i, test.toParse)
+		}
+	}
+}

parser/parser_windows.go

@@ -4,15 +4,18 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 )
 
 func normalizePath(path string) string {
-	return strings.Replace(path, "\\", "/", -1)
+	// use lower case, as Windows file systems will almost always be case insensitive 
+	return strings.ToLower(strings.Replace(path, "\\", "/", -1))
 }
 
 func getPkgPath(fname string, isDir bool) (string, error) {
-	if !path.IsAbs(fname) {
+	// path.IsAbs doesn't work properly on Windows; use filepath.IsAbs instead
+	if !filepath.IsAbs(fname) {
 		pwd, err := os.Getwd()
 		if err != nil {
 			return "", err

tests/non_string_keyed_map_test.go

@@ -0,0 +1,44 @@
+package tests
+
+import (
+	"os"
+	"testing"
+
+	"github.com/mailru/easyjson/gen"
+)
+
+type IntMap map[int]string
+type IntMapSlice []IntMap
+type IntMapArray [2]IntMap
+type IntMapPtr *IntMap
+type IntMapMap map[string]IntMap
+
+func TestNonStringKeyedtMapEncoder(t *testing.T) {
+	f := "non_string_keyed_map_easyjson.go"
+	for _, test := range []struct {
+		Data interface{}
+	}{
+		{
+			Data: IntMap{},
+		},
+		{
+			Data: IntMapSlice{},
+		},
+		{
+			Data: IntMapArray{},
+		},
+		{
+			Data: IntMapPtr(nil),
+		},
+		{
+			Data: IntMapMap{},
+		},
+	} {
+		g := gen.NewGenerator(f)
+		g.Add(test.Data)
+		e := g.Run(os.Stdout)
+		if e == nil {
+			t.Errorf("generation for %#v should have errored", test.Data)
+		}
+	}
+}