Merge branch 'master' into nested-easyjson-marshal

# Conflicts:
#	tests/basic_test.go

easyjson/main.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/mailru/easyjson/bootstrap"
@@ -23,18 +25,28 @@ var leaveTemps = flag.Bool("leave_temps", false, "do not delete temporary files"
 var stubs = flag.Bool("stubs", false, "only generate stubs for marshallers/unmarshallers methods")
 var noformat = flag.Bool("noformat", false, "do not run 'gofmt -w' on output file")
 var specifiedName = flag.String("output_filename", "", "specify the filename of the output")
+var processPkg = flag.Bool("pkg", false, "process the whole package instead of just the given file")
 
 func generate(fname string) (err error) {
+	fInfo, err := os.Stat(fname)
+	if err != nil {
+		return err
+	}
+
 	p := parser.Parser{AllStructs: *allStructs}
-	if err := p.Parse(fname); err != nil {
+	if err := p.Parse(fname, fInfo.IsDir()); err != nil {
 		return fmt.Errorf("Error parsing %v: %v", fname, err)
 	}
 
 	var outName string
-	if s := strings.TrimSuffix(fname, ".go"); s == fname {
-		return fmt.Errorf("Filename must end in '.go'")
+	if fInfo.IsDir() {
+		outName = filepath.Join(fname, p.PkgName+"_easyjson.go")
 	} else {
-		outName = s + "_easyjson.go"
+		if s := strings.TrimSuffix(fname, ".go"); s == fname {
+			return errors.New("Filename must end in '.go'")
+		} else {
+			outName = s + "_easyjson.go"
+		}
 	}
 
 	if *specifiedName != "" {
@@ -67,6 +79,10 @@ func main() {
 	files := flag.Args()
 
 	gofile := os.Getenv("GOFILE")
+	if *processPkg {
+		gofile = filepath.Dir(gofile)
+	}
+
 	if len(files) == 0 && gofile != "" {
 		files = []string{gofile}
 	} else if len(files) == 0 {

gen/decoder.go

@@ -340,10 +340,14 @@ func (g *Generator) genSliceArrayDecoder(t reflect.Type) error {
 	typ := g.getType(t)
 
 	fmt.Fprintln(g.out, "func "+fname+"(in *jlexer.Lexer, out *"+typ+") {")
+	fmt.Fprintln(g.out, " isTopLevel := in.IsStart()")
 	err := g.genTypeDecoderNoCheck(t, "*out", fieldTags{}, 1)
 	if err != nil {
 		return err
 	}
+	fmt.Fprintln(g.out, "  if isTopLevel {")
+	fmt.Fprintln(g.out, "    in.Consumed()")
+	fmt.Fprintln(g.out, "  }")
 	fmt.Fprintln(g.out, "}")
 
 	return nil
@@ -358,7 +362,11 @@ func (g *Generator) genStructDecoder(t reflect.Type) error {
 	typ := g.getType(t)
 
 	fmt.Fprintln(g.out, "func "+fname+"(in *jlexer.Lexer, out *"+typ+") {")
+	fmt.Fprintln(g.out, "  isTopLevel := in.IsStart()")
 	fmt.Fprintln(g.out, "  if in.IsNull() {")
+	fmt.Fprintln(g.out, "    if isTopLevel {")
+	fmt.Fprintln(g.out, "      in.Consumed()")
+	fmt.Fprintln(g.out, "    }")
 	fmt.Fprintln(g.out, "    in.Skip()")
 	fmt.Fprintln(g.out, "    return")
 	fmt.Fprintln(g.out, "  }")
@@ -404,6 +412,9 @@ func (g *Generator) genStructDecoder(t reflect.Type) error {
 	fmt.Fprintln(g.out, "    in.WantComma()")
 	fmt.Fprintln(g.out, "  }")
 	fmt.Fprintln(g.out, "  in.Delim('}')")
+	fmt.Fprintln(g.out, "  if isTopLevel {")
+	fmt.Fprintln(g.out, "    in.Consumed()")
+	fmt.Fprintln(g.out, "  }")
 
 	for _, f := range fs {
 		g.genRequiredFieldCheck(t, f)

gen/encoder.go

@@ -121,7 +121,7 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT
 		if t.Elem().Kind() == reflect.Uint8 {
 			fmt.Fprintln(g.out, ws+"out.Base64Bytes("+in+")")
 		} else {
-			fmt.Fprintln(g.out, ws+"if "+in+" == nil {")
+			fmt.Fprintln(g.out, ws+"if "+in+" == nil && (out.Flags & jwriter.NilSliceAsEmpty) == 0 {")
 			fmt.Fprintln(g.out, ws+`  out.RawString("null")`)
 			fmt.Fprintln(g.out, ws+"} else {")
 			fmt.Fprintln(g.out, ws+"  out.RawByte('[')")
@@ -178,7 +178,7 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT
 		}
 		tmpVar := g.uniqueVarName()
 
-		fmt.Fprintln(g.out, ws+"if "+in+" == nil {")
+		fmt.Fprintln(g.out, ws+"if "+in+" == nil && (out.Flags & jwriter.NilMapAsEmpty) == 0 {")
 		fmt.Fprintln(g.out, ws+"  out.RawString(`null`)")
 		fmt.Fprintln(g.out, ws+"} else {")
 		fmt.Fprintln(g.out, ws+"  out.RawByte('{')")

gen/generator.go

@@ -202,8 +202,18 @@ func (g *Generator) Run(out io.Writer) error {
 	return err
 }
 
+// fixes vendored paths
+func fixPkgPathVendoring(pkgPath string) string {
+	const vendor = "/vendor/"
+	if i := strings.LastIndex(pkgPath, vendor); i != -1 {
+		return pkgPath[i+len(vendor):]
+	}
+	return pkgPath
+}
+
 // pkgAlias creates and returns and import alias for a given package.
 func (g *Generator) pkgAlias(pkgPath string) string {
+	pkgPath = fixPkgPathVendoring(pkgPath)
 	if alias := g.imports[pkgPath]; alias != "" {
 		return alias
 	}

gen/generator_test.go

@@ -47,3 +47,19 @@ func TestJoinFunctionNameParts(t *testing.T) {
 		}
 	}
 }
+
+func TestFixVendorPath(t *testing.T) {
+	for i, test := range []struct {
+		In, Out string
+	}{
+		{"", ""},
+		{"time", "time"},
+		{"project/vendor/subpackage", "subpackage"},
+	} {
+		got := fixPkgPathVendoring(test.In)
+		if got != test.Out {
+			t.Errorf("[%d] fixPkgPathVendoring(%s) = %s; want %s", i, test.In, got, test.Out)
+		}
+	}
+
+}

jlexer/lexer.go

@@ -519,8 +519,9 @@ func (r *Lexer) SkipRecursive() {
 	r.err = &LexerError{
 		Reason: "EOF reached while skipping array/object or token",
 		Offset: r.pos,
-		Data: string(r.Data[r.pos:]),
-	}}
+		Data:   string(r.Data[r.pos:]),
+	}
+}
 
 // Raw fetches the next item recursively as a data slice
 func (r *Lexer) Raw() []byte {
@@ -531,6 +532,34 @@ func (r *Lexer) Raw() []byte {
 	return r.Data[r.start:r.pos]
 }
 
+// IsStart returns whether the lexer is positioned at the start
+// of an input string.
+func (r *Lexer) IsStart() bool {
+	return r.pos == 0
+}
+
+// Consumed reads all remaining bytes from the input, publishing an error if
+// there is anything but whitespace remaining.
+func (r *Lexer) Consumed() {
+	if r.pos > len(r.Data) {
+		return
+	}
+
+	for _, c := range r.Data[r.pos:] {
+		if c != ' ' && c != '\t' && c != '\r' && c != '\n' {
+			r.err = &LexerError{
+				Reason: "invalid character '" + string(c) + "' after top-level value",
+				Offset: r.pos,
+				Data:   string(r.Data[r.pos:]),
+			}
+			return
+		}
+
+		r.pos++
+		r.start++
+	}
+}
+
 // UnsafeString returns the string value if the token is a string literal.
 //
 // Warning: returned string may point to the input buffer, so the string should not outlive

jlexer/lexer_test.go

@@ -222,3 +222,27 @@ func TestInterface(t *testing.T) {
 		}
 	}
 }
+
+func TestConsumed(t *testing.T) {
+	for i, test := range []struct {
+		toParse   string
+		wantError bool
+	}{
+		{toParse: "", wantError: false},
+		{toParse: "   ", wantError: false},
+		{toParse: "\r\n", wantError: false},
+		{toParse: "\t\t", wantError: false},
+
+		{toParse: "{", wantError: true},
+	} {
+		l := Lexer{Data: []byte(test.toParse)}
+		l.Consumed()
+
+		err := l.Error()
+		if err != nil && !test.wantError {
+			t.Errorf("[%d, %q] Consumed() error: %v", i, test.toParse, err)
+		} else if err == nil && test.wantError {
+			t.Errorf("[%d, %q] Consumed() ok; want error", i, test.toParse)
+		}
+	}
+}

jwriter/writer.go

@@ -10,8 +10,19 @@ import (
 	"github.com/mailru/easyjson/buffer"
 )
 
+// Flags describe various encoding options. The behavior may be actually implemented in the encoder, but
+// Flags field in Writer is used to set and pass them around.
+type Flags int
+
+const (
+	NilMapAsEmpty   Flags = 1 << iota // Encode nil map as '{}' rather than 'null'.
+	NilSliceAsEmpty                   // Encode nil slice as '[]' rather than 'null'.
+)
+
 // Writer is a JSON writer.
 type Writer struct {
+	Flags Flags
+
 	Error  error
 	Buffer buffer.Buffer
 }

parser/parser.go

@@ -34,6 +34,8 @@ func (p *Parser) needType(comments string) bool {
 
 func (v *visitor) Visit(n ast.Node) (w ast.Visitor) {
 	switch n := n.(type) {
+	case *ast.Package:
+		return v
 	case *ast.File:
 		v.PkgName = n.Name.String()
 		return v
@@ -61,18 +63,29 @@ func (v *visitor) Visit(n ast.Node) (w ast.Visitor) {
 	return nil
 }
 
-func (p *Parser) Parse(fname string) error {
+func (p *Parser) Parse(fname string, isDir bool) error {
 	var err error
-	if p.PkgPath, err = getPkgPath(fname); err != nil {
+	if p.PkgPath, err = getPkgPath(fname, isDir); err != nil {
 		return err
 	}
 
 	fset := token.NewFileSet()
-	f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
-	if err != nil {
-		return err
-	}
+	if isDir {
+		packages, err := parser.ParseDir(fset, fname, nil, parser.ParseComments)
+		if err != nil {
+			return err
+		}
 
-	ast.Walk(&visitor{Parser: p}, f)
+		for _, pckg := range packages {
+			ast.Walk(&visitor{Parser: p}, pckg)
+		}
+	} else {
+		f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
+		if err != nil {
+			return err
+		}
+
+		ast.Walk(&visitor{Parser: p}, f)
+	}
 	return nil
 }

parser/parser_unix.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 )
 
-func getPkgPath(fname string) (string, error) {
+func getPkgPath(fname string, isDir bool) (string, error) {
 	if !path.IsAbs(fname) {
 		pwd, err := os.Getwd()
 		if err != nil {
@@ -21,7 +21,11 @@ func getPkgPath(fname string) (string, error) {
 	for _, p := range strings.Split(os.Getenv("GOPATH"), ":") {
 		prefix := path.Join(p, "src") + "/"
 		if rel := strings.TrimPrefix(fname, prefix); rel != fname {
-			return path.Dir(rel), nil
+			if !isDir {
+				return path.Dir(rel), nil
+			} else {
+				return path.Clean(rel), nil
+			}
 		}
 	}
 

tests/basic_test.go

@@ -157,6 +157,33 @@ func TestUnderflowArray(t *testing.T) {
 	}
 }
 
+func TestEncodingFlags(t *testing.T) {
+	for i, test := range []struct {
+		Flags jwriter.Flags
+		In    easyjson.Marshaler
+		Want  string
+	}{
+		{0, EncodingFlagsTestMap{}, `{"F":null}`},
+		{0, EncodingFlagsTestSlice{}, `{"F":null}`},
+		{jwriter.NilMapAsEmpty, EncodingFlagsTestMap{}, `{"F":{}}`},
+		{jwriter.NilSliceAsEmpty, EncodingFlagsTestSlice{}, `{"F":[]}`},
+	} {
+		w := &jwriter.Writer{Flags: test.Flags}
+		test.In.MarshalEasyJSON(w)
+
+		data, err := w.BuildBytes()
+		if err != nil {
+			t.Errorf("[%v] easyjson.Marshal(%+v) error: %v", i, test.In, err)
+		}
+
+		v := string(data)
+		if v != test.Want {
+			t.Errorf("[%v] easyjson.Marshal(%+v) = %v; want %v", i, test.In, v, test.Want)
+		}
+	}
+
+}
+
 func TestNestedEasyJsonMarshal(t *testing.T) {
 	n := map[string]*NestedEasyMarshaler{
 		"Value":  {},

tests/data.go

@@ -624,3 +624,13 @@ type RequiredOptionalStruct struct {
 	FirstName string `json:"first_name,required"`
 	Lastname  string `json:"last_name"`
 }
+
+//easyjson:json
+type EncodingFlagsTestMap struct {
+	F map[string]string
+}
+
+//easyjson:json
+type EncodingFlagsTestSlice struct {
+	F []string
+}