diff --git a/easyjson/main.go b/easyjson/main.go index 8ae9223..c9a6d9d 100644 --- a/easyjson/main.go +++ b/easyjson/main.go @@ -1,9 +1,11 @@ package main import ( + "errors" "flag" "fmt" "os" + "path/filepath" "strings" "github.com/mailru/easyjson/bootstrap" @@ -23,18 +25,28 @@ var leaveTemps = flag.Bool("leave_temps", false, "do not delete temporary files" var stubs = flag.Bool("stubs", false, "only generate stubs for marshallers/unmarshallers methods") var noformat = flag.Bool("noformat", false, "do not run 'gofmt -w' on output file") var specifiedName = flag.String("output_filename", "", "specify the filename of the output") +var processPkg = flag.Bool("pkg", false, "process the whole package instead of just the given file") func generate(fname string) (err error) { + fInfo, err := os.Stat(fname) + if err != nil { + return err + } + p := parser.Parser{AllStructs: *allStructs} - if err := p.Parse(fname); err != nil { + if err := p.Parse(fname, fInfo.IsDir()); err != nil { return fmt.Errorf("Error parsing %v: %v", fname, err) } var outName string - if s := strings.TrimSuffix(fname, ".go"); s == fname { - return fmt.Errorf("Filename must end in '.go'") + if fInfo.IsDir() { + outName = filepath.Join(fname, p.PkgName+"_easyjson.go") } else { - outName = s + "_easyjson.go" + if s := strings.TrimSuffix(fname, ".go"); s == fname { + return errors.New("Filename must end in '.go'") + } else { + outName = s + "_easyjson.go" + } } if *specifiedName != "" { @@ -67,6 +79,10 @@ func main() { files := flag.Args() gofile := os.Getenv("GOFILE") + if *processPkg { + gofile = filepath.Dir(gofile) + } + if len(files) == 0 && gofile != "" { files = []string{gofile} } else if len(files) == 0 { diff --git a/gen/decoder.go b/gen/decoder.go index 681584f..bafa162 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -340,10 +340,14 @@ func (g *Generator) genSliceArrayDecoder(t reflect.Type) error { typ := g.getType(t) fmt.Fprintln(g.out, "func "+fname+"(in *jlexer.Lexer, out *"+typ+") {") + fmt.Fprintln(g.out, " isTopLevel := in.IsStart()") err := g.genTypeDecoderNoCheck(t, "*out", fieldTags{}, 1) if err != nil { return err } + fmt.Fprintln(g.out, " if isTopLevel {") + fmt.Fprintln(g.out, " in.Consumed()") + fmt.Fprintln(g.out, " }") fmt.Fprintln(g.out, "}") return nil @@ -358,7 +362,11 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { typ := g.getType(t) fmt.Fprintln(g.out, "func "+fname+"(in *jlexer.Lexer, out *"+typ+") {") + fmt.Fprintln(g.out, " isTopLevel := in.IsStart()") fmt.Fprintln(g.out, " if in.IsNull() {") + fmt.Fprintln(g.out, " if isTopLevel {") + fmt.Fprintln(g.out, " in.Consumed()") + fmt.Fprintln(g.out, " }") fmt.Fprintln(g.out, " in.Skip()") fmt.Fprintln(g.out, " return") fmt.Fprintln(g.out, " }") @@ -404,6 +412,9 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { fmt.Fprintln(g.out, " in.WantComma()") fmt.Fprintln(g.out, " }") fmt.Fprintln(g.out, " in.Delim('}')") + fmt.Fprintln(g.out, " if isTopLevel {") + fmt.Fprintln(g.out, " in.Consumed()") + fmt.Fprintln(g.out, " }") for _, f := range fs { g.genRequiredFieldCheck(t, f) diff --git a/gen/encoder.go b/gen/encoder.go index b67f1e9..30dbbc9 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -121,7 +121,7 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT if t.Elem().Kind() == reflect.Uint8 { fmt.Fprintln(g.out, ws+"out.Base64Bytes("+in+")") } else { - fmt.Fprintln(g.out, ws+"if "+in+" == nil {") + fmt.Fprintln(g.out, ws+"if "+in+" == nil && (out.Flags & jwriter.NilSliceAsEmpty) == 0 {") fmt.Fprintln(g.out, ws+` out.RawString("null")`) fmt.Fprintln(g.out, ws+"} else {") fmt.Fprintln(g.out, ws+" out.RawByte('[')") @@ -178,7 +178,7 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT } tmpVar := g.uniqueVarName() - fmt.Fprintln(g.out, ws+"if "+in+" == nil {") + fmt.Fprintln(g.out, ws+"if "+in+" == nil && (out.Flags & jwriter.NilMapAsEmpty) == 0 {") fmt.Fprintln(g.out, ws+" out.RawString(`null`)") fmt.Fprintln(g.out, ws+"} else {") fmt.Fprintln(g.out, ws+" out.RawByte('{')") diff --git a/gen/generator.go b/gen/generator.go index 78edaff..32e806e 100644 --- a/gen/generator.go +++ b/gen/generator.go @@ -202,8 +202,18 @@ func (g *Generator) Run(out io.Writer) error { return err } +// fixes vendored paths +func fixPkgPathVendoring(pkgPath string) string { + const vendor = "/vendor/" + if i := strings.LastIndex(pkgPath, vendor); i != -1 { + return pkgPath[i+len(vendor):] + } + return pkgPath +} + // pkgAlias creates and returns and import alias for a given package. func (g *Generator) pkgAlias(pkgPath string) string { + pkgPath = fixPkgPathVendoring(pkgPath) if alias := g.imports[pkgPath]; alias != "" { return alias } diff --git a/gen/generator_test.go b/gen/generator_test.go index d7b6b0f..62c03f0 100644 --- a/gen/generator_test.go +++ b/gen/generator_test.go @@ -47,3 +47,19 @@ func TestJoinFunctionNameParts(t *testing.T) { } } } + +func TestFixVendorPath(t *testing.T) { + for i, test := range []struct { + In, Out string + }{ + {"", ""}, + {"time", "time"}, + {"project/vendor/subpackage", "subpackage"}, + } { + got := fixPkgPathVendoring(test.In) + if got != test.Out { + t.Errorf("[%d] fixPkgPathVendoring(%s) = %s; want %s", i, test.In, got, test.Out) + } + } + +} diff --git a/jlexer/lexer.go b/jlexer/lexer.go index c3d4ed2..eac6cf5 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -519,8 +519,9 @@ func (r *Lexer) SkipRecursive() { r.err = &LexerError{ Reason: "EOF reached while skipping array/object or token", Offset: r.pos, - Data: string(r.Data[r.pos:]), - }} + Data: string(r.Data[r.pos:]), + } +} // Raw fetches the next item recursively as a data slice func (r *Lexer) Raw() []byte { @@ -531,6 +532,34 @@ func (r *Lexer) Raw() []byte { return r.Data[r.start:r.pos] } +// IsStart returns whether the lexer is positioned at the start +// of an input string. +func (r *Lexer) IsStart() bool { + return r.pos == 0 +} + +// Consumed reads all remaining bytes from the input, publishing an error if +// there is anything but whitespace remaining. +func (r *Lexer) Consumed() { + if r.pos > len(r.Data) { + return + } + + for _, c := range r.Data[r.pos:] { + if c != ' ' && c != '\t' && c != '\r' && c != '\n' { + r.err = &LexerError{ + Reason: "invalid character '" + string(c) + "' after top-level value", + Offset: r.pos, + Data: string(r.Data[r.pos:]), + } + return + } + + r.pos++ + r.start++ + } +} + // UnsafeString returns the string value if the token is a string literal. // // Warning: returned string may point to the input buffer, so the string should not outlive diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index 58f5fe9..e3add8c 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -222,3 +222,27 @@ func TestInterface(t *testing.T) { } } } + +func TestConsumed(t *testing.T) { + for i, test := range []struct { + toParse string + wantError bool + }{ + {toParse: "", wantError: false}, + {toParse: " ", wantError: false}, + {toParse: "\r\n", wantError: false}, + {toParse: "\t\t", wantError: false}, + + {toParse: "{", wantError: true}, + } { + l := Lexer{Data: []byte(test.toParse)} + l.Consumed() + + err := l.Error() + if err != nil && !test.wantError { + t.Errorf("[%d, %q] Consumed() error: %v", i, test.toParse, err) + } else if err == nil && test.wantError { + t.Errorf("[%d, %q] Consumed() ok; want error", i, test.toParse) + } + } +} diff --git a/jwriter/writer.go b/jwriter/writer.go index acd670f..a3ef534 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -10,8 +10,19 @@ import ( "github.com/mailru/easyjson/buffer" ) +// Flags describe various encoding options. The behavior may be actually implemented in the encoder, but +// Flags field in Writer is used to set and pass them around. +type Flags int + +const ( + NilMapAsEmpty Flags = 1 << iota // Encode nil map as '{}' rather than 'null'. + NilSliceAsEmpty // Encode nil slice as '[]' rather than 'null'. +) + // Writer is a JSON writer. type Writer struct { + Flags Flags + Error error Buffer buffer.Buffer } diff --git a/parser/parser.go b/parser/parser.go index 69c098b..1c0b94c 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -34,6 +34,8 @@ func (p *Parser) needType(comments string) bool { func (v *visitor) Visit(n ast.Node) (w ast.Visitor) { switch n := n.(type) { + case *ast.Package: + return v case *ast.File: v.PkgName = n.Name.String() return v @@ -61,18 +63,29 @@ func (v *visitor) Visit(n ast.Node) (w ast.Visitor) { return nil } -func (p *Parser) Parse(fname string) error { +func (p *Parser) Parse(fname string, isDir bool) error { var err error - if p.PkgPath, err = getPkgPath(fname); err != nil { + if p.PkgPath, err = getPkgPath(fname, isDir); err != nil { return err } fset := token.NewFileSet() - f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments) - if err != nil { - return err - } + if isDir { + packages, err := parser.ParseDir(fset, fname, nil, parser.ParseComments) + if err != nil { + return err + } - ast.Walk(&visitor{Parser: p}, f) + for _, pckg := range packages { + ast.Walk(&visitor{Parser: p}, pckg) + } + } else { + f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments) + if err != nil { + return err + } + + ast.Walk(&visitor{Parser: p}, f) + } return nil } diff --git a/parser/parser_unix.go b/parser/parser_unix.go index 2142aaa..a1b9d84 100644 --- a/parser/parser_unix.go +++ b/parser/parser_unix.go @@ -9,7 +9,7 @@ import ( "strings" ) -func getPkgPath(fname string) (string, error) { +func getPkgPath(fname string, isDir bool) (string, error) { if !path.IsAbs(fname) { pwd, err := os.Getwd() if err != nil { @@ -21,7 +21,11 @@ func getPkgPath(fname string) (string, error) { for _, p := range strings.Split(os.Getenv("GOPATH"), ":") { prefix := path.Join(p, "src") + "/" if rel := strings.TrimPrefix(fname, prefix); rel != fname { - return path.Dir(rel), nil + if !isDir { + return path.Dir(rel), nil + } else { + return path.Clean(rel), nil + } } } diff --git a/tests/basic_test.go b/tests/basic_test.go index 8292788..25b1bfc 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -157,6 +157,33 @@ func TestUnderflowArray(t *testing.T) { } } +func TestEncodingFlags(t *testing.T) { + for i, test := range []struct { + Flags jwriter.Flags + In easyjson.Marshaler + Want string + }{ + {0, EncodingFlagsTestMap{}, `{"F":null}`}, + {0, EncodingFlagsTestSlice{}, `{"F":null}`}, + {jwriter.NilMapAsEmpty, EncodingFlagsTestMap{}, `{"F":{}}`}, + {jwriter.NilSliceAsEmpty, EncodingFlagsTestSlice{}, `{"F":[]}`}, + } { + w := &jwriter.Writer{Flags: test.Flags} + test.In.MarshalEasyJSON(w) + + data, err := w.BuildBytes() + if err != nil { + t.Errorf("[%v] easyjson.Marshal(%+v) error: %v", i, test.In, err) + } + + v := string(data) + if v != test.Want { + t.Errorf("[%v] easyjson.Marshal(%+v) = %v; want %v", i, test.In, v, test.Want) + } + } + +} + func TestNestedEasyJsonMarshal(t *testing.T) { n := map[string]*NestedEasyMarshaler{ "Value": {}, diff --git a/tests/data.go b/tests/data.go index 1716617..a46a4a5 100644 --- a/tests/data.go +++ b/tests/data.go @@ -624,3 +624,13 @@ type RequiredOptionalStruct struct { FirstName string `json:"first_name,required"` Lastname string `json:"last_name"` } + +//easyjson:json +type EncodingFlagsTestMap struct { + F map[string]string +} + +//easyjson:json +type EncodingFlagsTestSlice struct { + F []string +}