From 3b3f3665d3dcae0319bca4c4c80d693f31bdf9a9 Mon Sep 17 00:00:00 2001 From: Kenneth Shaw Date: Thu, 3 Oct 2019 07:44:02 +0700 Subject: [PATCH 01/12] Convert exec.Command for gofmt to standard go/format package Converts exec.Command call for formatting the generated Go code from the bootstrapper to use the standard `go/format` package for formatting. Fixes an issue / problem that the `cdproto-gen` tool is encountering. --- bootstrap/bootstrap.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index a461bf1..134244b 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -7,6 +7,7 @@ package bootstrap import ( "fmt" + "go/format" "io/ioutil" "os" "os/exec" @@ -176,18 +177,21 @@ func (g *Generator) Run() error { if err = cmd.Run(); err != nil { return err } - f.Close() - if !g.NoFormat { - cmd = exec.Command("gofmt", "-w", f.Name()) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - - if err = cmd.Run(); err != nil { - return err - } + // move unformatted file to out path + if g.NoFormat { + return os.Rename(f.Name(), g.OutName) } - return os.Rename(f.Name(), g.OutName) + // format file and write to out path + in, err := ioutil.ReadFile(f.Name()) + if err != nil { + return err + } + out, err := format.Source(in) + if err != nil { + return err + } + return ioutil.WriteFile(g.OutName, out, 0644) } From e6347e1e0b410cb4150218d87c46aac343c19956 Mon Sep 17 00:00:00 2001 From: Alexandr Mayorskiy Date: Tue, 8 Oct 2019 22:47:07 +0300 Subject: [PATCH 02/12] optimize writer --- jwriter/writer.go | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/jwriter/writer.go b/jwriter/writer.go index b9ed7cc..eb8547c 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -270,16 +270,25 @@ func (w *Writer) Bool(v bool) { const chars = "0123456789abcdef" -func isNotEscapedSingleChar(c byte, escapeHTML bool) bool { - // Note: might make sense to use a table if there are more chars to escape. With 4 chars - // it benchmarks the same. - if escapeHTML { - return c != '<' && c != '>' && c != '&' && c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf - } else { - return c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf +func getTable(falseValues ...int) [128]bool { + table := [128]bool{} + + for i := 0; i < 128; i++ { + table[i] = true } + + for _, v := range falseValues { + table[v] = false + } + + return table } +var ( + htmlEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '&', '<', '>', '\\') + htmlNoEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '\\') +) + func (w *Writer) String(s string) { w.Buffer.AppendByte('"') @@ -288,15 +297,23 @@ func (w *Writer) String(s string) { p := 0 // last non-escape symbol + var escapeTable [128]bool + if w.NoEscapeHTML { + escapeTable = htmlNoEscapeTable + } else { + escapeTable = htmlEscapeTable + } + for i := 0; i < len(s); { c := s[i] - if isNotEscapedSingleChar(c, !w.NoEscapeHTML) { - // single-width character, no escaping is required - i++ - continue - } else if c < utf8.RuneSelf { - // single-with character, need to escape + if c < utf8.RuneSelf { + if escapeTable[c] { + // single-width character, no escaping is required + i++ + continue + } + w.Buffer.AppendString(s[p:i]) switch c { case '\t': From 39f83e2d0b2008f9c99a80eedcbc61bb0b836c3f Mon Sep 17 00:00:00 2001 From: Alexandr Mayorskiy Date: Wed, 9 Oct 2019 11:59:07 +0300 Subject: [PATCH 03/12] add tests for htmlescaping --- Makefile | 2 ++ tests/html.go | 5 +++++ tests/html_test.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 tests/html.go create mode 100644 tests/html_test.go diff --git a/Makefile b/Makefile index 7b9ac94..66cc402 100644 --- a/Makefile +++ b/Makefile @@ -18,10 +18,12 @@ generate: build ./tests/custom_map_key_type.go \ ./tests/embedded_type.go \ ./tests/reference_to_pointer.go \ + ./tests/html.go \ bin/easyjson -all ./tests/data.go bin/easyjson -all ./tests/nothing.go bin/easyjson -all ./tests/errors.go + bin/easyjson -all ./tests/html.go bin/easyjson -snake_case ./tests/snake.go bin/easyjson -omit_empty ./tests/omitempty.go bin/easyjson -build_tags=use_easyjson ./benchmark/data.go diff --git a/tests/html.go b/tests/html.go new file mode 100644 index 0000000..575e760 --- /dev/null +++ b/tests/html.go @@ -0,0 +1,5 @@ +package tests + +type Struct struct { + Test string +} diff --git a/tests/html_test.go b/tests/html_test.go new file mode 100644 index 0000000..b579936 --- /dev/null +++ b/tests/html_test.go @@ -0,0 +1,33 @@ +package tests + +import ( + "testing" + + "github.com/mailru/easyjson/jwriter" +) + +func TestHTML(t *testing.T) { + s := Struct{ + Test: "test", + } + + j := jwriter.Writer{ + NoEscapeHTML: false, + } + s.MarshalEasyJSON(&j) + + data, _ := j.BuildBytes() + + if string(data) != `{"Test":"\u003cb\u003etest\u003c/b\u003e"}` { + t.Fatal("EscapeHTML error:", string(data)) + } + + j.NoEscapeHTML = true + s.MarshalEasyJSON(&j) + + data, _ = j.BuildBytes() + + if string(data) != `{"Test":"test"}` { + t.Fatal("NoEscapeHTML error:", string(data)) + } +} From 6892787366a955b9ce3eb5584a3bc8dd6419e721 Mon Sep 17 00:00:00 2001 From: ferhat elmas Date: Thu, 7 Nov 2019 13:21:23 +0100 Subject: [PATCH 04/12] fix some typos in readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3bdcf2d..95997ae 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ Additional option notes: ## Generated Marshaler/Unmarshaler Funcs For Go struct types, easyjson generates the funcs `MarshalEasyJSON` / -`UnmarshalEasyJSON` for marshaling/unmarshaling JSON. In turn, these satisify +`UnmarshalEasyJSON` for marshaling/unmarshaling JSON. In turn, these satisfy the `easyjson.Marshaler` and `easyjson.Unmarshaler` interfaces and when used in conjunction with `easyjson.Marshal` / `easyjson.Unmarshal` avoid unnecessary reflection / type assertions during marshaling/unmarshaling to/from JSON for Go @@ -102,17 +102,17 @@ utility funcs that are available. ## Controlling easyjson Marshaling and Unmarshaling Behavior Go types can provide their own `MarshalEasyJSON` and `UnmarshalEasyJSON` funcs -that satisify the `easyjson.Marshaler` / `easyjson.Unmarshaler` interfaces. +that satisfy the `easyjson.Marshaler` / `easyjson.Unmarshaler` interfaces. These will be used by `easyjson.Marshal` and `easyjson.Unmarshal` when defined for a Go type. -Go types can also satisify the `easyjson.Optional` interface, which allows the +Go types can also satisfy the `easyjson.Optional` interface, which allows the type to define its own `omitempty` logic. ## Type Wrappers easyjson provides additional type wrappers defined in the `easyjson/opt` -package. These wrap the standard Go primitives and in turn satisify the +package. These wrap the standard Go primitives and in turn satisfy the easyjson interfaces. The `easyjson/opt` type wrappers are useful when needing to distinguish between @@ -174,7 +174,7 @@ for more information. needs to be known prior to sending the data. Currently this is not possible with easyjson's architecture. -* easyjson parser and codegen based on reflection, so it wont works on `package main` +* easyjson parser and codegen based on reflection, so it won't work on `package main` files, because they cant be imported by parser. ## Benchmarks @@ -239,7 +239,7 @@ since the memory is not freed between marshaling operations. ### easyjson vs 'ujson' python module [ujson](https://github.com/esnme/ultrajson) is using C code for parsing, so it -is interesting to see how plain golang compares to that. It is imporant to note +is interesting to see how plain golang compares to that. It is important to note that the resulting object for python is slower to access, since the library parses JSON object into dictionaries. From d6768890eced49d7982c7f412403df21b2e47997 Mon Sep 17 00:00:00 2001 From: Aravind Gopalan Date: Thu, 21 Nov 2019 18:03:16 -0800 Subject: [PATCH 05/12] Allow empty maps for required map fields (issue #256) As reported in issue #256, currently, an empty map is unmarshalled into nil by easyjson If a field is marked "required" or "!omitempty", then the empty map is a valid input and we should unmarshal it into an empty map. Similar logic for marshalling. This change is to fix the behavior --- .gitignore | 1 + gen/decoder.go | 13 +++++++++---- tests/data.go | 6 ++++++ tests/required_test.go | 23 +++++++++++++++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 26156fb..fbfaf7a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *.iml .idea *.swp +bin/* diff --git a/gen/decoder.go b/gen/decoder.go index ab79869..cd8d42d 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -228,16 +228,21 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field } // else assume the caller knows what they are doing and that the custom unmarshaler performs the translation from string or integer keys to the key type elem := t.Elem() tmpVar := g.uniqueVarName() + keepEmpty := tags.required || tags.noOmitEmpty || (!g.omitEmpty && !tags.omitEmpty) fmt.Fprintln(g.out, ws+"if in.IsNull() {") fmt.Fprintln(g.out, ws+" in.Skip()") fmt.Fprintln(g.out, ws+"} else {") fmt.Fprintln(g.out, ws+" in.Delim('{')") - fmt.Fprintln(g.out, ws+" if !in.IsDelim('}') {") + if !keepEmpty { + fmt.Fprintln(g.out, ws+" if !in.IsDelim('}') {") + } fmt.Fprintln(g.out, ws+" "+out+" = make("+g.getType(t)+")") - fmt.Fprintln(g.out, ws+" } else {") - fmt.Fprintln(g.out, ws+" "+out+" = nil") - fmt.Fprintln(g.out, ws+" }") + if !keepEmpty { + fmt.Fprintln(g.out, ws+" } else {") + fmt.Fprintln(g.out, ws+" "+out+" = nil") + fmt.Fprintln(g.out, ws+" }") + } fmt.Fprintln(g.out, ws+" for !in.IsDelim('}') {") // NOTE: extra check for TextUnmarshaler. It overrides default methods. diff --git a/tests/data.go b/tests/data.go index 6ae90a0..5e6c610 100644 --- a/tests/data.go +++ b/tests/data.go @@ -678,6 +678,12 @@ type RequiredOptionalStruct struct { Lastname string `json:"last_name"` } +type RequiredOptionalMap struct { + ReqMap map[int]string `json:"req_map,required"` + OmitEmptyMap map[int]string `json:"oe_map,omitempty"` + NoOmitEmptyMap map[int]string `json:"noe_map,!omitempty"` +} + //easyjson:json type EncodingFlagsTestMap struct { F map[string]string diff --git a/tests/required_test.go b/tests/required_test.go index 8cc743d..36a37c8 100644 --- a/tests/required_test.go +++ b/tests/required_test.go @@ -2,6 +2,7 @@ package tests import ( "fmt" + "reflect" "testing" ) @@ -26,3 +27,25 @@ func TestRequiredField(t *testing.T) { } } } + +func TestRequiredOptionalMap(t *testing.T) { + baseJson := `{"req_map":{}, "oe_map":{}, "noe_map":{}, "oe_slice":[]}` + wantDecoding := RequiredOptionalMap{MapIntString{}, nil, MapIntString{}} + + var v RequiredOptionalMap + if err := v.UnmarshalJSON([]byte(baseJson)); err != nil { + t.Errorf("%s. UnmarshalJSON didn't expect error: %v", baseJson, err) + } + if !reflect.DeepEqual(v, wantDecoding) { + t.Errorf("%s. UnmarshalJSON expected to gen: %v. got: %v", baseJson, wantDecoding, v) + } + + baseStruct := RequiredOptionalMap{MapIntString{}, MapIntString{}, MapIntString{}} + wantJson := `{"req_map":{},"noe_map":{}}` + data, err := baseStruct.MarshalJSON() + if err != nil { + t.Errorf("MarshalJSON didn't expect error: %v on %v", err, data) + } else if string(data) != wantJson { + t.Errorf("%v. MarshalJSON wanted: %s got %s", baseStruct, wantJson, string(data)) + } +} From e17df41dc6bf789b2e0fb658a69e6ee27c540b0e Mon Sep 17 00:00:00 2001 From: Anton Sulaev Date: Mon, 17 Feb 2020 15:02:51 +0300 Subject: [PATCH 06/12] add interfaces to provide marshal/unmarshal logic for unknown fields in struct --- Makefile | 2 ++ gen/decoder.go | 12 ++++++++ gen/encoder.go | 8 ++++++ helpers.go | 10 +++++++ tests/unknown_fields.go | 17 ++++++++++++ tests/unknown_fields_test.go | 54 ++++++++++++++++++++++++++++++++++++ unknown_fields.go | 34 +++++++++++++++++++++++ 7 files changed, 137 insertions(+) create mode 100644 tests/unknown_fields.go create mode 100644 tests/unknown_fields_test.go create mode 100644 unknown_fields.go diff --git a/Makefile b/Makefile index 66cc402..80449f0 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ generate: build ./tests/embedded_type.go \ ./tests/reference_to_pointer.go \ ./tests/html.go \ + ./tests/unknown_fields.go \ bin/easyjson -all ./tests/data.go bin/easyjson -all ./tests/nothing.go @@ -34,6 +35,7 @@ generate: build bin/easyjson ./tests/reference_to_pointer.go bin/easyjson ./tests/key_marshaler_map.go bin/easyjson -disallow_unknown_fields ./tests/disallow_unknown.go + bin/easyjson ./tests/unknown_fields.go test: generate go test \ diff --git a/gen/decoder.go b/gen/decoder.go index cd8d42d..9438568 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -94,6 +94,16 @@ func hasCustomUnmarshaler(t reflect.Type) bool { t.Implements(reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()) } +func hasUnknownsUnmarshaler(t reflect.Type) bool { + t = reflect.PtrTo(t) + return t.Implements(reflect.TypeOf((*easyjson.UnknownsUnmarshaler)(nil)).Elem()) +} + +func hasUnknownsMarshaler(t reflect.Type) bool { + t = reflect.PtrTo(t) + return t.Implements(reflect.TypeOf((*easyjson.UnknownsMarshaler)(nil)).Elem()) +} + // genTypeDecoderNoCheck generates decoding code for the type t. func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags fieldTags, indent int) error { ws := strings.Repeat(" ", indent) @@ -485,6 +495,8 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { Reason: "unknown field", Data: key, })`) + } else if hasUnknownsUnmarshaler(t) { + fmt.Fprintln(g.out, " out.UnmarshalUnknown(in, key)") } else { fmt.Fprintln(g.out, " in.SkipRecursive()") } diff --git a/gen/encoder.go b/gen/encoder.go index e86d531..6274d4f 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -393,6 +393,14 @@ func (g *Generator) genStructEncoder(t reflect.Type) error { } } + if hasUnknownsMarshaler(t) { + if !firstCondition { + fmt.Fprintln(g.out, " in.MarshalUnknowns(out, false)") + } else { + fmt.Fprintln(g.out, " in.MarshalUnknowns(out, first)") + } + } + fmt.Fprintln(g.out, " out.RawByte('}')") fmt.Fprintln(g.out, "}") diff --git a/helpers.go b/helpers.go index b86b87d..04ac635 100644 --- a/helpers.go +++ b/helpers.go @@ -26,6 +26,16 @@ type Optional interface { IsDefined() bool } +// UnknownsUnmarshaler provides a method to unmarshal unknown struct fileds and save them as you want +type UnknownsUnmarshaler interface { + UnmarshalUnknown(in *jlexer.Lexer, key string) +} + +// UnknownsMarshaler provides a method to write additional struct fields +type UnknownsMarshaler interface { + MarshalUnknowns(w *jwriter.Writer, first bool) +} + // Marshal returns data as a single byte slice. Method is suboptimal as the data is likely to be copied // from a chain of smaller chunks. func Marshal(v Marshaler) ([]byte, error) { diff --git a/tests/unknown_fields.go b/tests/unknown_fields.go new file mode 100644 index 0000000..3d1b089 --- /dev/null +++ b/tests/unknown_fields.go @@ -0,0 +1,17 @@ +package tests + +import "github.com/mailru/easyjson" + +//easyjson:json +type StructWithUnknownsProxy struct { + easyjson.UnknownFieldsProxy + + Field1 string +} + +//easyjson:json +type StructWithUnknownsProxyWithOmitempty struct { + easyjson.UnknownFieldsProxy + + Field1 string `json:",omitempty"` +} diff --git a/tests/unknown_fields_test.go b/tests/unknown_fields_test.go new file mode 100644 index 0000000..fd1114f --- /dev/null +++ b/tests/unknown_fields_test.go @@ -0,0 +1,54 @@ +package tests + +import ( + "reflect" + "testing" +) + +func TestUnknownFieldsProxy(t *testing.T) { + baseJson := `{"Field1":"123","Field2":"321"}` + + s := StructWithUnknownsProxy{} + + err := s.UnmarshalJSON([]byte(baseJson)) + if err != nil { + t.Errorf("UnmarshalJSON didn't expect error: %v", err) + } + + if s.Field1 != "123" { + t.Errorf("UnmarshalJSON expected to parse Field1 as \"123\". got: %v", s.Field1) + } + + data, err := s.MarshalJSON() + if err != nil { + t.Errorf("MarshalJSON didn't expect error: %v", err) + } + + if !reflect.DeepEqual(baseJson, string(data)) { + t.Errorf("MarshalJSON expected to gen: %v. got: %v", baseJson, string(data)) + } +} + +func TestUnknownFieldsProxyWithOmitempty(t *testing.T) { + baseJson := `{"Field1":"123","Field2":"321"}` + + s := StructWithUnknownsProxyWithOmitempty{} + + err := s.UnmarshalJSON([]byte(baseJson)) + if err != nil { + t.Errorf("UnmarshalJSON didn't expect error: %v", err) + } + + if s.Field1 != "123" { + t.Errorf("UnmarshalJSON expected to parse Field1 as \"123\". got: %v", s.Field1) + } + + data, err := s.MarshalJSON() + if err != nil { + t.Errorf("MarshalJSON didn't expect error: %v", err) + } + + if !reflect.DeepEqual(baseJson, string(data)) { + t.Errorf("MarshalJSON expected to gen: %v. got: %v", baseJson, string(data)) + } +} diff --git a/unknown_fields.go b/unknown_fields.go new file mode 100644 index 0000000..6cfdf83 --- /dev/null +++ b/unknown_fields.go @@ -0,0 +1,34 @@ +package easyjson + +import ( + json "encoding/json" + + jlexer "github.com/mailru/easyjson/jlexer" + "github.com/mailru/easyjson/jwriter" +) + +// UnknownFieldsProxy implemets UnknownsUnmarshaler and UnknownsMarshaler +// use it as embedded field in your structure to parse and then serialize unknown struct fields +type UnknownFieldsProxy struct { + unknownFields map[string]interface{} +} + +func (s *UnknownFieldsProxy) UnmarshalUnknown(in *jlexer.Lexer, key string) { + if s.unknownFields == nil { + s.unknownFields = make(map[string]interface{}, 1) + } + s.unknownFields[key] = in.Interface() +} + +func (s UnknownFieldsProxy) MarshalUnknowns(out *jwriter.Writer, first bool) { + for key, val := range s.unknownFields { + if first { + first = false + } else { + out.RawByte(',') + } + out.String(string(key)) + out.RawByte(':') + out.Raw(json.Marshal(val)) + } +} From 323cc237497f3e69d09df489328dae71316be11a Mon Sep 17 00:00:00 2001 From: Anton Sulaev Date: Tue, 18 Feb 2020 13:46:49 +0300 Subject: [PATCH 07/12] optimize deafult UnknownFieldsProxy realisations with []byte in place of interface{} --- unknown_fields.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/unknown_fields.go b/unknown_fields.go index 6cfdf83..55538ea 100644 --- a/unknown_fields.go +++ b/unknown_fields.go @@ -1,8 +1,6 @@ package easyjson import ( - json "encoding/json" - jlexer "github.com/mailru/easyjson/jlexer" "github.com/mailru/easyjson/jwriter" ) @@ -10,14 +8,14 @@ import ( // UnknownFieldsProxy implemets UnknownsUnmarshaler and UnknownsMarshaler // use it as embedded field in your structure to parse and then serialize unknown struct fields type UnknownFieldsProxy struct { - unknownFields map[string]interface{} + unknownFields map[string][]byte } func (s *UnknownFieldsProxy) UnmarshalUnknown(in *jlexer.Lexer, key string) { if s.unknownFields == nil { - s.unknownFields = make(map[string]interface{}, 1) + s.unknownFields = make(map[string][]byte, 1) } - s.unknownFields[key] = in.Interface() + s.unknownFields[key] = in.Raw() } func (s UnknownFieldsProxy) MarshalUnknowns(out *jwriter.Writer, first bool) { @@ -29,6 +27,6 @@ func (s UnknownFieldsProxy) MarshalUnknowns(out *jwriter.Writer, first bool) { } out.String(string(key)) out.RawByte(':') - out.Raw(json.Marshal(val)) + out.Raw(val, nil) } } From 52fd0e53caf3bba4f2dd5c69880f8bb6caecd03a Mon Sep 17 00:00:00 2001 From: Carlo Alberto Ferraris Date: Fri, 28 Feb 2020 19:04:54 +0900 Subject: [PATCH 08/12] Make Buffer.EnsureSpace inlineable Split the slow path into a separate function, so that the fast path in EnsureSpace becomes inlineable. This allows code in jwriter to inline the fast path. --- buffer/pool.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/buffer/pool.go b/buffer/pool.go index 07fb4bc..4c508f7 100644 --- a/buffer/pool.go +++ b/buffer/pool.go @@ -78,9 +78,12 @@ type Buffer struct { // EnsureSpace makes sure that the current chunk contains at least s free bytes, // possibly creating a new chunk. func (b *Buffer) EnsureSpace(s int) { - if cap(b.Buf)-len(b.Buf) >= s { - return + if cap(b.Buf)-len(b.Buf) < s { + b.ensureSpaceSlow(s) } +} + +func (b *Buffer) ensureSpaceSlow(s int) { l := len(b.Buf) if l > 0 { if cap(b.toPool) != cap(b.Buf) { @@ -105,18 +108,14 @@ func (b *Buffer) EnsureSpace(s int) { // AppendByte appends a single byte to buffer. func (b *Buffer) AppendByte(data byte) { - if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined. - b.EnsureSpace(1) - } + b.EnsureSpace(1) b.Buf = append(b.Buf, data) } // AppendBytes appends a byte slice to buffer. func (b *Buffer) AppendBytes(data []byte) { for len(data) > 0 { - if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined. - b.EnsureSpace(1) - } + b.EnsureSpace(1) sz := cap(b.Buf) - len(b.Buf) if sz > len(data) { @@ -131,9 +130,7 @@ func (b *Buffer) AppendBytes(data []byte) { // AppendBytes appends a string to buffer. func (b *Buffer) AppendString(data string) { for len(data) > 0 { - if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined. - b.EnsureSpace(1) - } + b.EnsureSpace(1) sz := cap(b.Buf) - len(b.Buf) if sz > len(data) { From 6f81292b372a4f63213763bca66a7211a26ec88b Mon Sep 17 00:00:00 2001 From: Carlo Alberto Ferraris Date: Fri, 28 Feb 2020 19:33:23 +0900 Subject: [PATCH 09/12] wip --- buffer/pool.go | 18 +++++++++++++++++- buffer/pool_test.go | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/buffer/pool.go b/buffer/pool.go index 07fb4bc..5e97991 100644 --- a/buffer/pool.go +++ b/buffer/pool.go @@ -113,6 +113,14 @@ func (b *Buffer) AppendByte(data byte) { // AppendBytes appends a byte slice to buffer. func (b *Buffer) AppendBytes(data []byte) { + if len(data) <= cap(b.Buf)-len(b.Buf) { + b.Buf = append(b.Buf, data...) // fast path + } else { + b.appendBytesSlow(data) + } +} + +func (b *Buffer) appendBytesSlow(data []byte) { for len(data) > 0 { if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined. b.EnsureSpace(1) @@ -128,8 +136,16 @@ func (b *Buffer) AppendBytes(data []byte) { } } -// AppendBytes appends a string to buffer. +// AppendString appends a string to buffer. func (b *Buffer) AppendString(data string) { + if len(data) <= cap(b.Buf)-len(b.Buf) { + b.Buf = append(b.Buf, data...) // fast path + } else { + b.appendStringSlow(data) + } +} + +func (b *Buffer) appendStringSlow(data string) { for len(data) > 0 { if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined. b.EnsureSpace(1) diff --git a/buffer/pool_test.go b/buffer/pool_test.go index 680623a..1f321d3 100644 --- a/buffer/pool_test.go +++ b/buffer/pool_test.go @@ -42,7 +42,7 @@ func TestAppendString(t *testing.T) { s := "test" for i := 0; i < 1000; i++ { - b.AppendBytes([]byte(s)) + b.AppendString(s) want = append(want, s...) } From 03e03cfa69bb3277cda8cb0c66c96c3c0b5edd9c Mon Sep 17 00:00:00 2001 From: Carlo Alberto Ferraris Date: Sat, 29 Feb 2020 16:28:16 +0900 Subject: [PATCH 10/12] Do not copy escape tables --- jwriter/writer.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jwriter/writer.go b/jwriter/writer.go index eb8547c..2c5b201 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -297,11 +297,9 @@ func (w *Writer) String(s string) { p := 0 // last non-escape symbol - var escapeTable [128]bool + escapeTable := &htmlEscapeTable if w.NoEscapeHTML { - escapeTable = htmlNoEscapeTable - } else { - escapeTable = htmlEscapeTable + escapeTable = &htmlNoEscapeTable } for i := 0; i < len(s); { From 237a09852687bcdc0a0f29944051f270a912e082 Mon Sep 17 00:00:00 2001 From: Ivan Boyarkin Date: Fri, 14 Feb 2020 09:31:50 +0100 Subject: [PATCH 11/12] parser: Fix go.mod endline comments parsing It's required because: - go.mod file can contain comments - easyjson should respect comments and be able to parse go.mod This commit replaces the logic of parsing go.mod file with the original one from golang.org/x/mod/modfile package. Tests have been introduced to check the behavior of `getModulePath` function. Ref: - https://golang.org/cmd/go/#hdr-The_go_mod_file --- parser/modulepath.go | 82 +++++++++++++++++++++++++++ parser/pkgpath.go | 33 +---------- parser/pkgpath_test.go | 39 +++++++++++++ parser/testdata/comments.go.mod | 4 ++ parser/testdata/comments_deps.go.mod | 8 +++ parser/testdata/default.go.mod | 3 + parser/testdata/missing_module.go.mod | 6 ++ 7 files changed, 143 insertions(+), 32 deletions(-) create mode 100644 parser/modulepath.go create mode 100644 parser/pkgpath_test.go create mode 100644 parser/testdata/comments.go.mod create mode 100644 parser/testdata/comments_deps.go.mod create mode 100644 parser/testdata/default.go.mod create mode 100644 parser/testdata/missing_module.go.mod diff --git a/parser/modulepath.go b/parser/modulepath.go new file mode 100644 index 0000000..3f8e7ca --- /dev/null +++ b/parser/modulepath.go @@ -0,0 +1,82 @@ +package parser + +import ( + "bytes" + "strconv" +) + +// Content of this file was copied from the package golang.org/x/mod/modfile +// https://github.com/golang/mod/blob/v0.2.0/modfile/read.go#L877 +// Under the BSD-3-Clause licence: +// golang.org/x/mod@v0.2.0/LICENSE +/* +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +var ( + slashSlash = []byte("//") + moduleStr = []byte("module") +) + +// modulePath returns the module path from the gomod file text. +// If it cannot find a module path, it returns an empty string. +// It is tolerant of unrelated problems in the go.mod file. +func modulePath(mod []byte) string { + for len(mod) > 0 { + line := mod + mod = nil + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, mod = line[:i], line[i+1:] + } + if i := bytes.Index(line, slashSlash); i >= 0 { + line = line[:i] + } + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, moduleStr) { + continue + } + line = line[len(moduleStr):] + n := len(line) + line = bytes.TrimSpace(line) + if len(line) == n || len(line) == 0 { + continue + } + + if line[0] == '"' || line[0] == '`' { + p, err := strconv.Unquote(string(line)) + if err != nil { + return "" // malformed quoted string or multiline module path + } + return p + } + + return string(line) + } + return "" // missing module path +} diff --git a/parser/pkgpath.go b/parser/pkgpath.go index 155d168..49f6efd 100644 --- a/parser/pkgpath.go +++ b/parser/pkgpath.go @@ -8,7 +8,6 @@ import ( "os/exec" "path" "path/filepath" - "strconv" "strings" "sync" ) @@ -91,7 +90,6 @@ func getPkgPathFromGoMod(fname string, isDir bool, goModPath string) (string, er } var ( - modulePrefix = []byte("\nmodule ") pkgPathFromGoModCache = make(map[string]string) ) @@ -109,36 +107,7 @@ func getModulePath(goModPath string) string { if err != nil { return "" } - var i int - if bytes.HasPrefix(data, modulePrefix[1:]) { - i = 0 - } else { - i = bytes.Index(data, modulePrefix) - if i < 0 { - return "" - } - i++ - } - line := data[i:] - - // Cut line at \n, drop trailing \r if present. - if j := bytes.IndexByte(line, '\n'); j >= 0 { - line = line[:j] - } - if line[len(line)-1] == '\r' { - line = line[:len(line)-1] - } - line = line[len("module "):] - - // If quoted, unquote. - pkgPath = strings.TrimSpace(string(line)) - if pkgPath != "" && pkgPath[0] == '"' { - s, err := strconv.Unquote(pkgPath) - if err != nil { - return "" - } - pkgPath = s - } + pkgPath = modulePath(data) return pkgPath } diff --git a/parser/pkgpath_test.go b/parser/pkgpath_test.go new file mode 100644 index 0000000..740f3d7 --- /dev/null +++ b/parser/pkgpath_test.go @@ -0,0 +1,39 @@ +package parser + +import "testing" + +func Test_getModulePath(t *testing.T) { + tests := map[string]struct { + goModPath string + want string + }{ + "valid go.mod without comments and deps": { + goModPath: "./testdata/default.go.mod", + want: "example.com/user/project", + }, + "valid go.mod with comments and without deps": { + goModPath: "./testdata/comments.go.mod", + want: "example.com/user/project", + }, + "valid go.mod with comments and deps": { + goModPath: "./testdata/comments_deps.go.mod", + want: "example.com/user/project", + }, + "actual easyjson go.mod": { + goModPath: "../go.mod", + want: "github.com/mailru/easyjson", + }, + "invalid go.mod with missing module": { + goModPath: "./testdata/missing_module.go", + want: "", + }, + } + for name := range tests { + tt := tests[name] + t.Run(name, func(t *testing.T) { + if got := getModulePath(tt.goModPath); got != tt.want { + t.Errorf("getModulePath() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/parser/testdata/comments.go.mod b/parser/testdata/comments.go.mod new file mode 100644 index 0000000..b42beb9 --- /dev/null +++ b/parser/testdata/comments.go.mod @@ -0,0 +1,4 @@ +// first-line comment which should bresk anything +module example.com/user/project // end-line comment which should not break anything + +go 1.13 diff --git a/parser/testdata/comments_deps.go.mod b/parser/testdata/comments_deps.go.mod new file mode 100644 index 0000000..6e2ea62 --- /dev/null +++ b/parser/testdata/comments_deps.go.mod @@ -0,0 +1,8 @@ +// first-line comment which should bresk anything +module example.com/user/project // end-line comment which should not break anything + +go 1.13 + +require ( + github.com/mailru/easyjson v0.7.0 +) diff --git a/parser/testdata/default.go.mod b/parser/testdata/default.go.mod new file mode 100644 index 0000000..77f4317 --- /dev/null +++ b/parser/testdata/default.go.mod @@ -0,0 +1,3 @@ +module example.com/user/project + +go 1.13 diff --git a/parser/testdata/missing_module.go.mod b/parser/testdata/missing_module.go.mod new file mode 100644 index 0000000..4e0332f --- /dev/null +++ b/parser/testdata/missing_module.go.mod @@ -0,0 +1,6 @@ + +go 1.13 + +require ( + github.com/mailru/easyjson v0.7.0 +) From 11e4deeba6402724c7fea2d096ffb6c8479d543c Mon Sep 17 00:00:00 2001 From: Carlo Alberto Ferraris Date: Fri, 28 Feb 2020 20:07:29 +0900 Subject: [PATCH 12/12] Use net.Buffers in Buffer.DumpTo --- buffer/pool.go | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/buffer/pool.go b/buffer/pool.go index 07fb4bc..57dcfd5 100644 --- a/buffer/pool.go +++ b/buffer/pool.go @@ -4,6 +4,7 @@ package buffer import ( "io" + "net" "sync" ) @@ -52,14 +53,12 @@ func putBuf(buf []byte) { // getBuf gets a chunk from reuse pool or creates a new one if reuse failed. func getBuf(size int) []byte { - if size < config.PooledSize { - return make([]byte, 0, size) - } - - if c := buffers[size]; c != nil { - v := c.Get() - if v != nil { - return v.([]byte) + if size >= config.PooledSize { + if c := buffers[size]; c != nil { + v := c.Get() + if v != nil { + return v.([]byte) + } } } return make([]byte, 0, size) @@ -156,18 +155,14 @@ func (b *Buffer) Size() int { // DumpTo outputs the contents of a buffer to a writer and resets the buffer. func (b *Buffer) DumpTo(w io.Writer) (written int, err error) { - var n int - for _, buf := range b.bufs { - if err == nil { - n, err = w.Write(buf) - written += n - } - putBuf(buf) + bufs := net.Buffers(b.bufs) + if len(b.Buf) > 0 { + bufs = append(bufs, b.Buf) } + n, err := bufs.WriteTo(w) - if err == nil { - n, err = w.Write(b.Buf) - written += n + for _, buf := range b.bufs { + putBuf(buf) } putBuf(b.toPool) @@ -175,7 +170,7 @@ func (b *Buffer) DumpTo(w io.Writer) (written int, err error) { b.Buf = nil b.toPool = nil - return + return int(n), err } // BuildBytes creates a single byte slice with all the contents of the buffer. Data is @@ -192,7 +187,7 @@ func (b *Buffer) BuildBytes(reuse ...[]byte) []byte { var ret []byte size := b.Size() - // If we got a buffer as argument and it is big enought, reuse it. + // If we got a buffer as argument and it is big enough, reuse it. if len(reuse) == 1 && cap(reuse[0]) >= size { ret = reuse[0][:0] } else {