Merge branch 'master' into fix-gomod-concurrency

This commit is contained in:
GoWebProd
2020-03-30 23:02:18 +03:00
committed by GitHub
24 changed files with 446 additions and 99 deletions
+1
View File
@@ -3,3 +3,4 @@
*.iml
.idea
*.swp
bin/*
+4
View File
@@ -18,10 +18,13 @@ generate: build
./tests/custom_map_key_type.go \
./tests/embedded_type.go \
./tests/reference_to_pointer.go \
./tests/html.go \
./tests/unknown_fields.go \
bin/easyjson -all ./tests/data.go
bin/easyjson -all ./tests/nothing.go
bin/easyjson -all ./tests/errors.go
bin/easyjson -all ./tests/html.go
bin/easyjson -snake_case ./tests/snake.go
bin/easyjson -omit_empty ./tests/omitempty.go
bin/easyjson -build_tags=use_easyjson ./benchmark/data.go
@@ -32,6 +35,7 @@ generate: build
bin/easyjson ./tests/reference_to_pointer.go
bin/easyjson ./tests/key_marshaler_map.go
bin/easyjson -disallow_unknown_fields ./tests/disallow_unknown.go
bin/easyjson ./tests/unknown_fields.go
test: generate
go test \
+6 -6
View File
@@ -79,7 +79,7 @@ Additional option notes:
## Generated Marshaler/Unmarshaler Funcs
For Go struct types, easyjson generates the funcs `MarshalEasyJSON` /
`UnmarshalEasyJSON` for marshaling/unmarshaling JSON. In turn, these satisify
`UnmarshalEasyJSON` for marshaling/unmarshaling JSON. In turn, these satisfy
the `easyjson.Marshaler` and `easyjson.Unmarshaler` interfaces and when used in
conjunction with `easyjson.Marshal` / `easyjson.Unmarshal` avoid unnecessary
reflection / type assertions during marshaling/unmarshaling to/from JSON for Go
@@ -102,17 +102,17 @@ utility funcs that are available.
## Controlling easyjson Marshaling and Unmarshaling Behavior
Go types can provide their own `MarshalEasyJSON` and `UnmarshalEasyJSON` funcs
that satisify the `easyjson.Marshaler` / `easyjson.Unmarshaler` interfaces.
that satisfy the `easyjson.Marshaler` / `easyjson.Unmarshaler` interfaces.
These will be used by `easyjson.Marshal` and `easyjson.Unmarshal` when defined
for a Go type.
Go types can also satisify the `easyjson.Optional` interface, which allows the
Go types can also satisfy the `easyjson.Optional` interface, which allows the
type to define its own `omitempty` logic.
## Type Wrappers
easyjson provides additional type wrappers defined in the `easyjson/opt`
package. These wrap the standard Go primitives and in turn satisify the
package. These wrap the standard Go primitives and in turn satisfy the
easyjson interfaces.
The `easyjson/opt` type wrappers are useful when needing to distinguish between
@@ -174,7 +174,7 @@ for more information.
needs to be known prior to sending the data. Currently this is not possible
with easyjson's architecture.
* easyjson parser and codegen based on reflection, so it wont works on `package main`
* easyjson parser and codegen based on reflection, so it won't work on `package main`
files, because they cant be imported by parser.
## Benchmarks
@@ -239,7 +239,7 @@ since the memory is not freed between marshaling operations.
### easyjson vs 'ujson' python module
[ujson](https://github.com/esnme/ultrajson) is using C code for parsing, so it
is interesting to see how plain golang compares to that. It is imporant to note
is interesting to see how plain golang compares to that. It is important to note
that the resulting object for python is slower to access, since the library
parses JSON object into dictionaries.
+14 -10
View File
@@ -7,6 +7,7 @@ package bootstrap
import (
"fmt"
"go/format"
"io/ioutil"
"os"
"os/exec"
@@ -176,18 +177,21 @@ func (g *Generator) Run() error {
if err = cmd.Run(); err != nil {
return err
}
f.Close()
if !g.NoFormat {
cmd = exec.Command("gofmt", "-w", f.Name())
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
if err = cmd.Run(); err != nil {
return err
}
// move unformatted file to out path
if g.NoFormat {
return os.Rename(f.Name(), g.OutName)
}
return os.Rename(f.Name(), g.OutName)
// format file and write to out path
in, err := ioutil.ReadFile(f.Name())
if err != nil {
return err
}
out, err := format.Source(in)
if err != nil {
return err
}
return ioutil.WriteFile(g.OutName, out, 0644)
}
+40 -32
View File
@@ -4,6 +4,7 @@ package buffer
import (
"io"
"net"
"sync"
)
@@ -52,14 +53,12 @@ func putBuf(buf []byte) {
// getBuf gets a chunk from reuse pool or creates a new one if reuse failed.
func getBuf(size int) []byte {
if size < config.PooledSize {
return make([]byte, 0, size)
}
if c := buffers[size]; c != nil {
v := c.Get()
if v != nil {
return v.([]byte)
if size >= config.PooledSize {
if c := buffers[size]; c != nil {
v := c.Get()
if v != nil {
return v.([]byte)
}
}
}
return make([]byte, 0, size)
@@ -78,9 +77,12 @@ type Buffer struct {
// EnsureSpace makes sure that the current chunk contains at least s free bytes,
// possibly creating a new chunk.
func (b *Buffer) EnsureSpace(s int) {
if cap(b.Buf)-len(b.Buf) >= s {
return
if cap(b.Buf)-len(b.Buf) < s {
b.ensureSpaceSlow(s)
}
}
func (b *Buffer) ensureSpaceSlow(s int) {
l := len(b.Buf)
if l > 0 {
if cap(b.toPool) != cap(b.Buf) {
@@ -105,18 +107,22 @@ func (b *Buffer) EnsureSpace(s int) {
// AppendByte appends a single byte to buffer.
func (b *Buffer) AppendByte(data byte) {
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
b.EnsureSpace(1)
}
b.EnsureSpace(1)
b.Buf = append(b.Buf, data)
}
// AppendBytes appends a byte slice to buffer.
func (b *Buffer) AppendBytes(data []byte) {
if len(data) <= cap(b.Buf)-len(b.Buf) {
b.Buf = append(b.Buf, data...) // fast path
} else {
b.appendBytesSlow(data)
}
}
func (b *Buffer) appendBytesSlow(data []byte) {
for len(data) > 0 {
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
b.EnsureSpace(1)
}
b.EnsureSpace(1)
sz := cap(b.Buf) - len(b.Buf)
if sz > len(data) {
@@ -128,12 +134,18 @@ func (b *Buffer) AppendBytes(data []byte) {
}
}
// AppendBytes appends a string to buffer.
// AppendString appends a string to buffer.
func (b *Buffer) AppendString(data string) {
if len(data) <= cap(b.Buf)-len(b.Buf) {
b.Buf = append(b.Buf, data...) // fast path
} else {
b.appendStringSlow(data)
}
}
func (b *Buffer) appendStringSlow(data string) {
for len(data) > 0 {
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
b.EnsureSpace(1)
}
b.EnsureSpace(1)
sz := cap(b.Buf) - len(b.Buf)
if sz > len(data) {
@@ -156,18 +168,14 @@ func (b *Buffer) Size() int {
// DumpTo outputs the contents of a buffer to a writer and resets the buffer.
func (b *Buffer) DumpTo(w io.Writer) (written int, err error) {
var n int
for _, buf := range b.bufs {
if err == nil {
n, err = w.Write(buf)
written += n
}
putBuf(buf)
bufs := net.Buffers(b.bufs)
if len(b.Buf) > 0 {
bufs = append(bufs, b.Buf)
}
n, err := bufs.WriteTo(w)
if err == nil {
n, err = w.Write(b.Buf)
written += n
for _, buf := range b.bufs {
putBuf(buf)
}
putBuf(b.toPool)
@@ -175,7 +183,7 @@ func (b *Buffer) DumpTo(w io.Writer) (written int, err error) {
b.Buf = nil
b.toPool = nil
return
return int(n), err
}
// BuildBytes creates a single byte slice with all the contents of the buffer. Data is
@@ -192,7 +200,7 @@ func (b *Buffer) BuildBytes(reuse ...[]byte) []byte {
var ret []byte
size := b.Size()
// If we got a buffer as argument and it is big enought, reuse it.
// If we got a buffer as argument and it is big enough, reuse it.
if len(reuse) == 1 && cap(reuse[0]) >= size {
ret = reuse[0][:0]
} else {
+1 -1
View File
@@ -42,7 +42,7 @@ func TestAppendString(t *testing.T) {
s := "test"
for i := 0; i < 1000; i++ {
b.AppendBytes([]byte(s))
b.AppendString(s)
want = append(want, s...)
}
+21 -4
View File
@@ -94,6 +94,16 @@ func hasCustomUnmarshaler(t reflect.Type) bool {
t.Implements(reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem())
}
func hasUnknownsUnmarshaler(t reflect.Type) bool {
t = reflect.PtrTo(t)
return t.Implements(reflect.TypeOf((*easyjson.UnknownsUnmarshaler)(nil)).Elem())
}
func hasUnknownsMarshaler(t reflect.Type) bool {
t = reflect.PtrTo(t)
return t.Implements(reflect.TypeOf((*easyjson.UnknownsMarshaler)(nil)).Elem())
}
// genTypeDecoderNoCheck generates decoding code for the type t.
func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags fieldTags, indent int) error {
ws := strings.Repeat(" ", indent)
@@ -228,16 +238,21 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
} // else assume the caller knows what they are doing and that the custom unmarshaler performs the translation from string or integer keys to the key type
elem := t.Elem()
tmpVar := g.uniqueVarName()
keepEmpty := tags.required || tags.noOmitEmpty || (!g.omitEmpty && !tags.omitEmpty)
fmt.Fprintln(g.out, ws+"if in.IsNull() {")
fmt.Fprintln(g.out, ws+" in.Skip()")
fmt.Fprintln(g.out, ws+"} else {")
fmt.Fprintln(g.out, ws+" in.Delim('{')")
fmt.Fprintln(g.out, ws+" if !in.IsDelim('}') {")
if !keepEmpty {
fmt.Fprintln(g.out, ws+" if !in.IsDelim('}') {")
}
fmt.Fprintln(g.out, ws+" "+out+" = make("+g.getType(t)+")")
fmt.Fprintln(g.out, ws+" } else {")
fmt.Fprintln(g.out, ws+" "+out+" = nil")
fmt.Fprintln(g.out, ws+" }")
if !keepEmpty {
fmt.Fprintln(g.out, ws+" } else {")
fmt.Fprintln(g.out, ws+" "+out+" = nil")
fmt.Fprintln(g.out, ws+" }")
}
fmt.Fprintln(g.out, ws+" for !in.IsDelim('}') {")
// NOTE: extra check for TextUnmarshaler. It overrides default methods.
@@ -480,6 +495,8 @@ func (g *Generator) genStructDecoder(t reflect.Type) error {
Reason: "unknown field",
Data: key,
})`)
} else if hasUnknownsUnmarshaler(t) {
fmt.Fprintln(g.out, " out.UnmarshalUnknown(in, key)")
} else {
fmt.Fprintln(g.out, " in.SkipRecursive()")
}
+8
View File
@@ -393,6 +393,14 @@ func (g *Generator) genStructEncoder(t reflect.Type) error {
}
}
if hasUnknownsMarshaler(t) {
if !firstCondition {
fmt.Fprintln(g.out, " in.MarshalUnknowns(out, false)")
} else {
fmt.Fprintln(g.out, " in.MarshalUnknowns(out, first)")
}
}
fmt.Fprintln(g.out, " out.RawByte('}')")
fmt.Fprintln(g.out, "}")
+10
View File
@@ -26,6 +26,16 @@ type Optional interface {
IsDefined() bool
}
// UnknownsUnmarshaler provides a method to unmarshal unknown struct fileds and save them as you want
type UnknownsUnmarshaler interface {
UnmarshalUnknown(in *jlexer.Lexer, key string)
}
// UnknownsMarshaler provides a method to write additional struct fields
type UnknownsMarshaler interface {
MarshalUnknowns(w *jwriter.Writer, first bool)
}
// Marshal returns data as a single byte slice. Method is suboptimal as the data is likely to be copied
// from a chain of smaller chunks.
func Marshal(v Marshaler) ([]byte, error) {
+28 -13
View File
@@ -270,16 +270,25 @@ func (w *Writer) Bool(v bool) {
const chars = "0123456789abcdef"
func isNotEscapedSingleChar(c byte, escapeHTML bool) bool {
// Note: might make sense to use a table if there are more chars to escape. With 4 chars
// it benchmarks the same.
if escapeHTML {
return c != '<' && c != '>' && c != '&' && c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf
} else {
return c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf
func getTable(falseValues ...int) [128]bool {
table := [128]bool{}
for i := 0; i < 128; i++ {
table[i] = true
}
for _, v := range falseValues {
table[v] = false
}
return table
}
var (
htmlEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '&', '<', '>', '\\')
htmlNoEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '\\')
)
func (w *Writer) String(s string) {
w.Buffer.AppendByte('"')
@@ -288,15 +297,21 @@ func (w *Writer) String(s string) {
p := 0 // last non-escape symbol
escapeTable := &htmlEscapeTable
if w.NoEscapeHTML {
escapeTable = &htmlNoEscapeTable
}
for i := 0; i < len(s); {
c := s[i]
if isNotEscapedSingleChar(c, !w.NoEscapeHTML) {
// single-width character, no escaping is required
i++
continue
} else if c < utf8.RuneSelf {
// single-with character, need to escape
if c < utf8.RuneSelf {
if escapeTable[c] {
// single-width character, no escaping is required
i++
continue
}
w.Buffer.AppendString(s[p:i])
switch c {
case '\t':
+82
View File
@@ -0,0 +1,82 @@
package parser
import (
"bytes"
"strconv"
)
// Content of this file was copied from the package golang.org/x/mod/modfile
// https://github.com/golang/mod/blob/v0.2.0/modfile/read.go#L877
// Under the BSD-3-Clause licence:
// golang.org/x/mod@v0.2.0/LICENSE
/*
Copyright (c) 2009 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
var (
slashSlash = []byte("//")
moduleStr = []byte("module")
)
// modulePath returns the module path from the gomod file text.
// If it cannot find a module path, it returns an empty string.
// It is tolerant of unrelated problems in the go.mod file.
func modulePath(mod []byte) string {
for len(mod) > 0 {
line := mod
mod = nil
if i := bytes.IndexByte(line, '\n'); i >= 0 {
line, mod = line[:i], line[i+1:]
}
if i := bytes.Index(line, slashSlash); i >= 0 {
line = line[:i]
}
line = bytes.TrimSpace(line)
if !bytes.HasPrefix(line, moduleStr) {
continue
}
line = line[len(moduleStr):]
n := len(line)
line = bytes.TrimSpace(line)
if len(line) == n || len(line) == 0 {
continue
}
if line[0] == '"' || line[0] == '`' {
p, err := strconv.Unquote(string(line))
if err != nil {
return "" // malformed quoted string or multiline module path
}
return p
}
return string(line)
}
return "" // missing module path
}
+1 -33
View File
@@ -8,7 +8,6 @@ import (
"os/exec"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
)
@@ -90,8 +89,6 @@ func getPkgPathFromGoMod(fname string, isDir bool, goModPath string) (string, er
return path.Clean(rel), nil
}
var modulePrefix = []byte("\nmodule ")
var pkgPathFromGoModCache = struct {
paths map[string]string
sync.RWMutex
@@ -117,36 +114,7 @@ func getModulePath(goModPath string) string {
if err != nil {
return ""
}
var i int
if bytes.HasPrefix(data, modulePrefix[1:]) {
i = 0
} else {
i = bytes.Index(data, modulePrefix)
if i < 0 {
return ""
}
i++
}
line := data[i:]
// Cut line at \n, drop trailing \r if present.
if j := bytes.IndexByte(line, '\n'); j >= 0 {
line = line[:j]
}
if line[len(line)-1] == '\r' {
line = line[:len(line)-1]
}
line = line[len("module "):]
// If quoted, unquote.
pkgPath = strings.TrimSpace(string(line))
if pkgPath != "" && pkgPath[0] == '"' {
s, err := strconv.Unquote(pkgPath)
if err != nil {
return ""
}
pkgPath = s
}
pkgPath = modulePath(data)
return pkgPath
}
+39
View File
@@ -0,0 +1,39 @@
package parser
import "testing"
func Test_getModulePath(t *testing.T) {
tests := map[string]struct {
goModPath string
want string
}{
"valid go.mod without comments and deps": {
goModPath: "./testdata/default.go.mod",
want: "example.com/user/project",
},
"valid go.mod with comments and without deps": {
goModPath: "./testdata/comments.go.mod",
want: "example.com/user/project",
},
"valid go.mod with comments and deps": {
goModPath: "./testdata/comments_deps.go.mod",
want: "example.com/user/project",
},
"actual easyjson go.mod": {
goModPath: "../go.mod",
want: "github.com/mailru/easyjson",
},
"invalid go.mod with missing module": {
goModPath: "./testdata/missing_module.go",
want: "",
},
}
for name := range tests {
tt := tests[name]
t.Run(name, func(t *testing.T) {
if got := getModulePath(tt.goModPath); got != tt.want {
t.Errorf("getModulePath() = %v, want %v", got, tt.want)
}
})
}
}
+4
View File
@@ -0,0 +1,4 @@
// first-line comment which should bresk anything
module example.com/user/project // end-line comment which should not break anything
go 1.13
+8
View File
@@ -0,0 +1,8 @@
// first-line comment which should bresk anything
module example.com/user/project // end-line comment which should not break anything
go 1.13
require (
github.com/mailru/easyjson v0.7.0
)
+3
View File
@@ -0,0 +1,3 @@
module example.com/user/project
go 1.13
+6
View File
@@ -0,0 +1,6 @@
go 1.13
require (
github.com/mailru/easyjson v0.7.0
)
+6
View File
@@ -678,6 +678,12 @@ type RequiredOptionalStruct struct {
Lastname string `json:"last_name"`
}
type RequiredOptionalMap struct {
ReqMap map[int]string `json:"req_map,required"`
OmitEmptyMap map[int]string `json:"oe_map,omitempty"`
NoOmitEmptyMap map[int]string `json:"noe_map,!omitempty"`
}
//easyjson:json
type EncodingFlagsTestMap struct {
F map[string]string
+5
View File
@@ -0,0 +1,5 @@
package tests
type Struct struct {
Test string
}
+33
View File
@@ -0,0 +1,33 @@
package tests
import (
"testing"
"github.com/mailru/easyjson/jwriter"
)
func TestHTML(t *testing.T) {
s := Struct{
Test: "<b>test</b>",
}
j := jwriter.Writer{
NoEscapeHTML: false,
}
s.MarshalEasyJSON(&j)
data, _ := j.BuildBytes()
if string(data) != `{"Test":"\u003cb\u003etest\u003c/b\u003e"}` {
t.Fatal("EscapeHTML error:", string(data))
}
j.NoEscapeHTML = true
s.MarshalEasyJSON(&j)
data, _ = j.BuildBytes()
if string(data) != `{"Test":"<b>test</b>"}` {
t.Fatal("NoEscapeHTML error:", string(data))
}
}

Some files were not shown because too many files have changed in this diff Show More