netbird/easyjson
0
0
Fork 0
mirror of https://github.com/netbirdio/easyjson.git synced 2026-05-22 18:44:42 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #88 from shmel1k/feature/slice_marshalling_error

Browse Source 
A flag for returning multiple parse errors if possible.
This commit is contained in:
Victor Starodub
2017-01-12 19:01:47 +04:00
committed by GitHub
parent 9d6630dc8c ebe9ab918e
commit db88438fca
4 changed files with 386 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1
View File
@@ -25,6 +25,7 @@ generate: root build
.root/bin/easyjson -all .root/src/$(PKG)/tests/data.go
.root/bin/easyjson -all .root/src/$(PKG)/tests/nothing.go
.root/bin/easyjson -all .root/src/$(PKG)/tests/errors.go
.root/bin/easyjson -snake_case .root/src/$(PKG)/tests/snake.go
.root/bin/easyjson -omit_empty .root/src/$(PKG)/tests/omitempty.go
.root/bin/easyjson -build_tags=use_easyjson .root/src/$(PKG)/benchmark/data.go
jlexer/lexer.go
+119 -63
View File
@@ -46,7 +46,10 @@ type Lexer struct {
firstElement bool // Whether current element is the first in array or an object.
wantSep byte // A comma or a colon character, which need to occur before a token.
err error // Error encountered during lexing, if any.
UseMultipleErrors bool // If we want to use multiple errors.
fatalError error // Fatal error occured during lexing. It is usually a syntax error.
nowSem bool // If semantic error occured during parsing.
multipleErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors.
}
// fetchToken scans the input for the next token.
@@ -148,7 +151,7 @@ func (r *Lexer) fetchToken() {
return
}
}
r.err = io.EOF
r.fatalError = io.EOF
return
}
@@ -369,7 +372,7 @@ func (r *Lexer) fetchString() {
// scanToken scans the next token if no token is currently available in the lexer.
func (r *Lexer) scanToken() {
if r.token.kind != tokenUndef || r.err != nil {
if r.token.kind != tokenUndef || r.fatalError != nil {
return
}
@@ -384,20 +387,20 @@ func (r *Lexer) consume() {
// Ok returns true if no error (including io.EOF) was encountered during scanning.
func (r *Lexer) Ok() bool {
return r.err == nil
return r.fatalError == nil
}
const maxErrorContextLen = 13
func (r *Lexer) errParse(what string) {
if r.err == nil {
if r.fatalError == nil {
var str string
if len(r.Data)-r.pos <= maxErrorContextLen {
str = string(r.Data)
} else {
str = string(r.Data[r.pos:r.pos+maxErrorContextLen-3]) + "..."
}
r.err = &LexerError{
r.fatalError = &LexerError{
Reason: what,
Offset: r.pos,
Data: str,
@@ -410,18 +413,39 @@ func (r *Lexer) errSyntax() {
}
func (r *Lexer) errInvalidToken(expected string) {
if r.err == nil {
var str string
if len(r.token.byteValue) <= maxErrorContextLen {
str = string(r.token.byteValue)
} else {
str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..."
if r.fatalError != nil {
return
}
if r.UseMultipleErrors {
r.pos = r.start
r.consume()
r.SkipRecursive()
switch expected {
case "[":
r.token.delimValue = ']'
r.token.kind = tokenDelim
case "{":
r.token.delimValue = '}'
r.token.kind = tokenDelim
}
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Reason: fmt.Sprintf("expected %s", expected),
Offset: r.pos,
Data: str,
}
Offset: r.start,
Data: string(r.Data[r.start:]),
})
return
}
var str string
if len(r.token.byteValue) <= maxErrorContextLen {
str = string(r.token.byteValue)
} else {
str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..."
}
r.fatalError = &LexerError{
Reason: fmt.Sprintf("expected %s", expected),
Offset: r.pos,
Data: str,
}
}
@@ -430,10 +454,13 @@ func (r *Lexer) Delim(c byte) {
if r.token.kind == tokenUndef && r.Ok() {
r.fetchToken()
}
if !r.Ok() || r.token.delimValue != c {
r.consume() // errInvalidToken can change token if UseMultipleErrors is enabled.
r.errInvalidToken(string([]byte{c}))
} else {
r.consume()
}
r.consume()
}
// IsDelim returns true if there was no scanning error and next token is the given delimiter.
@@ -477,7 +504,6 @@ func (r *Lexer) Skip() {
// Note: no syntax validation is performed on the skipped data.
func (r *Lexer) SkipRecursive() {
r.scanToken()
var start, end byte
if r.token.delimValue == '{' {
@@ -516,7 +542,7 @@ func (r *Lexer) SkipRecursive() {
wasEscape = false
}
r.pos = len(r.Data)
r.err = &LexerError{
r.fatalError = &LexerError{
Reason: "EOF reached while skipping array/object or token",
Offset: r.pos,
Data: string(r.Data[r.pos:]),
@@ -547,7 +573,7 @@ func (r *Lexer) Consumed() {
for _, c := range r.Data[r.pos:] {
if c != ' ' && c != '\t' && c != '\r' && c != '\n' {
r.err = &LexerError{
r.fatalError = &LexerError{
Reason: "invalid character '" + string(c) + "' after top-level value",
Offset: r.pos,
Data: string(r.Data[r.pos:]),
@@ -586,7 +612,6 @@ func (r *Lexer) String() string {
if !r.Ok() || r.token.kind != tokenString {
r.errInvalidToken("string")
return ""
}
ret := string(r.token.byteValue)
r.consume()
@@ -605,7 +630,7 @@ func (r *Lexer) Bytes() []byte {
ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue)))
len, err := base64.StdEncoding.Decode(ret, r.token.byteValue)
if err != nil {
r.err = &LexerError{
r.fatalError = &LexerError{
Reason: err.Error(),
}
return nil
@@ -623,7 +648,6 @@ func (r *Lexer) Bool() bool {
if !r.Ok() || r.token.kind != tokenBool {
r.errInvalidToken("bool")
return false
}
ret := r.token.boolValue
r.consume()
@@ -637,7 +661,6 @@ func (r *Lexer) number() string {
if !r.Ok() || r.token.kind != tokenNumber {
r.errInvalidToken("number")
return ""
}
ret := bytesToStr(r.token.byteValue)
r.consume()
@@ -652,9 +675,10 @@ func (r *Lexer) Uint8() uint8 {
n, err := strconv.ParseUint(s, 10, 8)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return uint8(n)
}
@@ -667,9 +691,10 @@ func (r *Lexer) Uint16() uint16 {
n, err := strconv.ParseUint(s, 10, 16)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return uint16(n)
}
@@ -682,9 +707,10 @@ func (r *Lexer) Uint32() uint32 {
n, err := strconv.ParseUint(s, 10, 32)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return uint32(n)
}
@@ -697,9 +723,10 @@ func (r *Lexer) Uint64() uint64 {
n, err := strconv.ParseUint(s, 10, 64)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return n
}
@@ -716,9 +743,10 @@ func (r *Lexer) Int8() int8 {
n, err := strconv.ParseInt(s, 10, 8)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return int8(n)
}
@@ -731,9 +759,10 @@ func (r *Lexer) Int16() int16 {
n, err := strconv.ParseInt(s, 10, 16)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return int16(n)
}
@@ -746,9 +775,10 @@ func (r *Lexer) Int32() int32 {
n, err := strconv.ParseInt(s, 10, 32)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return int32(n)
}
@@ -761,9 +791,10 @@ func (r *Lexer) Int64() int64 {
n, err := strconv.ParseInt(s, 10, 64)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return n
}
@@ -780,9 +811,10 @@ func (r *Lexer) Uint8Str() uint8 {
n, err := strconv.ParseUint(s, 10, 8)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return uint8(n)
}
@@ -795,9 +827,10 @@ func (r *Lexer) Uint16Str() uint16 {
n, err := strconv.ParseUint(s, 10, 16)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return uint16(n)
}
@@ -810,9 +843,10 @@ func (r *Lexer) Uint32Str() uint32 {
n, err := strconv.ParseUint(s, 10, 32)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return uint32(n)
}
@@ -825,9 +859,10 @@ func (r *Lexer) Uint64Str() uint64 {
n, err := strconv.ParseUint(s, 10, 64)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return n
}
@@ -844,9 +879,10 @@ func (r *Lexer) Int8Str() int8 {
n, err := strconv.ParseInt(s, 10, 8)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return int8(n)
}
@@ -859,9 +895,10 @@ func (r *Lexer) Int16Str() int16 {
n, err := strconv.ParseInt(s, 10, 16)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return int16(n)
}
@@ -874,9 +911,10 @@ func (r *Lexer) Int32Str() int32 {
n, err := strconv.ParseInt(s, 10, 32)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return int32(n)
}
@@ -889,9 +927,10 @@ func (r *Lexer) Int64Str() int64 {
n, err := strconv.ParseInt(s, 10, 64)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return n
}
@@ -908,9 +947,10 @@ func (r *Lexer) Float32() float32 {
n, err := strconv.ParseFloat(s, 32)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return float32(n)
}
@@ -923,23 +963,39 @@ func (r *Lexer) Float64() float64 {
n, err := strconv.ParseFloat(s, 64)
if err != nil {
r.err = &LexerError{
r.addNonfatalError(&LexerError{
Offset: r.start,
Reason: err.Error(),
}
})
}
return n
}
func (r *Lexer) Error() error {
return r.err
return r.fatalError
}
func (r *Lexer) AddError(e error) {
if r.err == nil {
r.err = e
if r.fatalError == nil {
r.fatalError = e
}
}
func (r *Lexer) addNonfatalError(err *LexerError) {
if r.UseMultipleErrors {
if len(r.multipleErrors) != 0 && r.multipleErrors[len(r.multipleErrors)-1].Offset == err.Offset {
return
}
r.multipleErrors = append(r.multipleErrors, err)
return
}
r.fatalError = err
}
func (r *Lexer) GetNonFatalErrors() []*LexerError {
return r.multipleErrors
}
// Interface fetches an interface{} analogous to the 'encoding/json' package.
func (r *Lexer) Interface() interface{} {
if r.token.kind == tokenUndef && r.Ok() {
tests/errors.go
+23
View File
@@ -0,0 +1,23 @@
package tests
//easyjson:json
type ErrorIntSlice []int
//easyjson:json
type ErrorBoolSlice []bool
//easyjson:json
type ErrorUintSlice []uint
//easyjson:json
type ErrorStruct struct {
Int int `json:"int"`
String string `json:"string"`
Slice []int `json:"slice"`
IntSlice []int `json:"int_slice"`
}
type ErrorNestedStruct struct {
ErrorStruct ErrorStruct `json:"error_struct"`
Int int `json:"int"`
}
tests/errors_test.go
+243
View File
@@ -0,0 +1,243 @@
package tests
import (
"testing"
"github.com/mailru/easyjson/jlexer"
)
func TestMultipleErrorsInt(t *testing.T) {
for i, test := range []struct {
Data []byte
Offsets []int
}{
{
Data: []byte(`[1, 2, 3, "4", "5"]`),
Offsets: []int{10, 15},
},
{
Data: []byte(`[1, {"2":"3"}, 3, "4"]`),
Offsets: []int{4, 18},
},
{
Data: []byte(`[1, "2", "3", "4", "5", "6"]`),
Offsets: []int{4, 9, 14, 19, 24},
},
{
Data: []byte(`[1, 2, 3, 4, "5"]`),
Offsets: []int{13},
},
{
Data: []byte(`[{"1": "2"}]`),
Offsets: []int{1},
},
} {
l := jlexer.Lexer{
Data: test.Data,
UseMultipleErrors: true,
}
var v ErrorIntSlice
v.UnmarshalEasyJSON(&l)
errors := l.GetNonFatalErrors()
if len(errors) != len(test.Offsets) {
t.Errorf("[%d] TestMultipleErrorsInt(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
return
}
for ii, e := range errors {
if e.Offset != test.Offsets[ii] {
t.Errorf("[%d] TestMultipleErrorsInt(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
}
}
}
}
func TestMultipleErrorsBool(t *testing.T) {
for i, test := range []struct {
Data []byte
Offsets []int
}{
{
Data: []byte(`[true, false, true, false]`),
},
{
Data: []byte(`["test", "value", "lol", "1"]`),
Offsets: []int{1, 9, 18, 25},
},
{
Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false]`),
Offsets: []int{7, 11},
},
} {
l := jlexer.Lexer{
Data: test.Data,
UseMultipleErrors: true,
}
var v ErrorBoolSlice
v.UnmarshalEasyJSON(&l)
errors := l.GetNonFatalErrors()
if len(errors) != len(test.Offsets) {
t.Errorf("[%d] TestMultipleErrorsBool(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
return
}
for ii, e := range errors {
if e.Offset != test.Offsets[ii] {
t.Errorf("[%d] TestMultipleErrorsBool(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
}
}
}
}
func TestMultipleErrorsUint(t *testing.T) {
for i, test := range []struct {
Data []byte
Offsets []int
}{
{
Data: []byte(`[42, 42, 42]`),
},
{
Data: []byte(`[17, "42", 32]`),
Offsets: []int{5},
},
{
Data: []byte(`["zz", "zz"]`),
Offsets: []int{1, 7},
},
{
Data: []byte(`[{}, 42]`),
Offsets: []int{1},
},
} {
l := jlexer.Lexer{
Data: test.Data,
UseMultipleErrors: true,
}
var v ErrorUintSlice
v.UnmarshalEasyJSON(&l)
errors := l.GetNonFatalErrors()
if len(errors) != len(test.Offsets) {
t.Errorf("[%d] TestMultipleErrorsUint(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
return
}
for ii, e := range errors {
if e.Offset != test.Offsets[ii] {
t.Errorf("[%d] TestMultipleErrorsUint(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
}
}
}
}
func TestMultipleErrorsStruct(t *testing.T) {
for i, test := range []struct {
Data []byte
Offsets []int
}{
{
Data: []byte(`{"string": "test", "slice":[42, 42, 42], "int_slice":[1, 2, 3]}`),
},
{
Data: []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`),
Offsets: []int{11, 64},
},
{
Data: []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`),
Offsets: []int{30, 61, 66},
},
{
Data: []byte(`{"string": "test", "slice": {}}`),
Offsets: []int{28},
},
{
Data: []byte(`{"slice":5, "string" : "test"}`),
Offsets: []int{9},
},
{
Data: []byte(`{"slice" : "test", "string" : "test"}`),
Offsets: []int{11},
},
{
Data: []byte(`{"slice": "", "string" : {}, "int":{}}`),
Offsets: []int{10, 25, 35},
},
} {
l := jlexer.Lexer{
Data: test.Data,
UseMultipleErrors: true,
}
var v ErrorStruct
v.UnmarshalEasyJSON(&l)
errors := l.GetNonFatalErrors()
if len(errors) != len(test.Offsets) {
t.Errorf("[%d] TestMultipleErrorsStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
return
}
for ii, e := range errors {
if e.Offset != test.Offsets[ii] {
t.Errorf("[%d] TestMultipleErrorsStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
}
}
}
}
func TestMultipleErrorsNestedStruct(t *testing.T) {
for i, test := range []struct {
Data []byte
Offsets []int
}{
{
Data: []byte(`{"error_struct":{}}`),
},
{
Data: []byte(`{"error_struct":5}`),
Offsets: []int{16},
},
{
Data: []byte(`{"error_struct":[]}`),
Offsets: []int{16},
},
{
Data: []byte(`{"error_struct":{"int":{}}}`),
Offsets: []int{23},
},
{
Data: []byte(`{"error_struct":{"int_slice":{}}, "int":4}`),
Offsets: []int{29},
},
{
Data: []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`),
Offsets: []int{30, 38, 51},
},
} {
l := jlexer.Lexer{
Data: test.Data,
UseMultipleErrors: true,
}
var v ErrorNestedStruct
v.UnmarshalEasyJSON(&l)
errors := l.GetNonFatalErrors()
if len(errors) != len(test.Offsets) {
t.Errorf("[%d] TestMultipleErrorsNestedStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
return
}
for ii, e := range errors {
if e.Offset != test.Offsets[ii] {
t.Errorf("[%d] TestMultipleErrorsNestedStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
}
}
}
}