diff --git a/Makefile b/Makefile index e2c9bd6..85ad6e2 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ generate: root build .root/bin/easyjson -all .root/src/$(PKG)/tests/data.go .root/bin/easyjson -all .root/src/$(PKG)/tests/nothing.go + .root/bin/easyjson -all .root/src/$(PKG)/tests/errors.go .root/bin/easyjson -snake_case .root/src/$(PKG)/tests/snake.go .root/bin/easyjson -omit_empty .root/src/$(PKG)/tests/omitempty.go .root/bin/easyjson -build_tags=use_easyjson .root/src/$(PKG)/benchmark/data.go diff --git a/jlexer/lexer.go b/jlexer/lexer.go index eac6cf5..7b48022 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -46,7 +46,10 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. - err error // Error encountered during lexing, if any. + UseMultipleErrors bool // If we want to use multiple errors. + fatalError error // Fatal error occured during lexing. It is usually a syntax error. + nowSem bool // If semantic error occured during parsing. + multipleErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. } // fetchToken scans the input for the next token. @@ -148,7 +151,7 @@ func (r *Lexer) fetchToken() { return } } - r.err = io.EOF + r.fatalError = io.EOF return } @@ -369,7 +372,7 @@ func (r *Lexer) fetchString() { // scanToken scans the next token if no token is currently available in the lexer. func (r *Lexer) scanToken() { - if r.token.kind != tokenUndef || r.err != nil { + if r.token.kind != tokenUndef || r.fatalError != nil { return } @@ -384,20 +387,20 @@ func (r *Lexer) consume() { // Ok returns true if no error (including io.EOF) was encountered during scanning. func (r *Lexer) Ok() bool { - return r.err == nil + return r.fatalError == nil } const maxErrorContextLen = 13 func (r *Lexer) errParse(what string) { - if r.err == nil { + if r.fatalError == nil { var str string if len(r.Data)-r.pos <= maxErrorContextLen { str = string(r.Data) } else { str = string(r.Data[r.pos:r.pos+maxErrorContextLen-3]) + "..." } - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: what, Offset: r.pos, Data: str, @@ -410,18 +413,39 @@ func (r *Lexer) errSyntax() { } func (r *Lexer) errInvalidToken(expected string) { - if r.err == nil { - var str string - if len(r.token.byteValue) <= maxErrorContextLen { - str = string(r.token.byteValue) - } else { - str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..." + if r.fatalError != nil { + return + } + if r.UseMultipleErrors { + r.pos = r.start + r.consume() + r.SkipRecursive() + switch expected { + case "[": + r.token.delimValue = ']' + r.token.kind = tokenDelim + case "{": + r.token.delimValue = '}' + r.token.kind = tokenDelim } - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ Reason: fmt.Sprintf("expected %s", expected), - Offset: r.pos, - Data: str, - } + Offset: r.start, + Data: string(r.Data[r.start:]), + }) + return + } + + var str string + if len(r.token.byteValue) <= maxErrorContextLen { + str = string(r.token.byteValue) + } else { + str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..." + } + r.fatalError = &LexerError{ + Reason: fmt.Sprintf("expected %s", expected), + Offset: r.pos, + Data: str, } } @@ -430,10 +454,13 @@ func (r *Lexer) Delim(c byte) { if r.token.kind == tokenUndef && r.Ok() { r.fetchToken() } + if !r.Ok() || r.token.delimValue != c { + r.consume() // errInvalidToken can change token if UseMultipleErrors is enabled. r.errInvalidToken(string([]byte{c})) + } else { + r.consume() } - r.consume() } // IsDelim returns true if there was no scanning error and next token is the given delimiter. @@ -477,7 +504,6 @@ func (r *Lexer) Skip() { // Note: no syntax validation is performed on the skipped data. func (r *Lexer) SkipRecursive() { r.scanToken() - var start, end byte if r.token.delimValue == '{' { @@ -516,7 +542,7 @@ func (r *Lexer) SkipRecursive() { wasEscape = false } r.pos = len(r.Data) - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: "EOF reached while skipping array/object or token", Offset: r.pos, Data: string(r.Data[r.pos:]), @@ -547,7 +573,7 @@ func (r *Lexer) Consumed() { for _, c := range r.Data[r.pos:] { if c != ' ' && c != '\t' && c != '\r' && c != '\n' { - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: "invalid character '" + string(c) + "' after top-level value", Offset: r.pos, Data: string(r.Data[r.pos:]), @@ -586,7 +612,6 @@ func (r *Lexer) String() string { if !r.Ok() || r.token.kind != tokenString { r.errInvalidToken("string") return "" - } ret := string(r.token.byteValue) r.consume() @@ -605,7 +630,7 @@ func (r *Lexer) Bytes() []byte { ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue))) len, err := base64.StdEncoding.Decode(ret, r.token.byteValue) if err != nil { - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: err.Error(), } return nil @@ -623,7 +648,6 @@ func (r *Lexer) Bool() bool { if !r.Ok() || r.token.kind != tokenBool { r.errInvalidToken("bool") return false - } ret := r.token.boolValue r.consume() @@ -637,7 +661,6 @@ func (r *Lexer) number() string { if !r.Ok() || r.token.kind != tokenNumber { r.errInvalidToken("number") return "" - } ret := bytesToStr(r.token.byteValue) r.consume() @@ -652,9 +675,10 @@ func (r *Lexer) Uint8() uint8 { n, err := strconv.ParseUint(s, 10, 8) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint8(n) } @@ -667,9 +691,10 @@ func (r *Lexer) Uint16() uint16 { n, err := strconv.ParseUint(s, 10, 16) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint16(n) } @@ -682,9 +707,10 @@ func (r *Lexer) Uint32() uint32 { n, err := strconv.ParseUint(s, 10, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint32(n) } @@ -697,9 +723,10 @@ func (r *Lexer) Uint64() uint64 { n, err := strconv.ParseUint(s, 10, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -716,9 +743,10 @@ func (r *Lexer) Int8() int8 { n, err := strconv.ParseInt(s, 10, 8) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int8(n) } @@ -731,9 +759,10 @@ func (r *Lexer) Int16() int16 { n, err := strconv.ParseInt(s, 10, 16) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int16(n) } @@ -746,9 +775,10 @@ func (r *Lexer) Int32() int32 { n, err := strconv.ParseInt(s, 10, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int32(n) } @@ -761,9 +791,10 @@ func (r *Lexer) Int64() int64 { n, err := strconv.ParseInt(s, 10, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -780,9 +811,10 @@ func (r *Lexer) Uint8Str() uint8 { n, err := strconv.ParseUint(s, 10, 8) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint8(n) } @@ -795,9 +827,10 @@ func (r *Lexer) Uint16Str() uint16 { n, err := strconv.ParseUint(s, 10, 16) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint16(n) } @@ -810,9 +843,10 @@ func (r *Lexer) Uint32Str() uint32 { n, err := strconv.ParseUint(s, 10, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint32(n) } @@ -825,9 +859,10 @@ func (r *Lexer) Uint64Str() uint64 { n, err := strconv.ParseUint(s, 10, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -844,9 +879,10 @@ func (r *Lexer) Int8Str() int8 { n, err := strconv.ParseInt(s, 10, 8) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int8(n) } @@ -859,9 +895,10 @@ func (r *Lexer) Int16Str() int16 { n, err := strconv.ParseInt(s, 10, 16) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int16(n) } @@ -874,9 +911,10 @@ func (r *Lexer) Int32Str() int32 { n, err := strconv.ParseInt(s, 10, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int32(n) } @@ -889,9 +927,10 @@ func (r *Lexer) Int64Str() int64 { n, err := strconv.ParseInt(s, 10, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -908,9 +947,10 @@ func (r *Lexer) Float32() float32 { n, err := strconv.ParseFloat(s, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return float32(n) } @@ -923,23 +963,39 @@ func (r *Lexer) Float64() float64 { n, err := strconv.ParseFloat(s, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } func (r *Lexer) Error() error { - return r.err + return r.fatalError } func (r *Lexer) AddError(e error) { - if r.err == nil { - r.err = e + if r.fatalError == nil { + r.fatalError = e } } +func (r *Lexer) addNonfatalError(err *LexerError) { + if r.UseMultipleErrors { + if len(r.multipleErrors) != 0 && r.multipleErrors[len(r.multipleErrors)-1].Offset == err.Offset { + return + } + r.multipleErrors = append(r.multipleErrors, err) + return + } + r.fatalError = err +} + +func (r *Lexer) GetNonFatalErrors() []*LexerError { + return r.multipleErrors +} + // Interface fetches an interface{} analogous to the 'encoding/json' package. func (r *Lexer) Interface() interface{} { if r.token.kind == tokenUndef && r.Ok() { diff --git a/tests/errors.go b/tests/errors.go new file mode 100644 index 0000000..2ec3299 --- /dev/null +++ b/tests/errors.go @@ -0,0 +1,23 @@ +package tests + +//easyjson:json +type ErrorIntSlice []int + +//easyjson:json +type ErrorBoolSlice []bool + +//easyjson:json +type ErrorUintSlice []uint + +//easyjson:json +type ErrorStruct struct { + Int int `json:"int"` + String string `json:"string"` + Slice []int `json:"slice"` + IntSlice []int `json:"int_slice"` +} + +type ErrorNestedStruct struct { + ErrorStruct ErrorStruct `json:"error_struct"` + Int int `json:"int"` +} diff --git a/tests/errors_test.go b/tests/errors_test.go new file mode 100644 index 0000000..756f7db --- /dev/null +++ b/tests/errors_test.go @@ -0,0 +1,243 @@ +package tests + +import ( + "testing" + + "github.com/mailru/easyjson/jlexer" +) + +func TestMultipleErrorsInt(t *testing.T) { + for i, test := range []struct { + Data []byte + Offsets []int + }{ + { + Data: []byte(`[1, 2, 3, "4", "5"]`), + Offsets: []int{10, 15}, + }, + { + Data: []byte(`[1, {"2":"3"}, 3, "4"]`), + Offsets: []int{4, 18}, + }, + { + Data: []byte(`[1, "2", "3", "4", "5", "6"]`), + Offsets: []int{4, 9, 14, 19, 24}, + }, + { + Data: []byte(`[1, 2, 3, 4, "5"]`), + Offsets: []int{13}, + }, + { + Data: []byte(`[{"1": "2"}]`), + Offsets: []int{1}, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + + var v ErrorIntSlice + + v.UnmarshalEasyJSON(&l) + + errors := l.GetNonFatalErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsInt(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return + } + + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsInt(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } + } + } +} + +func TestMultipleErrorsBool(t *testing.T) { + for i, test := range []struct { + Data []byte + Offsets []int + }{ + { + Data: []byte(`[true, false, true, false]`), + }, + { + Data: []byte(`["test", "value", "lol", "1"]`), + Offsets: []int{1, 9, 18, 25}, + }, + { + Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false]`), + Offsets: []int{7, 11}, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + + var v ErrorBoolSlice + v.UnmarshalEasyJSON(&l) + + errors := l.GetNonFatalErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsBool(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsBool(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } + } + } +} + +func TestMultipleErrorsUint(t *testing.T) { + for i, test := range []struct { + Data []byte + Offsets []int + }{ + { + Data: []byte(`[42, 42, 42]`), + }, + { + Data: []byte(`[17, "42", 32]`), + Offsets: []int{5}, + }, + { + Data: []byte(`["zz", "zz"]`), + Offsets: []int{1, 7}, + }, + { + Data: []byte(`[{}, 42]`), + Offsets: []int{1}, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + + var v ErrorUintSlice + v.UnmarshalEasyJSON(&l) + + errors := l.GetNonFatalErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsUint(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsUint(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } + } + } +} + +func TestMultipleErrorsStruct(t *testing.T) { + for i, test := range []struct { + Data []byte + Offsets []int + }{ + { + Data: []byte(`{"string": "test", "slice":[42, 42, 42], "int_slice":[1, 2, 3]}`), + }, + { + Data: []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`), + Offsets: []int{11, 64}, + }, + { + Data: []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`), + Offsets: []int{30, 61, 66}, + }, + { + Data: []byte(`{"string": "test", "slice": {}}`), + Offsets: []int{28}, + }, + { + Data: []byte(`{"slice":5, "string" : "test"}`), + Offsets: []int{9}, + }, + { + Data: []byte(`{"slice" : "test", "string" : "test"}`), + Offsets: []int{11}, + }, + { + Data: []byte(`{"slice": "", "string" : {}, "int":{}}`), + Offsets: []int{10, 25, 35}, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + var v ErrorStruct + v.UnmarshalEasyJSON(&l) + + errors := l.GetNonFatalErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } + } + } +} + +func TestMultipleErrorsNestedStruct(t *testing.T) { + for i, test := range []struct { + Data []byte + Offsets []int + }{ + { + Data: []byte(`{"error_struct":{}}`), + }, + { + Data: []byte(`{"error_struct":5}`), + Offsets: []int{16}, + }, + { + Data: []byte(`{"error_struct":[]}`), + Offsets: []int{16}, + }, + { + Data: []byte(`{"error_struct":{"int":{}}}`), + Offsets: []int{23}, + }, + { + Data: []byte(`{"error_struct":{"int_slice":{}}, "int":4}`), + Offsets: []int{29}, + }, + { + Data: []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`), + Offsets: []int{30, 38, 51}, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + var v ErrorNestedStruct + v.UnmarshalEasyJSON(&l) + + errors := l.GetNonFatalErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsNestedStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsNestedStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } + } + } +}