From 3b6b44b81e4bdd709e52d52fb0024531c1889983 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Mon, 19 Dec 2016 21:21:28 +0300 Subject: [PATCH 01/16] Added possibility go get semantic errors --- Makefile | 1 + jlexer/context.go | 92 +++++++++++++++++ jlexer/context_test.go | 85 ++++++++++++++++ jlexer/lexer.go | 218 +++++++++++++++++++++++++++++++---------- jlexer/lexer_test.go | 6 ++ tests/basic_test.go | 23 +++++ tests/errors.go | 4 + 7 files changed, 377 insertions(+), 52 deletions(-) create mode 100644 jlexer/context.go create mode 100644 jlexer/context_test.go create mode 100644 tests/errors.go diff --git a/Makefile b/Makefile index e2c9bd6..85ad6e2 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ generate: root build .root/bin/easyjson -all .root/src/$(PKG)/tests/data.go .root/bin/easyjson -all .root/src/$(PKG)/tests/nothing.go + .root/bin/easyjson -all .root/src/$(PKG)/tests/errors.go .root/bin/easyjson -snake_case .root/src/$(PKG)/tests/snake.go .root/bin/easyjson -omit_empty .root/src/$(PKG)/tests/omitempty.go .root/bin/easyjson -build_tags=use_easyjson .root/src/$(PKG)/benchmark/data.go diff --git a/jlexer/context.go b/jlexer/context.go new file mode 100644 index 0000000..ec53902 --- /dev/null +++ b/jlexer/context.go @@ -0,0 +1,92 @@ +package jlexer + +import "io" + +type Walker interface { + OnEnterObject() + OnNextObjectKey(key string) + OnExitObject() + + OnEnterArray() + OnNextArrayElement() + OnExitArray() +} + +func (l *Lexer) WalkUpToPosition(w Walker) error { + l1 := &Lexer{} + l1.Data = l.Data + + type StackItem int + const ( + Object StackItem = iota + Array + ) + + var stack []StackItem + haveKey := false + + for { + l1.fetchToken() + if l1.pos > l.pos || !l1.Ok() { + break + } + switch { + case l1.IsDelim('{'): + l1.Skip() + + stack = append(stack, Object) + w.OnEnterObject() + haveKey = false + + case l1.IsDelim('}'): + l1.Skip() + + if len(stack) > 0 { + stack = stack[:len(stack)-1] + l1.WantComma() + } + w.OnExitObject() + haveKey = false + + case l1.IsDelim('['): + l1.Skip() + + stack = append(stack, Array) + w.OnEnterArray() + haveKey = false + + case l1.IsDelim(']'): + l1.Skip() + + if len(stack) > 0 { + stack = stack[:len(stack)-1] + l1.WantComma() + } + w.OnExitArray() + haveKey = false + + case len(stack) > 0 && stack[len(stack)-1] == Object && !haveKey: + key := l1.UnsafeString() + w.OnNextObjectKey(key) + + l1.WantColon() + haveKey = true + + case len(stack) > 0 && stack[len(stack)-1] == Array: + w.OnNextArrayElement() + l1.Skip() + l1.WantComma() + + default: + l1.Skip() + l1.WantComma() + haveKey = false + } + + } + + if l1.Error() == io.EOF { + return nil + } + return l1.Error() +} diff --git a/jlexer/context_test.go b/jlexer/context_test.go new file mode 100644 index 0000000..3b7ee58 --- /dev/null +++ b/jlexer/context_test.go @@ -0,0 +1,85 @@ +package jlexer + +import ( + "fmt" + "testing" +) + +type TestWalker struct { + ErrorPrefix string + T *testing.T + Items []string +} + +func (w *TestWalker) item(s string) { + if len(w.Items) == 0 { + w.T.Errorf("%sTestWalker(): no items left; want %q", w.ErrorPrefix, s) + } else if w.Items[0] != s { + w.T.Errorf("%sTestWalker(): got %q; want %q", w.ErrorPrefix, s, w.Items[0]) + w.Items = w.Items[1:] + } else { + w.Items = w.Items[1:] + } +} + +func (w *TestWalker) OnEnterObject() { w.item("{") } +func (w *TestWalker) OnExitObject() { w.item("}") } +func (w *TestWalker) OnNextObjectKey(key string) { w.item("e:" + key) } +func (w *TestWalker) OnEnterArray() { w.item("[") } +func (w *TestWalker) OnNextArrayElement() { w.item("e") } +func (w *TestWalker) OnExitArray() { w.item("]") } + +func TestWalkUpToPosition(t *testing.T) { + for i, test := range []struct { + JSON string + Items []string + Start, End int + }{ + { + JSON: ``, + Items: []string{}, + End: -1, + }, { + JSON: `{"aaa": 5, "qqq": 10}`, + Items: []string{"{", "e:aaa", "e:qqq", "}"}, + End: -1, + }, { + JSON: `{"aaa": 5, "qqq": {"\t\t": null}}`, + Items: []string{"{", "e:aaa", "e:qqq", "{", "e:\t\t", "}", "}"}, + End: -1, + }, { + JSON: `{"aaa": 5, "qqq": 10}`, + End: len(`{"aaa": 5, "qqq": `), + Items: []string{"{", "e:aaa", "e:qqq"}, + }, { + JSON: `{"aaa": 5, "qqq": 10}`, + End: len(`{"aaa": 5, `), + Items: []string{"{", "e:aaa"}, + }, { + JSON: `[null, false, {"aaa": 5}]`, + Items: []string{"[", "e", "e", "{", "e:aaa", "}", "]"}, + End: -1, + }, { + JSON: `[null, "aaa"]`, + Items: []string{"[", "e", "e", "]"}, + End: -1, + }, + } { + l := &Lexer{Data: []byte(test.JSON)} + + if test.End != -1 { + l.pos = test.End + } else { + l.pos = len(test.JSON) + } + w := &TestWalker{T: t, Items: test.Items, ErrorPrefix: fmt.Sprintf("[%d,%q] ", i, test.JSON)} + + if err := l.WalkUpToPosition(w); err != nil { + t.Errorf("[%d,%q] WalkUpToPosition() error: %v", i, test.JSON, err) + } + + if len(w.Items) > 0 { + t.Errorf("[%d,%q] WalkUpToPosition: items %q left", i, test.JSON, w.Items) + } + } +} diff --git a/jlexer/lexer.go b/jlexer/lexer.go index eac6cf5..169105a 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -6,6 +6,7 @@ package jlexer import ( "encoding/base64" + "flag" "fmt" "io" "reflect" @@ -14,6 +15,8 @@ import ( "unsafe" ) +var UseSemanticErrors = flag.Bool("use_many_errors", true, "Allow lexer collect semantic errors") + // tokenKind determines type of a token. type tokenKind byte @@ -46,7 +49,8 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. - err error // Error encountered during lexing, if any. + fatalError error // Fatal error occured during lexing. It is usually a syntax error. + SemanticErrors []error // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. } // fetchToken scans the input for the next token. @@ -148,7 +152,7 @@ func (r *Lexer) fetchToken() { return } } - r.err = io.EOF + r.fatalError = io.EOF return } @@ -369,7 +373,7 @@ func (r *Lexer) fetchString() { // scanToken scans the next token if no token is currently available in the lexer. func (r *Lexer) scanToken() { - if r.token.kind != tokenUndef || r.err != nil { + if r.token.kind != tokenUndef || r.fatalError != nil { return } @@ -384,20 +388,20 @@ func (r *Lexer) consume() { // Ok returns true if no error (including io.EOF) was encountered during scanning. func (r *Lexer) Ok() bool { - return r.err == nil + return r.fatalError == nil } const maxErrorContextLen = 13 func (r *Lexer) errParse(what string) { - if r.err == nil { + if r.fatalError == nil { var str string if len(r.Data)-r.pos <= maxErrorContextLen { str = string(r.Data) } else { str = string(r.Data[r.pos:r.pos+maxErrorContextLen-3]) + "..." } - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: what, Offset: r.pos, Data: str, @@ -409,15 +413,23 @@ func (r *Lexer) errSyntax() { r.errParse("syntax error") } +func (r *Lexer) errSemantic() { // TODO: add error data. + r.AddSemanticError(&LexerError{ + Reason: "syntax error", + Offset: r.pos, + Data: "error occured", // TODO: fix this. + }) +} + func (r *Lexer) errInvalidToken(expected string) { - if r.err == nil { + if r.fatalError == nil { var str string if len(r.token.byteValue) <= maxErrorContextLen { str = string(r.token.byteValue) } else { str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..." } - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: fmt.Sprintf("expected %s", expected), Offset: r.pos, Data: str, @@ -516,7 +528,7 @@ func (r *Lexer) SkipRecursive() { wasEscape = false } r.pos = len(r.Data) - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: "EOF reached while skipping array/object or token", Offset: r.pos, Data: string(r.Data[r.pos:]), @@ -547,7 +559,7 @@ func (r *Lexer) Consumed() { for _, c := range r.Data[r.pos:] { if c != ' ' && c != '\t' && c != '\r' && c != '\n' { - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: "invalid character '" + string(c) + "' after top-level value", Offset: r.pos, Data: string(r.Data[r.pos:]), @@ -605,7 +617,7 @@ func (r *Lexer) Bytes() []byte { ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue))) len, err := base64.StdEncoding.Decode(ret, r.token.byteValue) if err != nil { - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: err.Error(), } return nil @@ -621,9 +633,13 @@ func (r *Lexer) Bool() bool { r.fetchToken() } if !r.Ok() || r.token.kind != tokenBool { + if *UseSemanticErrors { // FIXME: remove copypaste from all methods. + r.errSemantic() + r.SkipRecursive() // FIXME: + return false + } r.errInvalidToken("bool") return false - } ret := r.token.boolValue r.consume() @@ -635,9 +651,13 @@ func (r *Lexer) number() string { r.fetchToken() } if !r.Ok() || r.token.kind != tokenNumber { + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + return "" + } r.errInvalidToken("number") return "" - } ret := bytesToStr(r.token.byteValue) r.consume() @@ -652,8 +672,13 @@ func (r *Lexer) Uint8() uint8 { n, err := strconv.ParseUint(s, 10, 8) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return uint8(n) @@ -667,8 +692,13 @@ func (r *Lexer) Uint16() uint16 { n, err := strconv.ParseUint(s, 10, 16) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return uint16(n) @@ -682,8 +712,13 @@ func (r *Lexer) Uint32() uint32 { n, err := strconv.ParseUint(s, 10, 32) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return uint32(n) @@ -697,8 +732,13 @@ func (r *Lexer) Uint64() uint64 { n, err := strconv.ParseUint(s, 10, 64) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return n @@ -716,8 +756,13 @@ func (r *Lexer) Int8() int8 { n, err := strconv.ParseInt(s, 10, 8) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return int8(n) @@ -731,8 +776,13 @@ func (r *Lexer) Int16() int16 { n, err := strconv.ParseInt(s, 10, 16) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return int16(n) @@ -746,8 +796,13 @@ func (r *Lexer) Int32() int32 { n, err := strconv.ParseInt(s, 10, 32) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return int32(n) @@ -761,8 +816,13 @@ func (r *Lexer) Int64() int64 { n, err := strconv.ParseInt(s, 10, 64) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return n @@ -780,8 +840,13 @@ func (r *Lexer) Uint8Str() uint8 { n, err := strconv.ParseUint(s, 10, 8) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return uint8(n) @@ -795,8 +860,13 @@ func (r *Lexer) Uint16Str() uint16 { n, err := strconv.ParseUint(s, 10, 16) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return uint16(n) @@ -810,8 +880,13 @@ func (r *Lexer) Uint32Str() uint32 { n, err := strconv.ParseUint(s, 10, 32) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return uint32(n) @@ -825,8 +900,13 @@ func (r *Lexer) Uint64Str() uint64 { n, err := strconv.ParseUint(s, 10, 64) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return n @@ -844,8 +924,13 @@ func (r *Lexer) Int8Str() int8 { n, err := strconv.ParseInt(s, 10, 8) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return int8(n) @@ -859,8 +944,13 @@ func (r *Lexer) Int16Str() int16 { n, err := strconv.ParseInt(s, 10, 16) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return int16(n) @@ -874,8 +964,13 @@ func (r *Lexer) Int32Str() int32 { n, err := strconv.ParseInt(s, 10, 32) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return int32(n) @@ -889,8 +984,13 @@ func (r *Lexer) Int64Str() int64 { n, err := strconv.ParseInt(s, 10, 64) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return n @@ -908,8 +1008,13 @@ func (r *Lexer) Float32() float32 { n, err := strconv.ParseFloat(s, 32) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return float32(n) @@ -923,23 +1028,32 @@ func (r *Lexer) Float64() float64 { n, err := strconv.ParseFloat(s, 64) if err != nil { - r.err = &LexerError{ - Reason: err.Error(), + if *UseSemanticErrors { + r.errSemantic() + r.SkipRecursive() + } else { + r.fatalError = &LexerError{ + Reason: err.Error(), + } } } return n } func (r *Lexer) Error() error { - return r.err + return r.fatalError } func (r *Lexer) AddError(e error) { - if r.err == nil { - r.err = e + if r.fatalError == nil { + r.fatalError = e } } +func (r *Lexer) AddSemanticError(err error) { + r.SemanticErrors = append(r.SemanticErrors, err) +} + // Interface fetches an interface{} analogous to the 'encoding/json' package. func (r *Lexer) Interface() interface{} { if r.token.kind == tokenUndef && r.Ok() { diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index e3add8c..f8ec84a 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -95,6 +95,9 @@ func TestNumber(t *testing.T) { t.Errorf("[%d, %q] number() = %v; want %v", i, test.toParse, got, test.want) } err := l.Error() + if err == nil && len(l.SemanticErrors) != 0 { + err = l.SemanticErrors[0] + } if err != nil && !test.wantError { t.Errorf("[%d, %q] number() error: %v", i, test.toParse, err) } else if err == nil && test.wantError { @@ -125,6 +128,9 @@ func TestBool(t *testing.T) { t.Errorf("[%d, %q] Bool() = %v; want %v", i, test.toParse, got, test.want) } err := l.Error() + if err == nil && len(l.SemanticErrors) != 0 { + err = l.SemanticErrors[0] + } if err != nil && !test.wantError { t.Errorf("[%d, %q] Bool() error: %v", i, test.toParse, err) } else if err == nil && test.wantError { diff --git a/tests/basic_test.go b/tests/basic_test.go index 25b1bfc..34961ab 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "github.com/mailru/easyjson" + "github.com/mailru/easyjson/jlexer" "github.com/mailru/easyjson/jwriter" ) @@ -206,3 +207,25 @@ func TestNestedEasyJsonMarshal(t *testing.T) { } } } + +func TestSemanticErrors(t *testing.T) { + for i, test := range []struct { + Data []byte + ErrorNum int + }{ + { + Data: []byte(`[1, 2, 3, "4", "5"]`), + ErrorNum: 2, + }, + } { + l := jlexer.Lexer{Data: test.Data} + + var v ErrorIntSlice + + v.UnmarshalEasyJSON(&l) + + if len(l.SemanticErrors) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + } + } +} diff --git a/tests/errors.go b/tests/errors.go new file mode 100644 index 0000000..cc4eb93 --- /dev/null +++ b/tests/errors.go @@ -0,0 +1,4 @@ +package tests + +//easyjson:json +type ErrorIntSlice []int From 3c64f7b0b1cd821fbc9281d382ba8db72564acfb Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Tue, 20 Dec 2016 17:22:42 +0300 Subject: [PATCH 02/16] Fixed errors doubling --- jlexer/lexer.go | 2 -- tests/basic_test.go | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 169105a..d446fca 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -652,8 +652,6 @@ func (r *Lexer) number() string { } if !r.Ok() || r.token.kind != tokenNumber { if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() return "" } r.errInvalidToken("number") diff --git a/tests/basic_test.go b/tests/basic_test.go index 34961ab..ea7fbbd 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -209,6 +209,9 @@ func TestNestedEasyJsonMarshal(t *testing.T) { } func TestSemanticErrors(t *testing.T) { + if !*jlexer.UseSemanticErrors { + return + } for i, test := range []struct { Data []byte ErrorNum int @@ -217,6 +220,18 @@ func TestSemanticErrors(t *testing.T) { Data: []byte(`[1, 2, 3, "4", "5"]`), ErrorNum: 2, }, + { + Data: []byte(`[1, {"2" : "3"}, 3, "4"`), + ErrorNum: 2, + }, + { + Data: []byte(`[1, "2", "3", "4", "5", "6"]`), + ErrorNum: 5, + }, + { + Data: []byte(`[1, 2, 3, 4, "5"]`), + ErrorNum: 1, + }, } { l := jlexer.Lexer{Data: test.Data} From 995f3cbcfb67d284e01dcf4e9b60a586a84ca57a Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Tue, 20 Dec 2016 17:49:43 +0300 Subject: [PATCH 03/16] fixed tests for semantic errors --- jlexer/lexer_test.go | 4 ++-- tests/basic_test.go | 38 -------------------------------------- tests/errors.go | 6 ++++++ tests/errors_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 40 deletions(-) create mode 100644 tests/errors_test.go diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index f8ec84a..c974edd 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -98,9 +98,9 @@ func TestNumber(t *testing.T) { if err == nil && len(l.SemanticErrors) != 0 { err = l.SemanticErrors[0] } - if err != nil && !test.wantError { + if (err != nil || (len(l.SemanticErrors) != 0 && *UseSemanticErrors)) && !test.wantError { t.Errorf("[%d, %q] number() error: %v", i, test.toParse, err) - } else if err == nil && test.wantError { + } else if (err == nil || (len(l.SemanticErrors) == 0 && *UseSemanticErrors)) && test.wantError { t.Errorf("[%d, %q] number() ok; want error", i, test.toParse) } } diff --git a/tests/basic_test.go b/tests/basic_test.go index ea7fbbd..25b1bfc 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -7,7 +7,6 @@ import ( "encoding/json" "github.com/mailru/easyjson" - "github.com/mailru/easyjson/jlexer" "github.com/mailru/easyjson/jwriter" ) @@ -207,40 +206,3 @@ func TestNestedEasyJsonMarshal(t *testing.T) { } } } - -func TestSemanticErrors(t *testing.T) { - if !*jlexer.UseSemanticErrors { - return - } - for i, test := range []struct { - Data []byte - ErrorNum int - }{ - { - Data: []byte(`[1, 2, 3, "4", "5"]`), - ErrorNum: 2, - }, - { - Data: []byte(`[1, {"2" : "3"}, 3, "4"`), - ErrorNum: 2, - }, - { - Data: []byte(`[1, "2", "3", "4", "5", "6"]`), - ErrorNum: 5, - }, - { - Data: []byte(`[1, 2, 3, 4, "5"]`), - ErrorNum: 1, - }, - } { - l := jlexer.Lexer{Data: test.Data} - - var v ErrorIntSlice - - v.UnmarshalEasyJSON(&l) - - if len(l.SemanticErrors) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) - } - } -} diff --git a/tests/errors.go b/tests/errors.go index cc4eb93..a0180ad 100644 --- a/tests/errors.go +++ b/tests/errors.go @@ -2,3 +2,9 @@ package tests //easyjson:json type ErrorIntSlice []int + +//easyjson:json +type ErrorBoolSlice []bool + +//easyjson:json +type ErrorUintSlice []uint diff --git a/tests/errors_test.go b/tests/errors_test.go new file mode 100644 index 0000000..a434f65 --- /dev/null +++ b/tests/errors_test.go @@ -0,0 +1,44 @@ +package tests + +import ( + "testing" + + "github.com/mailru/easyjson/jlexer" +) + +func TestSemanticErrorsInt(t *testing.T) { + if !*jlexer.UseSemanticErrors { + return + } + for i, test := range []struct { + Data []byte + ErrorNum int + }{ + { + Data: []byte(`[1, 2, 3, "4", "5"]`), + ErrorNum: 2, + }, + { + Data: []byte(`[1, {"2" : "3"}, 3, "4"`), + ErrorNum: 2, + }, + { + Data: []byte(`[1, "2", "3", "4", "5", "6"]`), + ErrorNum: 5, + }, + { + Data: []byte(`[1, 2, 3, 4, "5"]`), + ErrorNum: 1, + }, + } { + l := jlexer.Lexer{Data: test.Data} + + var v ErrorIntSlice + + v.UnmarshalEasyJSON(&l) + + if len(l.SemanticErrors) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + } + } +} From 59d60409b7f162dfda83985dd42379aa580ccaa2 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Tue, 20 Dec 2016 18:04:55 +0300 Subject: [PATCH 04/16] Added tests for bool & uint --- tests/errors_test.go | 63 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/errors_test.go b/tests/errors_test.go index a434f65..af27396 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -42,3 +42,66 @@ func TestSemanticErrorsInt(t *testing.T) { } } } + +func TestSemanticErrorsBool(t *testing.T) { + if !*jlexer.UseSemanticErrors { + return + } + for i, test := range []struct { + Data []byte + ErrorNum int + }{ + { + Data: []byte(`[true, false, true, false]`), + }, + { + Data: []byte(`["test", "value", "lol", "1"]`), + ErrorNum: 4, + }, + { + Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false`), + ErrorNum: 2, + }, + } { + l := jlexer.Lexer{Data: test.Data} + + var v ErrorBoolSlice + v.UnmarshalEasyJSON(&l) + + if len(l.SemanticErrors) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + } + } +} + +func TestSemanticErrorsUint(t *testing.T) { + if !*jlexer.UseSemanticErrors { + return + } + + for i, test := range []struct { + Data []byte + ErrorNum int + }{ + { + Data: []byte(`[42, 42, 42]`), + }, + { + Data: []byte(`[17, "42", 32]`), + ErrorNum: 1, + }, + { + Data: []byte(`["zz", "zz"]`), + ErrorNum: 2, + }, + } { + l := jlexer.Lexer{Data: test.Data} + + var v ErrorUintSlice + v.UnmarshalEasyJSON(&l) + + if len(l.SemanticErrors) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + } + } +} From d7092650959df7dee08301201e36d35d9505b8c2 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Wed, 21 Dec 2016 19:21:43 +0300 Subject: [PATCH 05/16] semantic errors error -> LexerError --- jlexer/lexer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index d446fca..4b3512d 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -49,8 +49,8 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. - fatalError error // Fatal error occured during lexing. It is usually a syntax error. - SemanticErrors []error // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. + fatalError error // Fatal error occured during lexing. It is usually a syntax error. + SemanticErrors []LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. } // fetchToken scans the input for the next token. @@ -414,7 +414,7 @@ func (r *Lexer) errSyntax() { } func (r *Lexer) errSemantic() { // TODO: add error data. - r.AddSemanticError(&LexerError{ + r.AddSemanticError(LexerError{ Reason: "syntax error", Offset: r.pos, Data: "error occured", // TODO: fix this. @@ -1048,7 +1048,7 @@ func (r *Lexer) AddError(e error) { } } -func (r *Lexer) AddSemanticError(err error) { +func (r *Lexer) AddSemanticError(err LexerError) { r.SemanticErrors = append(r.SemanticErrors, err) } From 57d06b81361b085cbc302c84289bdcbc336939a7 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Thu, 22 Dec 2016 03:33:54 +0300 Subject: [PATCH 06/16] minor improvements --- jlexer/lexer.go | 8 ++++---- jlexer/lexer_test.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 4b3512d..455e965 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -49,8 +49,8 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. - fatalError error // Fatal error occured during lexing. It is usually a syntax error. - SemanticErrors []LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. + fatalError error // Fatal error occured during lexing. It is usually a syntax error. + SemanticErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. } // fetchToken scans the input for the next token. @@ -414,7 +414,7 @@ func (r *Lexer) errSyntax() { } func (r *Lexer) errSemantic() { // TODO: add error data. - r.AddSemanticError(LexerError{ + r.AddSemanticError(&LexerError{ Reason: "syntax error", Offset: r.pos, Data: "error occured", // TODO: fix this. @@ -1048,7 +1048,7 @@ func (r *Lexer) AddError(e error) { } } -func (r *Lexer) AddSemanticError(err LexerError) { +func (r *Lexer) AddSemanticError(err *LexerError) { r.SemanticErrors = append(r.SemanticErrors, err) } diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index c974edd..40dbcfd 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -82,7 +82,7 @@ func TestNumber(t *testing.T) { {toParse: "12.35E-15", want: "12.35E-15"}, {toParse: "12.35E15", want: "12.35E15"}, - {toParse: `"a"`, wantError: true}, + // {toParse: `"a"`, wantError: true}, // FIXME(shmel1k): disable UseSemanticErrors for tests. {toParse: "123junk", wantError: true}, {toParse: "1.2.3", wantError: true}, {toParse: "1e2e3", wantError: true}, @@ -98,9 +98,9 @@ func TestNumber(t *testing.T) { if err == nil && len(l.SemanticErrors) != 0 { err = l.SemanticErrors[0] } - if (err != nil || (len(l.SemanticErrors) != 0 && *UseSemanticErrors)) && !test.wantError { + if err != nil && !test.wantError { t.Errorf("[%d, %q] number() error: %v", i, test.toParse, err) - } else if (err == nil || (len(l.SemanticErrors) == 0 && *UseSemanticErrors)) && test.wantError { + } else if err == nil && test.wantError { t.Errorf("[%d, %q] number() ok; want error", i, test.toParse) } } From b31503d96cd5748bc8867e2e74a608b2b5fdfc35 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Thu, 22 Dec 2016 18:09:15 +0300 Subject: [PATCH 07/16] UseSemanticErrors -> false --- jlexer/lexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 455e965..ba091ee 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -15,7 +15,7 @@ import ( "unsafe" ) -var UseSemanticErrors = flag.Bool("use_many_errors", true, "Allow lexer collect semantic errors") +var UseSemanticErrors = flag.Bool("use_many_errors", false, "Allow lexer collect semantic errors") // tokenKind determines type of a token. type tokenKind byte From ed08545e5aa4dcb9e4c5caef84c79c992ef4b348 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Sun, 8 Jan 2017 17:47:01 +0300 Subject: [PATCH 08/16] Fixed copypaste and comments from github --- gen/decoder.go | 2 + jlexer/lexer.go | 264 ++++++++++++++++--------------------------- jlexer/lexer_test.go | 8 +- tests/errors.go | 13 +++ tests/errors_test.go | 140 +++++++++++++++++++---- 5 files changed, 235 insertions(+), 192 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index bafa162..0f7c5d7 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -118,6 +118,7 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" "+out+" = append("+out+", "+tmpVar+")") fmt.Fprintln(g.out, ws+" in.WantComma()") + fmt.Fprintln(g.out, ws+" in.ConsumeSemantic()") fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+" in.Delim(']')") fmt.Fprintln(g.out, ws+"}") @@ -409,6 +410,7 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { fmt.Fprintln(g.out, " default:") fmt.Fprintln(g.out, " in.SkipRecursive()") fmt.Fprintln(g.out, " }") + fmt.Fprintln(g.out, " in.ConsumeSemantic()") fmt.Fprintln(g.out, " in.WantComma()") fmt.Fprintln(g.out, " }") fmt.Fprintln(g.out, " in.Delim('}')") diff --git a/jlexer/lexer.go b/jlexer/lexer.go index ba091ee..6647d5c 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -6,7 +6,6 @@ package jlexer import ( "encoding/base64" - "flag" "fmt" "io" "reflect" @@ -15,8 +14,6 @@ import ( "unsafe" ) -var UseSemanticErrors = flag.Bool("use_many_errors", false, "Allow lexer collect semantic errors") - // tokenKind determines type of a token. type tokenKind byte @@ -49,12 +46,17 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. - fatalError error // Fatal error occured during lexing. It is usually a syntax error. - SemanticErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. + UseMultipleErrors bool // If we want to use multiple errors. + fatalError error // Fatal error occured during lexing. It is usually a syntax error. + nowSem bool // If semantic error occured during parsing. + semanticErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. } // fetchToken scans the input for the next token. func (r *Lexer) fetchToken() { + if r.nowSem { + return + } r.token.kind = tokenUndef r.start = r.pos @@ -413,15 +415,42 @@ func (r *Lexer) errSyntax() { r.errParse("syntax error") } -func (r *Lexer) errSemantic() { // TODO: add error data. +func (r *Lexer) ConsumeSemantic() { + r.nowSem = false +} + +func (r *Lexer) errSemantic(expected string) { + pos := r.pos + r.nowSem = true + r.SkipRecursive() + if len(expected) != 0 { + if expected[0] == '[' { + r.token.delimValue = '[' + } + if expected[0] == '{' { + r.token.delimValue = '{' + } + if expected[0] == ']' { + r.token.delimValue = ']' + return + } + if expected[0] == '}' { + r.token.delimValue = '}' + return + } + } r.AddSemanticError(&LexerError{ - Reason: "syntax error", - Offset: r.pos, - Data: "error occured", // TODO: fix this. + Reason: "invalid token", + Offset: pos, + Data: expected, }) } func (r *Lexer) errInvalidToken(expected string) { + if r.UseMultipleErrors { + r.errSemantic(expected) + return + } if r.fatalError == nil { var str string if len(r.token.byteValue) <= maxErrorContextLen { @@ -450,6 +479,9 @@ func (r *Lexer) Delim(c byte) { // IsDelim returns true if there was no scanning error and next token is the given delimiter. func (r *Lexer) IsDelim(c byte) bool { + if r.nowSem { + return true + } if r.token.kind == tokenUndef && r.Ok() { r.fetchToken() } @@ -489,7 +521,6 @@ func (r *Lexer) Skip() { // Note: no syntax validation is performed on the skipped data. func (r *Lexer) SkipRecursive() { r.scanToken() - var start, end byte if r.token.delimValue == '{' { @@ -598,7 +629,6 @@ func (r *Lexer) String() string { if !r.Ok() || r.token.kind != tokenString { r.errInvalidToken("string") return "" - } ret := string(r.token.byteValue) r.consume() @@ -633,11 +663,6 @@ func (r *Lexer) Bool() bool { r.fetchToken() } if !r.Ok() || r.token.kind != tokenBool { - if *UseSemanticErrors { // FIXME: remove copypaste from all methods. - r.errSemantic() - r.SkipRecursive() // FIXME: - return false - } r.errInvalidToken("bool") return false } @@ -651,9 +676,6 @@ func (r *Lexer) number() string { r.fetchToken() } if !r.Ok() || r.token.kind != tokenNumber { - if *UseSemanticErrors { - return "" - } r.errInvalidToken("number") return "" } @@ -664,19 +686,14 @@ func (r *Lexer) number() string { func (r *Lexer) Uint8() uint8 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 8) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint8(n) @@ -684,19 +701,14 @@ func (r *Lexer) Uint8() uint8 { func (r *Lexer) Uint16() uint16 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 16) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint16(n) @@ -704,19 +716,14 @@ func (r *Lexer) Uint16() uint16 { func (r *Lexer) Uint32() uint32 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint32(n) @@ -724,19 +731,14 @@ func (r *Lexer) Uint32() uint32 { func (r *Lexer) Uint64() uint64 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -748,19 +750,14 @@ func (r *Lexer) Uint() uint { func (r *Lexer) Int8() int8 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 8) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int8(n) @@ -768,19 +765,14 @@ func (r *Lexer) Int8() int8 { func (r *Lexer) Int16() int16 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 16) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int16(n) @@ -788,19 +780,14 @@ func (r *Lexer) Int16() int16 { func (r *Lexer) Int32() int32 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int32(n) @@ -808,19 +795,14 @@ func (r *Lexer) Int32() int32 { func (r *Lexer) Int64() int64 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -832,19 +814,14 @@ func (r *Lexer) Int() int { func (r *Lexer) Uint8Str() uint8 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 8) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint8(n) @@ -852,19 +829,14 @@ func (r *Lexer) Uint8Str() uint8 { func (r *Lexer) Uint16Str() uint16 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 16) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint16(n) @@ -872,19 +844,14 @@ func (r *Lexer) Uint16Str() uint16 { func (r *Lexer) Uint32Str() uint32 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint32(n) @@ -892,19 +859,14 @@ func (r *Lexer) Uint32Str() uint32 { func (r *Lexer) Uint64Str() uint64 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -916,19 +878,14 @@ func (r *Lexer) UintStr() uint { func (r *Lexer) Int8Str() int8 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 8) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int8(n) @@ -936,19 +893,14 @@ func (r *Lexer) Int8Str() int8 { func (r *Lexer) Int16Str() int16 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 16) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int16(n) @@ -956,19 +908,14 @@ func (r *Lexer) Int16Str() int16 { func (r *Lexer) Int32Str() int32 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int32(n) @@ -976,19 +923,14 @@ func (r *Lexer) Int32Str() int32 { func (r *Lexer) Int64Str() int64 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -1000,19 +942,14 @@ func (r *Lexer) IntStr() int { func (r *Lexer) Float32() float32 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseFloat(s, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return float32(n) @@ -1020,19 +957,14 @@ func (r *Lexer) Float32() float32 { func (r *Lexer) Float64() float64 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseFloat(s, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -1049,7 +981,11 @@ func (r *Lexer) AddError(e error) { } func (r *Lexer) AddSemanticError(err *LexerError) { - r.SemanticErrors = append(r.SemanticErrors, err) + r.semanticErrors = append(r.semanticErrors, err) +} + +func (r *Lexer) GetSemanticErrors() []*LexerError { + return r.semanticErrors } // Interface fetches an interface{} analogous to the 'encoding/json' package. diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index 40dbcfd..e3add8c 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -82,7 +82,7 @@ func TestNumber(t *testing.T) { {toParse: "12.35E-15", want: "12.35E-15"}, {toParse: "12.35E15", want: "12.35E15"}, - // {toParse: `"a"`, wantError: true}, // FIXME(shmel1k): disable UseSemanticErrors for tests. + {toParse: `"a"`, wantError: true}, {toParse: "123junk", wantError: true}, {toParse: "1.2.3", wantError: true}, {toParse: "1e2e3", wantError: true}, @@ -95,9 +95,6 @@ func TestNumber(t *testing.T) { t.Errorf("[%d, %q] number() = %v; want %v", i, test.toParse, got, test.want) } err := l.Error() - if err == nil && len(l.SemanticErrors) != 0 { - err = l.SemanticErrors[0] - } if err != nil && !test.wantError { t.Errorf("[%d, %q] number() error: %v", i, test.toParse, err) } else if err == nil && test.wantError { @@ -128,9 +125,6 @@ func TestBool(t *testing.T) { t.Errorf("[%d, %q] Bool() = %v; want %v", i, test.toParse, got, test.want) } err := l.Error() - if err == nil && len(l.SemanticErrors) != 0 { - err = l.SemanticErrors[0] - } if err != nil && !test.wantError { t.Errorf("[%d, %q] Bool() error: %v", i, test.toParse, err) } else if err == nil && test.wantError { diff --git a/tests/errors.go b/tests/errors.go index a0180ad..2ec3299 100644 --- a/tests/errors.go +++ b/tests/errors.go @@ -8,3 +8,16 @@ type ErrorBoolSlice []bool //easyjson:json type ErrorUintSlice []uint + +//easyjson:json +type ErrorStruct struct { + Int int `json:"int"` + String string `json:"string"` + Slice []int `json:"slice"` + IntSlice []int `json:"int_slice"` +} + +type ErrorNestedStruct struct { + ErrorStruct ErrorStruct `json:"error_struct"` + Int int `json:"int"` +} diff --git a/tests/errors_test.go b/tests/errors_test.go index af27396..ee1ea72 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -7,9 +7,6 @@ import ( ) func TestSemanticErrorsInt(t *testing.T) { - if !*jlexer.UseSemanticErrors { - return - } for i, test := range []struct { Data []byte ErrorNum int @@ -19,7 +16,7 @@ func TestSemanticErrorsInt(t *testing.T) { ErrorNum: 2, }, { - Data: []byte(`[1, {"2" : "3"}, 3, "4"`), + Data: []byte(`[1, {"2":"3"}, 3, "4"]`), ErrorNum: 2, }, { @@ -30,23 +27,29 @@ func TestSemanticErrorsInt(t *testing.T) { Data: []byte(`[1, 2, 3, 4, "5"]`), ErrorNum: 1, }, + { + Data: []byte(`[{"1": "2"}]`), + ErrorNum: 1, + }, } { - l := jlexer.Lexer{Data: test.Data} + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } var v ErrorIntSlice v.UnmarshalEasyJSON(&l) - if len(l.SemanticErrors) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsInt(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + t.Errorf("%v", l.GetSemanticErrors()) + t.Errorf("%v", l.Error()) } } } func TestSemanticErrorsBool(t *testing.T) { - if !*jlexer.UseSemanticErrors { - return - } for i, test := range []struct { Data []byte ErrorNum int @@ -59,26 +62,26 @@ func TestSemanticErrorsBool(t *testing.T) { ErrorNum: 4, }, { - Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false`), + Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false]`), ErrorNum: 2, }, } { - l := jlexer.Lexer{Data: test.Data} + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } var v ErrorBoolSlice v.UnmarshalEasyJSON(&l) - if len(l.SemanticErrors) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsBool(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + t.Errorf("%v", l.Error()) } } } func TestSemanticErrorsUint(t *testing.T) { - if !*jlexer.UseSemanticErrors { - return - } - for i, test := range []struct { Data []byte ErrorNum int @@ -94,14 +97,109 @@ func TestSemanticErrorsUint(t *testing.T) { Data: []byte(`["zz", "zz"]`), ErrorNum: 2, }, + { + Data: []byte(`[{}, 42]`), + ErrorNum: 1, + }, } { - l := jlexer.Lexer{Data: test.Data} + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } var v ErrorUintSlice v.UnmarshalEasyJSON(&l) - if len(l.SemanticErrors) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsUint(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + } + } +} + +func TestSemanticErrorsStruct(t *testing.T) { + for i, test := range []struct { + Data []byte + ErrorNum int + }{ + { + Data: []byte(`{"string": "test", "slice":[42, 42, 42], "int_slice":[1, 2, 3]}`), + }, + { + Data: []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`), + ErrorNum: 2, + }, + { + Data: []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`), + ErrorNum: 3, + }, + { + Data: []byte(`{"string": "test", "slice": {}}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"slice":5, "string" : "test"}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"slice" : "test", "string" : "test"}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"slice": "", "string" : {}, "int":{}}`), + ErrorNum: 3, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + var v ErrorStruct + v.UnmarshalEasyJSON(&l) + + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + } + } +} + +func TestSemanticErrorsNestedStruct(t *testing.T) { + for i, test := range []struct { + Data []byte + ErrorNum int + }{ + { + Data: []byte(`{"error_struct":{}}`), + }, + { + Data: []byte(`{"error_struct":5}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"error_struct":[]}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"error_struct":{"int":{}}}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"error_struct":{"int_slice":{}}, "int":4}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`), + ErrorNum: 3, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + var v ErrorNestedStruct + v.UnmarshalEasyJSON(&l) + + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsNestedStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) } } } From 0c9c34b8cca370ef8206a62d6a75f625afc006dc Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Wed, 11 Jan 2017 03:18:07 +0300 Subject: [PATCH 09/16] Fixed tests, added comments --- gen/decoder.go | 2 - jlexer/lexer.go | 103 ++++++++++++-------------- tests/errors_test.go | 173 ++++++++++++++++++++++++++----------------- 3 files changed, 151 insertions(+), 127 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index 0f7c5d7..bafa162 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -118,7 +118,6 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" "+out+" = append("+out+", "+tmpVar+")") fmt.Fprintln(g.out, ws+" in.WantComma()") - fmt.Fprintln(g.out, ws+" in.ConsumeSemantic()") fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+" in.Delim(']')") fmt.Fprintln(g.out, ws+"}") @@ -410,7 +409,6 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { fmt.Fprintln(g.out, " default:") fmt.Fprintln(g.out, " in.SkipRecursive()") fmt.Fprintln(g.out, " }") - fmt.Fprintln(g.out, " in.ConsumeSemantic()") fmt.Fprintln(g.out, " in.WantComma()") fmt.Fprintln(g.out, " }") fmt.Fprintln(g.out, " in.Delim('}')") diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 6647d5c..b23950d 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -49,14 +49,11 @@ type Lexer struct { UseMultipleErrors bool // If we want to use multiple errors. fatalError error // Fatal error occured during lexing. It is usually a syntax error. nowSem bool // If semantic error occured during parsing. - semanticErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. + multipleErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. } // fetchToken scans the input for the next token. func (r *Lexer) fetchToken() { - if r.nowSem { - return - } r.token.kind = tokenUndef r.start = r.pos @@ -415,34 +412,30 @@ func (r *Lexer) errSyntax() { r.errParse("syntax error") } -func (r *Lexer) ConsumeSemantic() { - r.nowSem = false -} - func (r *Lexer) errSemantic(expected string) { - pos := r.pos - r.nowSem = true + r.pos = r.start + r.consume() r.SkipRecursive() - if len(expected) != 0 { - if expected[0] == '[' { - r.token.delimValue = '[' - } - if expected[0] == '{' { - r.token.delimValue = '{' - } - if expected[0] == ']' { - r.token.delimValue = ']' - return - } - if expected[0] == '}' { - r.token.delimValue = '}' - return - } + switch expected { + case "[": + r.token.delimValue = ']' + r.token.kind = tokenDelim + case "{": + r.token.delimValue = '}' + r.token.kind = tokenDelim + case "]": + r.token.delimValue = ']' + r.token.kind = tokenDelim + return + case "}": + r.token.delimValue = '}' + r.token.kind = tokenDelim + return } - r.AddSemanticError(&LexerError{ - Reason: "invalid token", - Offset: pos, - Data: expected, + r.AddMultipleError(&LexerError{ + Reason: fmt.Sprintf("expected %s", expected), + Offset: r.start, + Data: string(r.Data[r.start:]), }) } @@ -471,17 +464,17 @@ func (r *Lexer) Delim(c byte) { if r.token.kind == tokenUndef && r.Ok() { r.fetchToken() } + if !r.Ok() || r.token.delimValue != c { + r.consume() // errInvalidToken can change token if UseMultipleErrors is enabled. r.errInvalidToken(string([]byte{c})) + } else { + r.consume() } - r.consume() } // IsDelim returns true if there was no scanning error and next token is the given delimiter. func (r *Lexer) IsDelim(c byte) bool { - if r.nowSem { - return true - } if r.token.kind == tokenUndef && r.Ok() { r.fetchToken() } @@ -686,7 +679,7 @@ func (r *Lexer) number() string { func (r *Lexer) Uint8() uint8 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -701,7 +694,7 @@ func (r *Lexer) Uint8() uint8 { func (r *Lexer) Uint16() uint16 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -716,7 +709,7 @@ func (r *Lexer) Uint16() uint16 { func (r *Lexer) Uint32() uint32 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -731,7 +724,7 @@ func (r *Lexer) Uint32() uint32 { func (r *Lexer) Uint64() uint64 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -750,7 +743,7 @@ func (r *Lexer) Uint() uint { func (r *Lexer) Int8() int8 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -765,7 +758,7 @@ func (r *Lexer) Int8() int8 { func (r *Lexer) Int16() int16 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -780,7 +773,7 @@ func (r *Lexer) Int16() int16 { func (r *Lexer) Int32() int32 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -795,7 +788,7 @@ func (r *Lexer) Int32() int32 { func (r *Lexer) Int64() int64 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -814,7 +807,7 @@ func (r *Lexer) Int() int { func (r *Lexer) Uint8Str() uint8 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -829,7 +822,7 @@ func (r *Lexer) Uint8Str() uint8 { func (r *Lexer) Uint16Str() uint16 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -844,7 +837,7 @@ func (r *Lexer) Uint16Str() uint16 { func (r *Lexer) Uint32Str() uint32 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -859,7 +852,7 @@ func (r *Lexer) Uint32Str() uint32 { func (r *Lexer) Uint64Str() uint64 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -878,7 +871,7 @@ func (r *Lexer) UintStr() uint { func (r *Lexer) Int8Str() int8 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -893,7 +886,7 @@ func (r *Lexer) Int8Str() int8 { func (r *Lexer) Int16Str() int16 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -908,7 +901,7 @@ func (r *Lexer) Int16Str() int16 { func (r *Lexer) Int32Str() int32 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -923,7 +916,7 @@ func (r *Lexer) Int32Str() int32 { func (r *Lexer) Int64Str() int64 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -942,7 +935,7 @@ func (r *Lexer) IntStr() int { func (r *Lexer) Float32() float32 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -957,7 +950,7 @@ func (r *Lexer) Float32() float32 { func (r *Lexer) Float64() float64 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -980,12 +973,12 @@ func (r *Lexer) AddError(e error) { } } -func (r *Lexer) AddSemanticError(err *LexerError) { - r.semanticErrors = append(r.semanticErrors, err) +func (r *Lexer) AddMultipleError(err *LexerError) { + r.multipleErrors = append(r.multipleErrors, err) } -func (r *Lexer) GetSemanticErrors() []*LexerError { - return r.semanticErrors +func (r *Lexer) GetMultipleErrors() []*LexerError { + return r.multipleErrors } // Interface fetches an interface{} analogous to the 'encoding/json' package. diff --git a/tests/errors_test.go b/tests/errors_test.go index ee1ea72..2575c22 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -6,30 +6,30 @@ import ( "github.com/mailru/easyjson/jlexer" ) -func TestSemanticErrorsInt(t *testing.T) { +func TestMultipleErrorsInt(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { - Data: []byte(`[1, 2, 3, "4", "5"]`), - ErrorNum: 2, + Data: []byte(`[1, 2, 3, "4", "5"]`), + Offsets: []int{10, 15}, }, { - Data: []byte(`[1, {"2":"3"}, 3, "4"]`), - ErrorNum: 2, + Data: []byte(`[1, {"2":"3"}, 3, "4"]`), + Offsets: []int{4, 18}, }, { - Data: []byte(`[1, "2", "3", "4", "5", "6"]`), - ErrorNum: 5, + Data: []byte(`[1, "2", "3", "4", "5", "6"]`), + Offsets: []int{4, 9, 14, 19, 24}, }, { - Data: []byte(`[1, 2, 3, 4, "5"]`), - ErrorNum: 1, + Data: []byte(`[1, 2, 3, 4, "5"]`), + Offsets: []int{13}, }, { - Data: []byte(`[{"1": "2"}]`), - ErrorNum: 1, + Data: []byte(`[{"1": "2"}]`), + Offsets: []int{1}, }, } { l := jlexer.Lexer{ @@ -41,29 +41,35 @@ func TestSemanticErrorsInt(t *testing.T) { v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsInt(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) - t.Errorf("%v", l.GetSemanticErrors()) - t.Errorf("%v", l.Error()) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsInt(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsInt(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } -func TestSemanticErrorsBool(t *testing.T) { +func TestMultipleErrorsBool(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { Data: []byte(`[true, false, true, false]`), }, { - Data: []byte(`["test", "value", "lol", "1"]`), - ErrorNum: 4, + Data: []byte(`["test", "value", "lol", "1"]`), + Offsets: []int{1, 9, 18, 25}, }, { - Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false]`), - ErrorNum: 2, + Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false]`), + Offsets: []int{7, 11}, }, } { l := jlexer.Lexer{ @@ -74,32 +80,38 @@ func TestSemanticErrorsBool(t *testing.T) { var v ErrorBoolSlice v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsBool(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) - t.Errorf("%v", l.Error()) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsBool(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsBool(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } -func TestSemanticErrorsUint(t *testing.T) { +func TestMultipleErrorsUint(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { Data: []byte(`[42, 42, 42]`), }, { - Data: []byte(`[17, "42", 32]`), - ErrorNum: 1, + Data: []byte(`[17, "42", 32]`), + Offsets: []int{5}, }, { - Data: []byte(`["zz", "zz"]`), - ErrorNum: 2, + Data: []byte(`["zz", "zz"]`), + Offsets: []int{1, 7}, }, { - Data: []byte(`[{}, 42]`), - ErrorNum: 1, + Data: []byte(`[{}, 42]`), + Offsets: []int{1}, }, } { l := jlexer.Lexer{ @@ -110,43 +122,50 @@ func TestSemanticErrorsUint(t *testing.T) { var v ErrorUintSlice v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsUint(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsUint(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsUint(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } -func TestSemanticErrorsStruct(t *testing.T) { +func TestMultipleErrorsStruct(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { Data: []byte(`{"string": "test", "slice":[42, 42, 42], "int_slice":[1, 2, 3]}`), }, { - Data: []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`), - ErrorNum: 2, + Data: []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`), + Offsets: []int{11, 64}, }, { - Data: []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`), - ErrorNum: 3, + Data: []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`), + Offsets: []int{30, 61, 66}, }, { - Data: []byte(`{"string": "test", "slice": {}}`), - ErrorNum: 1, + Data: []byte(`{"string": "test", "slice": {}}`), + Offsets: []int{28}, }, { - Data: []byte(`{"slice":5, "string" : "test"}`), - ErrorNum: 1, + Data: []byte(`{"slice":5, "string" : "test"}`), + Offsets: []int{9}, }, { - Data: []byte(`{"slice" : "test", "string" : "test"}`), - ErrorNum: 1, + Data: []byte(`{"slice" : "test", "string" : "test"}`), + Offsets: []int{11}, }, { - Data: []byte(`{"slice": "", "string" : {}, "int":{}}`), - ErrorNum: 3, + Data: []byte(`{"slice": "", "string" : {}, "int":{}}`), + Offsets: []int{10, 25, 35}, }, } { l := jlexer.Lexer{ @@ -156,39 +175,46 @@ func TestSemanticErrorsStruct(t *testing.T) { var v ErrorStruct v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } -func TestSemanticErrorsNestedStruct(t *testing.T) { +func TestMultipleErrorsNestedStruct(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { Data: []byte(`{"error_struct":{}}`), }, { - Data: []byte(`{"error_struct":5}`), - ErrorNum: 1, + Data: []byte(`{"error_struct":5}`), + Offsets: []int{16}, }, { - Data: []byte(`{"error_struct":[]}`), - ErrorNum: 1, + Data: []byte(`{"error_struct":[]}`), + Offsets: []int{16}, }, { - Data: []byte(`{"error_struct":{"int":{}}}`), - ErrorNum: 1, + Data: []byte(`{"error_struct":{"int":{}}}`), + Offsets: []int{23}, }, { - Data: []byte(`{"error_struct":{"int_slice":{}}, "int":4}`), - ErrorNum: 1, + Data: []byte(`{"error_struct":{"int_slice":{}}, "int":4}`), + Offsets: []int{29}, }, { - Data: []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`), - ErrorNum: 3, + Data: []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`), + Offsets: []int{30, 38, 51}, }, } { l := jlexer.Lexer{ @@ -198,8 +224,15 @@ func TestSemanticErrorsNestedStruct(t *testing.T) { var v ErrorNestedStruct v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsNestedStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsNestedStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsNestedStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } From 4bab4951d6e32bc304b05aeb044b2d2e8d3f5d92 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Wed, 11 Jan 2017 19:46:47 +0300 Subject: [PATCH 10/16] Codestyle fixes --- jlexer/lexer.go | 139 +++++++++++++++++++++++++------------------ tests/errors_test.go | 15 +++-- 2 files changed, 92 insertions(+), 62 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index b23950d..388b552 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -432,7 +432,7 @@ func (r *Lexer) errSemantic(expected string) { r.token.kind = tokenDelim return } - r.AddMultipleError(&LexerError{ + r.addNonfatalError(&LexerError{ Reason: fmt.Sprintf("expected %s", expected), Offset: r.start, Data: string(r.Data[r.start:]), @@ -679,60 +679,64 @@ func (r *Lexer) number() string { func (r *Lexer) Uint8() uint8 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 8) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint8(n) } func (r *Lexer) Uint16() uint16 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 16) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint16(n) } func (r *Lexer) Uint32() uint32 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 32) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint32(n) } func (r *Lexer) Uint64() uint64 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 64) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -743,60 +747,64 @@ func (r *Lexer) Uint() uint { func (r *Lexer) Int8() int8 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 8) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int8(n) } func (r *Lexer) Int16() int16 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 16) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int16(n) } func (r *Lexer) Int32() int32 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 32) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int32(n) } func (r *Lexer) Int64() int64 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 64) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -807,60 +815,64 @@ func (r *Lexer) Int() int { func (r *Lexer) Uint8Str() uint8 { s := r.UnsafeString() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 8) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint8(n) } func (r *Lexer) Uint16Str() uint16 { s := r.UnsafeString() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 16) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint16(n) } func (r *Lexer) Uint32Str() uint32 { s := r.UnsafeString() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 32) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return uint32(n) } func (r *Lexer) Uint64Str() uint64 { s := r.UnsafeString() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 64) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -871,30 +883,32 @@ func (r *Lexer) UintStr() uint { func (r *Lexer) Int8Str() int8 { s := r.UnsafeString() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 8) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int8(n) } func (r *Lexer) Int16Str() int16 { s := r.UnsafeString() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 16) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int16(n) } @@ -907,24 +921,26 @@ func (r *Lexer) Int32Str() int32 { n, err := strconv.ParseInt(s, 10, 32) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return int32(n) } func (r *Lexer) Int64Str() int64 { s := r.UnsafeString() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 64) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -935,30 +951,32 @@ func (r *Lexer) IntStr() int { func (r *Lexer) Float32() float32 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseFloat(s, 32) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return float32(n) } func (r *Lexer) Float64() float64 { s := r.number() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } n, err := strconv.ParseFloat(s, 64) if err != nil { - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + }) } return n } @@ -973,11 +991,18 @@ func (r *Lexer) AddError(e error) { } } -func (r *Lexer) AddMultipleError(err *LexerError) { - r.multipleErrors = append(r.multipleErrors, err) +func (r *Lexer) addNonfatalError(err *LexerError) { + if r.UseMultipleErrors { + if len(r.multipleErrors) != 0 && r.multipleErrors[len(r.multipleErrors)-1].Offset == err.Offset { + return + } + r.multipleErrors = append(r.multipleErrors, err) + return + } + r.fatalError = err } -func (r *Lexer) GetMultipleErrors() []*LexerError { +func (r *Lexer) GetNonfatalErrors() []*LexerError { return r.multipleErrors } diff --git a/tests/errors_test.go b/tests/errors_test.go index 2575c22..4d0e99b 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -41,10 +41,11 @@ func TestMultipleErrorsInt(t *testing.T) { v.UnmarshalEasyJSON(&l) - errors := l.GetMultipleErrors() + errors := l.GetNonfatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsInt(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return } for ii, e := range errors { @@ -80,10 +81,11 @@ func TestMultipleErrorsBool(t *testing.T) { var v ErrorBoolSlice v.UnmarshalEasyJSON(&l) - errors := l.GetMultipleErrors() + errors := l.GetNonfatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsBool(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return } for ii, e := range errors { if e.Offset != test.Offsets[ii] { @@ -122,10 +124,11 @@ func TestMultipleErrorsUint(t *testing.T) { var v ErrorUintSlice v.UnmarshalEasyJSON(&l) - errors := l.GetMultipleErrors() + errors := l.GetNonfatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsUint(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return } for ii, e := range errors { if e.Offset != test.Offsets[ii] { @@ -175,10 +178,11 @@ func TestMultipleErrorsStruct(t *testing.T) { var v ErrorStruct v.UnmarshalEasyJSON(&l) - errors := l.GetMultipleErrors() + errors := l.GetNonfatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return } for ii, e := range errors { if e.Offset != test.Offsets[ii] { @@ -224,10 +228,11 @@ func TestMultipleErrorsNestedStruct(t *testing.T) { var v ErrorNestedStruct v.UnmarshalEasyJSON(&l) - errors := l.GetMultipleErrors() + errors := l.GetNonfatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsNestedStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + return } for ii, e := range errors { if e.Offset != test.Offsets[ii] { From 0e676032de96c2d9641a8ff467d068fc7685cf46 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Wed, 11 Jan 2017 19:49:25 +0300 Subject: [PATCH 11/16] Removed context --- jlexer/context.go | 92 ------------------------------------------ jlexer/context_test.go | 85 -------------------------------------- 2 files changed, 177 deletions(-) delete mode 100644 jlexer/context.go delete mode 100644 jlexer/context_test.go diff --git a/jlexer/context.go b/jlexer/context.go deleted file mode 100644 index ec53902..0000000 --- a/jlexer/context.go +++ /dev/null @@ -1,92 +0,0 @@ -package jlexer - -import "io" - -type Walker interface { - OnEnterObject() - OnNextObjectKey(key string) - OnExitObject() - - OnEnterArray() - OnNextArrayElement() - OnExitArray() -} - -func (l *Lexer) WalkUpToPosition(w Walker) error { - l1 := &Lexer{} - l1.Data = l.Data - - type StackItem int - const ( - Object StackItem = iota - Array - ) - - var stack []StackItem - haveKey := false - - for { - l1.fetchToken() - if l1.pos > l.pos || !l1.Ok() { - break - } - switch { - case l1.IsDelim('{'): - l1.Skip() - - stack = append(stack, Object) - w.OnEnterObject() - haveKey = false - - case l1.IsDelim('}'): - l1.Skip() - - if len(stack) > 0 { - stack = stack[:len(stack)-1] - l1.WantComma() - } - w.OnExitObject() - haveKey = false - - case l1.IsDelim('['): - l1.Skip() - - stack = append(stack, Array) - w.OnEnterArray() - haveKey = false - - case l1.IsDelim(']'): - l1.Skip() - - if len(stack) > 0 { - stack = stack[:len(stack)-1] - l1.WantComma() - } - w.OnExitArray() - haveKey = false - - case len(stack) > 0 && stack[len(stack)-1] == Object && !haveKey: - key := l1.UnsafeString() - w.OnNextObjectKey(key) - - l1.WantColon() - haveKey = true - - case len(stack) > 0 && stack[len(stack)-1] == Array: - w.OnNextArrayElement() - l1.Skip() - l1.WantComma() - - default: - l1.Skip() - l1.WantComma() - haveKey = false - } - - } - - if l1.Error() == io.EOF { - return nil - } - return l1.Error() -} diff --git a/jlexer/context_test.go b/jlexer/context_test.go deleted file mode 100644 index 3b7ee58..0000000 --- a/jlexer/context_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package jlexer - -import ( - "fmt" - "testing" -) - -type TestWalker struct { - ErrorPrefix string - T *testing.T - Items []string -} - -func (w *TestWalker) item(s string) { - if len(w.Items) == 0 { - w.T.Errorf("%sTestWalker(): no items left; want %q", w.ErrorPrefix, s) - } else if w.Items[0] != s { - w.T.Errorf("%sTestWalker(): got %q; want %q", w.ErrorPrefix, s, w.Items[0]) - w.Items = w.Items[1:] - } else { - w.Items = w.Items[1:] - } -} - -func (w *TestWalker) OnEnterObject() { w.item("{") } -func (w *TestWalker) OnExitObject() { w.item("}") } -func (w *TestWalker) OnNextObjectKey(key string) { w.item("e:" + key) } -func (w *TestWalker) OnEnterArray() { w.item("[") } -func (w *TestWalker) OnNextArrayElement() { w.item("e") } -func (w *TestWalker) OnExitArray() { w.item("]") } - -func TestWalkUpToPosition(t *testing.T) { - for i, test := range []struct { - JSON string - Items []string - Start, End int - }{ - { - JSON: ``, - Items: []string{}, - End: -1, - }, { - JSON: `{"aaa": 5, "qqq": 10}`, - Items: []string{"{", "e:aaa", "e:qqq", "}"}, - End: -1, - }, { - JSON: `{"aaa": 5, "qqq": {"\t\t": null}}`, - Items: []string{"{", "e:aaa", "e:qqq", "{", "e:\t\t", "}", "}"}, - End: -1, - }, { - JSON: `{"aaa": 5, "qqq": 10}`, - End: len(`{"aaa": 5, "qqq": `), - Items: []string{"{", "e:aaa", "e:qqq"}, - }, { - JSON: `{"aaa": 5, "qqq": 10}`, - End: len(`{"aaa": 5, `), - Items: []string{"{", "e:aaa"}, - }, { - JSON: `[null, false, {"aaa": 5}]`, - Items: []string{"[", "e", "e", "{", "e:aaa", "}", "]"}, - End: -1, - }, { - JSON: `[null, "aaa"]`, - Items: []string{"[", "e", "e", "]"}, - End: -1, - }, - } { - l := &Lexer{Data: []byte(test.JSON)} - - if test.End != -1 { - l.pos = test.End - } else { - l.pos = len(test.JSON) - } - w := &TestWalker{T: t, Items: test.Items, ErrorPrefix: fmt.Sprintf("[%d,%q] ", i, test.JSON)} - - if err := l.WalkUpToPosition(w); err != nil { - t.Errorf("[%d,%q] WalkUpToPosition() error: %v", i, test.JSON, err) - } - - if len(w.Items) > 0 { - t.Errorf("[%d,%q] WalkUpToPosition: items %q left", i, test.JSON, w.Items) - } - } -} From 172c47e5a18f4d102ea900c3b8a82f8f6fd2af55 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Wed, 11 Jan 2017 23:49:11 +0300 Subject: [PATCH 12/16] minor fix --- jlexer/lexer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 388b552..79fe350 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -412,7 +412,7 @@ func (r *Lexer) errSyntax() { r.errParse("syntax error") } -func (r *Lexer) errSemantic(expected string) { +func (r *Lexer) errMultiple(expected string) { r.pos = r.start r.consume() r.SkipRecursive() @@ -441,7 +441,7 @@ func (r *Lexer) errSemantic(expected string) { func (r *Lexer) errInvalidToken(expected string) { if r.UseMultipleErrors { - r.errSemantic(expected) + r.errMultiple(expected) return } if r.fatalError == nil { @@ -915,7 +915,7 @@ func (r *Lexer) Int16Str() int16 { func (r *Lexer) Int32Str() int32 { s := r.UnsafeString() - if !r.Ok() || len(r.multipleErrors) != 0 { + if !r.Ok() { return 0 } From cba450a81e412ffc2cb16b4857c6fa6462c0ff35 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Thu, 12 Jan 2017 16:30:30 +0300 Subject: [PATCH 13/16] Added comment for errMultiple --- jlexer/lexer.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 79fe350..255ef26 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -412,6 +412,10 @@ func (r *Lexer) errSyntax() { r.errParse("syntax error") } +// errMultiple adds multiple error if UseMultipleErrors is enabled. +// +// Function changes lexer's token if we expected '[' or '{' for slices/maps in order not to +// call IsDelim() or Delim() functions. func (r *Lexer) errMultiple(expected string) { r.pos = r.start r.consume() From d3c70aea595ee6ac8ef46c2932366ca0be408882 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Thu, 12 Jan 2017 16:34:28 +0300 Subject: [PATCH 14/16] Renamed function --- jlexer/lexer.go | 2 +- tests/errors_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 255ef26..56b9b23 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -1006,7 +1006,7 @@ func (r *Lexer) addNonfatalError(err *LexerError) { r.fatalError = err } -func (r *Lexer) GetNonfatalErrors() []*LexerError { +func (r *Lexer) GetNonFatalErrors() []*LexerError { return r.multipleErrors } diff --git a/tests/errors_test.go b/tests/errors_test.go index 4d0e99b..756f7db 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -41,7 +41,7 @@ func TestMultipleErrorsInt(t *testing.T) { v.UnmarshalEasyJSON(&l) - errors := l.GetNonfatalErrors() + errors := l.GetNonFatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsInt(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) @@ -81,7 +81,7 @@ func TestMultipleErrorsBool(t *testing.T) { var v ErrorBoolSlice v.UnmarshalEasyJSON(&l) - errors := l.GetNonfatalErrors() + errors := l.GetNonFatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsBool(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) @@ -124,7 +124,7 @@ func TestMultipleErrorsUint(t *testing.T) { var v ErrorUintSlice v.UnmarshalEasyJSON(&l) - errors := l.GetNonfatalErrors() + errors := l.GetNonFatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsUint(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) @@ -178,7 +178,7 @@ func TestMultipleErrorsStruct(t *testing.T) { var v ErrorStruct v.UnmarshalEasyJSON(&l) - errors := l.GetNonfatalErrors() + errors := l.GetNonFatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) @@ -228,7 +228,7 @@ func TestMultipleErrorsNestedStruct(t *testing.T) { var v ErrorNestedStruct v.UnmarshalEasyJSON(&l) - errors := l.GetNonfatalErrors() + errors := l.GetNonFatalErrors() if len(errors) != len(test.Offsets) { t.Errorf("[%d] TestMultipleErrorsNestedStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) From c4649045ea641fcf8a72c9ede3b211ff42b7cfb6 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Thu, 12 Jan 2017 17:18:23 +0300 Subject: [PATCH 15/16] minor fixes --- jlexer/lexer.go | 80 +++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 56b9b23..662b45d 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -412,54 +412,48 @@ func (r *Lexer) errSyntax() { r.errParse("syntax error") } -// errMultiple adds multiple error if UseMultipleErrors is enabled. -// -// Function changes lexer's token if we expected '[' or '{' for slices/maps in order not to -// call IsDelim() or Delim() functions. -func (r *Lexer) errMultiple(expected string) { - r.pos = r.start - r.consume() - r.SkipRecursive() - switch expected { - case "[": - r.token.delimValue = ']' - r.token.kind = tokenDelim - case "{": - r.token.delimValue = '}' - r.token.kind = tokenDelim - case "]": - r.token.delimValue = ']' - r.token.kind = tokenDelim - return - case "}": - r.token.delimValue = '}' - r.token.kind = tokenDelim - return - } - r.addNonfatalError(&LexerError{ - Reason: fmt.Sprintf("expected %s", expected), - Offset: r.start, - Data: string(r.Data[r.start:]), - }) -} - func (r *Lexer) errInvalidToken(expected string) { - if r.UseMultipleErrors { - r.errMultiple(expected) + if r.fatalError != nil { return } - if r.fatalError == nil { - var str string - if len(r.token.byteValue) <= maxErrorContextLen { - str = string(r.token.byteValue) - } else { - str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..." + if r.UseMultipleErrors { + r.pos = r.start + r.consume() + r.SkipRecursive() + switch expected { + case "[": + r.token.delimValue = ']' + r.token.kind = tokenDelim + case "{": + r.token.delimValue = '}' + r.token.kind = tokenDelim + case "]": + r.token.delimValue = ']' + r.token.kind = tokenDelim + return + case "}": + r.token.delimValue = '}' + r.token.kind = tokenDelim + return } - r.fatalError = &LexerError{ + r.addNonfatalError(&LexerError{ Reason: fmt.Sprintf("expected %s", expected), - Offset: r.pos, - Data: str, - } + Offset: r.start, + Data: string(r.Data[r.start:]), + }) + return + } + + var str string + if len(r.token.byteValue) <= maxErrorContextLen { + str = string(r.token.byteValue) + } else { + str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..." + } + r.fatalError = &LexerError{ + Reason: fmt.Sprintf("expected %s", expected), + Offset: r.pos, + Data: str, } } From ebe9ab918ef2435ed9fc8b6183a4ddedf1ded2d7 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Thu, 12 Jan 2017 17:50:06 +0300 Subject: [PATCH 16/16] Removed unused code --- jlexer/lexer.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 662b45d..7b48022 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -427,14 +427,6 @@ func (r *Lexer) errInvalidToken(expected string) { case "{": r.token.delimValue = '}' r.token.kind = tokenDelim - case "]": - r.token.delimValue = ']' - r.token.kind = tokenDelim - return - case "}": - r.token.delimValue = '}' - r.token.kind = tokenDelim - return } r.addNonfatalError(&LexerError{ Reason: fmt.Sprintf("expected %s", expected),