From ed08545e5aa4dcb9e4c5caef84c79c992ef4b348 Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Sun, 8 Jan 2017 17:47:01 +0300 Subject: [PATCH] Fixed copypaste and comments from github --- gen/decoder.go | 2 + jlexer/lexer.go | 264 ++++++++++++++++--------------------------- jlexer/lexer_test.go | 8 +- tests/errors.go | 13 +++ tests/errors_test.go | 140 +++++++++++++++++++---- 5 files changed, 235 insertions(+), 192 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index bafa162..0f7c5d7 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -118,6 +118,7 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" "+out+" = append("+out+", "+tmpVar+")") fmt.Fprintln(g.out, ws+" in.WantComma()") + fmt.Fprintln(g.out, ws+" in.ConsumeSemantic()") fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+" in.Delim(']')") fmt.Fprintln(g.out, ws+"}") @@ -409,6 +410,7 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { fmt.Fprintln(g.out, " default:") fmt.Fprintln(g.out, " in.SkipRecursive()") fmt.Fprintln(g.out, " }") + fmt.Fprintln(g.out, " in.ConsumeSemantic()") fmt.Fprintln(g.out, " in.WantComma()") fmt.Fprintln(g.out, " }") fmt.Fprintln(g.out, " in.Delim('}')") diff --git a/jlexer/lexer.go b/jlexer/lexer.go index ba091ee..6647d5c 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -6,7 +6,6 @@ package jlexer import ( "encoding/base64" - "flag" "fmt" "io" "reflect" @@ -15,8 +14,6 @@ import ( "unsafe" ) -var UseSemanticErrors = flag.Bool("use_many_errors", false, "Allow lexer collect semantic errors") - // tokenKind determines type of a token. type tokenKind byte @@ -49,12 +46,17 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. - fatalError error // Fatal error occured during lexing. It is usually a syntax error. - SemanticErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. + UseMultipleErrors bool // If we want to use multiple errors. + fatalError error // Fatal error occured during lexing. It is usually a syntax error. + nowSem bool // If semantic error occured during parsing. + semanticErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. } // fetchToken scans the input for the next token. func (r *Lexer) fetchToken() { + if r.nowSem { + return + } r.token.kind = tokenUndef r.start = r.pos @@ -413,15 +415,42 @@ func (r *Lexer) errSyntax() { r.errParse("syntax error") } -func (r *Lexer) errSemantic() { // TODO: add error data. +func (r *Lexer) ConsumeSemantic() { + r.nowSem = false +} + +func (r *Lexer) errSemantic(expected string) { + pos := r.pos + r.nowSem = true + r.SkipRecursive() + if len(expected) != 0 { + if expected[0] == '[' { + r.token.delimValue = '[' + } + if expected[0] == '{' { + r.token.delimValue = '{' + } + if expected[0] == ']' { + r.token.delimValue = ']' + return + } + if expected[0] == '}' { + r.token.delimValue = '}' + return + } + } r.AddSemanticError(&LexerError{ - Reason: "syntax error", - Offset: r.pos, - Data: "error occured", // TODO: fix this. + Reason: "invalid token", + Offset: pos, + Data: expected, }) } func (r *Lexer) errInvalidToken(expected string) { + if r.UseMultipleErrors { + r.errSemantic(expected) + return + } if r.fatalError == nil { var str string if len(r.token.byteValue) <= maxErrorContextLen { @@ -450,6 +479,9 @@ func (r *Lexer) Delim(c byte) { // IsDelim returns true if there was no scanning error and next token is the given delimiter. func (r *Lexer) IsDelim(c byte) bool { + if r.nowSem { + return true + } if r.token.kind == tokenUndef && r.Ok() { r.fetchToken() } @@ -489,7 +521,6 @@ func (r *Lexer) Skip() { // Note: no syntax validation is performed on the skipped data. func (r *Lexer) SkipRecursive() { r.scanToken() - var start, end byte if r.token.delimValue == '{' { @@ -598,7 +629,6 @@ func (r *Lexer) String() string { if !r.Ok() || r.token.kind != tokenString { r.errInvalidToken("string") return "" - } ret := string(r.token.byteValue) r.consume() @@ -633,11 +663,6 @@ func (r *Lexer) Bool() bool { r.fetchToken() } if !r.Ok() || r.token.kind != tokenBool { - if *UseSemanticErrors { // FIXME: remove copypaste from all methods. - r.errSemantic() - r.SkipRecursive() // FIXME: - return false - } r.errInvalidToken("bool") return false } @@ -651,9 +676,6 @@ func (r *Lexer) number() string { r.fetchToken() } if !r.Ok() || r.token.kind != tokenNumber { - if *UseSemanticErrors { - return "" - } r.errInvalidToken("number") return "" } @@ -664,19 +686,14 @@ func (r *Lexer) number() string { func (r *Lexer) Uint8() uint8 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 8) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint8(n) @@ -684,19 +701,14 @@ func (r *Lexer) Uint8() uint8 { func (r *Lexer) Uint16() uint16 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 16) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint16(n) @@ -704,19 +716,14 @@ func (r *Lexer) Uint16() uint16 { func (r *Lexer) Uint32() uint32 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint32(n) @@ -724,19 +731,14 @@ func (r *Lexer) Uint32() uint32 { func (r *Lexer) Uint64() uint64 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -748,19 +750,14 @@ func (r *Lexer) Uint() uint { func (r *Lexer) Int8() int8 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 8) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int8(n) @@ -768,19 +765,14 @@ func (r *Lexer) Int8() int8 { func (r *Lexer) Int16() int16 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 16) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int16(n) @@ -788,19 +780,14 @@ func (r *Lexer) Int16() int16 { func (r *Lexer) Int32() int32 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int32(n) @@ -808,19 +795,14 @@ func (r *Lexer) Int32() int32 { func (r *Lexer) Int64() int64 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -832,19 +814,14 @@ func (r *Lexer) Int() int { func (r *Lexer) Uint8Str() uint8 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 8) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint8(n) @@ -852,19 +829,14 @@ func (r *Lexer) Uint8Str() uint8 { func (r *Lexer) Uint16Str() uint16 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 16) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint16(n) @@ -872,19 +844,14 @@ func (r *Lexer) Uint16Str() uint16 { func (r *Lexer) Uint32Str() uint32 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return uint32(n) @@ -892,19 +859,14 @@ func (r *Lexer) Uint32Str() uint32 { func (r *Lexer) Uint64Str() uint64 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseUint(s, 10, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -916,19 +878,14 @@ func (r *Lexer) UintStr() uint { func (r *Lexer) Int8Str() int8 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 8) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int8(n) @@ -936,19 +893,14 @@ func (r *Lexer) Int8Str() int8 { func (r *Lexer) Int16Str() int16 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 16) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int16(n) @@ -956,19 +908,14 @@ func (r *Lexer) Int16Str() int16 { func (r *Lexer) Int32Str() int32 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return int32(n) @@ -976,19 +923,14 @@ func (r *Lexer) Int32Str() int32 { func (r *Lexer) Int64Str() int64 { s := r.UnsafeString() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseInt(s, 10, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -1000,19 +942,14 @@ func (r *Lexer) IntStr() int { func (r *Lexer) Float32() float32 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseFloat(s, 32) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return float32(n) @@ -1020,19 +957,14 @@ func (r *Lexer) Float32() float32 { func (r *Lexer) Float64() float64 { s := r.number() - if !r.Ok() { + if !r.Ok() || len(r.semanticErrors) != 0 { return 0 } n, err := strconv.ParseFloat(s, 64) if err != nil { - if *UseSemanticErrors { - r.errSemantic() - r.SkipRecursive() - } else { - r.fatalError = &LexerError{ - Reason: err.Error(), - } + r.fatalError = &LexerError{ + Reason: err.Error(), } } return n @@ -1049,7 +981,11 @@ func (r *Lexer) AddError(e error) { } func (r *Lexer) AddSemanticError(err *LexerError) { - r.SemanticErrors = append(r.SemanticErrors, err) + r.semanticErrors = append(r.semanticErrors, err) +} + +func (r *Lexer) GetSemanticErrors() []*LexerError { + return r.semanticErrors } // Interface fetches an interface{} analogous to the 'encoding/json' package. diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index 40dbcfd..e3add8c 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -82,7 +82,7 @@ func TestNumber(t *testing.T) { {toParse: "12.35E-15", want: "12.35E-15"}, {toParse: "12.35E15", want: "12.35E15"}, - // {toParse: `"a"`, wantError: true}, // FIXME(shmel1k): disable UseSemanticErrors for tests. + {toParse: `"a"`, wantError: true}, {toParse: "123junk", wantError: true}, {toParse: "1.2.3", wantError: true}, {toParse: "1e2e3", wantError: true}, @@ -95,9 +95,6 @@ func TestNumber(t *testing.T) { t.Errorf("[%d, %q] number() = %v; want %v", i, test.toParse, got, test.want) } err := l.Error() - if err == nil && len(l.SemanticErrors) != 0 { - err = l.SemanticErrors[0] - } if err != nil && !test.wantError { t.Errorf("[%d, %q] number() error: %v", i, test.toParse, err) } else if err == nil && test.wantError { @@ -128,9 +125,6 @@ func TestBool(t *testing.T) { t.Errorf("[%d, %q] Bool() = %v; want %v", i, test.toParse, got, test.want) } err := l.Error() - if err == nil && len(l.SemanticErrors) != 0 { - err = l.SemanticErrors[0] - } if err != nil && !test.wantError { t.Errorf("[%d, %q] Bool() error: %v", i, test.toParse, err) } else if err == nil && test.wantError { diff --git a/tests/errors.go b/tests/errors.go index a0180ad..2ec3299 100644 --- a/tests/errors.go +++ b/tests/errors.go @@ -8,3 +8,16 @@ type ErrorBoolSlice []bool //easyjson:json type ErrorUintSlice []uint + +//easyjson:json +type ErrorStruct struct { + Int int `json:"int"` + String string `json:"string"` + Slice []int `json:"slice"` + IntSlice []int `json:"int_slice"` +} + +type ErrorNestedStruct struct { + ErrorStruct ErrorStruct `json:"error_struct"` + Int int `json:"int"` +} diff --git a/tests/errors_test.go b/tests/errors_test.go index af27396..ee1ea72 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -7,9 +7,6 @@ import ( ) func TestSemanticErrorsInt(t *testing.T) { - if !*jlexer.UseSemanticErrors { - return - } for i, test := range []struct { Data []byte ErrorNum int @@ -19,7 +16,7 @@ func TestSemanticErrorsInt(t *testing.T) { ErrorNum: 2, }, { - Data: []byte(`[1, {"2" : "3"}, 3, "4"`), + Data: []byte(`[1, {"2":"3"}, 3, "4"]`), ErrorNum: 2, }, { @@ -30,23 +27,29 @@ func TestSemanticErrorsInt(t *testing.T) { Data: []byte(`[1, 2, 3, 4, "5"]`), ErrorNum: 1, }, + { + Data: []byte(`[{"1": "2"}]`), + ErrorNum: 1, + }, } { - l := jlexer.Lexer{Data: test.Data} + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } var v ErrorIntSlice v.UnmarshalEasyJSON(&l) - if len(l.SemanticErrors) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsInt(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + t.Errorf("%v", l.GetSemanticErrors()) + t.Errorf("%v", l.Error()) } } } func TestSemanticErrorsBool(t *testing.T) { - if !*jlexer.UseSemanticErrors { - return - } for i, test := range []struct { Data []byte ErrorNum int @@ -59,26 +62,26 @@ func TestSemanticErrorsBool(t *testing.T) { ErrorNum: 4, }, { - Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false`), + Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false]`), ErrorNum: 2, }, } { - l := jlexer.Lexer{Data: test.Data} + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } var v ErrorBoolSlice v.UnmarshalEasyJSON(&l) - if len(l.SemanticErrors) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsBool(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + t.Errorf("%v", l.Error()) } } } func TestSemanticErrorsUint(t *testing.T) { - if !*jlexer.UseSemanticErrors { - return - } - for i, test := range []struct { Data []byte ErrorNum int @@ -94,14 +97,109 @@ func TestSemanticErrorsUint(t *testing.T) { Data: []byte(`["zz", "zz"]`), ErrorNum: 2, }, + { + Data: []byte(`[{}, 42]`), + ErrorNum: 1, + }, } { - l := jlexer.Lexer{Data: test.Data} + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } var v ErrorUintSlice v.UnmarshalEasyJSON(&l) - if len(l.SemanticErrors) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrors(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.SemanticErrors)) + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsUint(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + } + } +} + +func TestSemanticErrorsStruct(t *testing.T) { + for i, test := range []struct { + Data []byte + ErrorNum int + }{ + { + Data: []byte(`{"string": "test", "slice":[42, 42, 42], "int_slice":[1, 2, 3]}`), + }, + { + Data: []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`), + ErrorNum: 2, + }, + { + Data: []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`), + ErrorNum: 3, + }, + { + Data: []byte(`{"string": "test", "slice": {}}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"slice":5, "string" : "test"}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"slice" : "test", "string" : "test"}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"slice": "", "string" : {}, "int":{}}`), + ErrorNum: 3, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + var v ErrorStruct + v.UnmarshalEasyJSON(&l) + + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + } + } +} + +func TestSemanticErrorsNestedStruct(t *testing.T) { + for i, test := range []struct { + Data []byte + ErrorNum int + }{ + { + Data: []byte(`{"error_struct":{}}`), + }, + { + Data: []byte(`{"error_struct":5}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"error_struct":[]}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"error_struct":{"int":{}}}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"error_struct":{"int_slice":{}}, "int":4}`), + ErrorNum: 1, + }, + { + Data: []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`), + ErrorNum: 3, + }, + } { + l := jlexer.Lexer{ + Data: test.Data, + UseMultipleErrors: true, + } + var v ErrorNestedStruct + v.UnmarshalEasyJSON(&l) + + if len(l.GetSemanticErrors()) != test.ErrorNum { + t.Errorf("[%d] TestSemanticErrorsNestedStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) } } }