From 0c9c34b8cca370ef8206a62d6a75f625afc006dc Mon Sep 17 00:00:00 2001 From: Aleksandr Petrukhin Date: Wed, 11 Jan 2017 03:18:07 +0300 Subject: [PATCH] Fixed tests, added comments --- gen/decoder.go | 2 - jlexer/lexer.go | 103 ++++++++++++-------------- tests/errors_test.go | 173 ++++++++++++++++++++++++++----------------- 3 files changed, 151 insertions(+), 127 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index 0f7c5d7..bafa162 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -118,7 +118,6 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field fmt.Fprintln(g.out, ws+" "+out+" = append("+out+", "+tmpVar+")") fmt.Fprintln(g.out, ws+" in.WantComma()") - fmt.Fprintln(g.out, ws+" in.ConsumeSemantic()") fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+" in.Delim(']')") fmt.Fprintln(g.out, ws+"}") @@ -410,7 +409,6 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { fmt.Fprintln(g.out, " default:") fmt.Fprintln(g.out, " in.SkipRecursive()") fmt.Fprintln(g.out, " }") - fmt.Fprintln(g.out, " in.ConsumeSemantic()") fmt.Fprintln(g.out, " in.WantComma()") fmt.Fprintln(g.out, " }") fmt.Fprintln(g.out, " in.Delim('}')") diff --git a/jlexer/lexer.go b/jlexer/lexer.go index 6647d5c..b23950d 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -49,14 +49,11 @@ type Lexer struct { UseMultipleErrors bool // If we want to use multiple errors. fatalError error // Fatal error occured during lexing. It is usually a syntax error. nowSem bool // If semantic error occured during parsing. - semanticErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. + multipleErrors []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors. } // fetchToken scans the input for the next token. func (r *Lexer) fetchToken() { - if r.nowSem { - return - } r.token.kind = tokenUndef r.start = r.pos @@ -415,34 +412,30 @@ func (r *Lexer) errSyntax() { r.errParse("syntax error") } -func (r *Lexer) ConsumeSemantic() { - r.nowSem = false -} - func (r *Lexer) errSemantic(expected string) { - pos := r.pos - r.nowSem = true + r.pos = r.start + r.consume() r.SkipRecursive() - if len(expected) != 0 { - if expected[0] == '[' { - r.token.delimValue = '[' - } - if expected[0] == '{' { - r.token.delimValue = '{' - } - if expected[0] == ']' { - r.token.delimValue = ']' - return - } - if expected[0] == '}' { - r.token.delimValue = '}' - return - } + switch expected { + case "[": + r.token.delimValue = ']' + r.token.kind = tokenDelim + case "{": + r.token.delimValue = '}' + r.token.kind = tokenDelim + case "]": + r.token.delimValue = ']' + r.token.kind = tokenDelim + return + case "}": + r.token.delimValue = '}' + r.token.kind = tokenDelim + return } - r.AddSemanticError(&LexerError{ - Reason: "invalid token", - Offset: pos, - Data: expected, + r.AddMultipleError(&LexerError{ + Reason: fmt.Sprintf("expected %s", expected), + Offset: r.start, + Data: string(r.Data[r.start:]), }) } @@ -471,17 +464,17 @@ func (r *Lexer) Delim(c byte) { if r.token.kind == tokenUndef && r.Ok() { r.fetchToken() } + if !r.Ok() || r.token.delimValue != c { + r.consume() // errInvalidToken can change token if UseMultipleErrors is enabled. r.errInvalidToken(string([]byte{c})) + } else { + r.consume() } - r.consume() } // IsDelim returns true if there was no scanning error and next token is the given delimiter. func (r *Lexer) IsDelim(c byte) bool { - if r.nowSem { - return true - } if r.token.kind == tokenUndef && r.Ok() { r.fetchToken() } @@ -686,7 +679,7 @@ func (r *Lexer) number() string { func (r *Lexer) Uint8() uint8 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -701,7 +694,7 @@ func (r *Lexer) Uint8() uint8 { func (r *Lexer) Uint16() uint16 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -716,7 +709,7 @@ func (r *Lexer) Uint16() uint16 { func (r *Lexer) Uint32() uint32 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -731,7 +724,7 @@ func (r *Lexer) Uint32() uint32 { func (r *Lexer) Uint64() uint64 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -750,7 +743,7 @@ func (r *Lexer) Uint() uint { func (r *Lexer) Int8() int8 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -765,7 +758,7 @@ func (r *Lexer) Int8() int8 { func (r *Lexer) Int16() int16 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -780,7 +773,7 @@ func (r *Lexer) Int16() int16 { func (r *Lexer) Int32() int32 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -795,7 +788,7 @@ func (r *Lexer) Int32() int32 { func (r *Lexer) Int64() int64 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -814,7 +807,7 @@ func (r *Lexer) Int() int { func (r *Lexer) Uint8Str() uint8 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -829,7 +822,7 @@ func (r *Lexer) Uint8Str() uint8 { func (r *Lexer) Uint16Str() uint16 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -844,7 +837,7 @@ func (r *Lexer) Uint16Str() uint16 { func (r *Lexer) Uint32Str() uint32 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -859,7 +852,7 @@ func (r *Lexer) Uint32Str() uint32 { func (r *Lexer) Uint64Str() uint64 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -878,7 +871,7 @@ func (r *Lexer) UintStr() uint { func (r *Lexer) Int8Str() int8 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -893,7 +886,7 @@ func (r *Lexer) Int8Str() int8 { func (r *Lexer) Int16Str() int16 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -908,7 +901,7 @@ func (r *Lexer) Int16Str() int16 { func (r *Lexer) Int32Str() int32 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -923,7 +916,7 @@ func (r *Lexer) Int32Str() int32 { func (r *Lexer) Int64Str() int64 { s := r.UnsafeString() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -942,7 +935,7 @@ func (r *Lexer) IntStr() int { func (r *Lexer) Float32() float32 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -957,7 +950,7 @@ func (r *Lexer) Float32() float32 { func (r *Lexer) Float64() float64 { s := r.number() - if !r.Ok() || len(r.semanticErrors) != 0 { + if !r.Ok() || len(r.multipleErrors) != 0 { return 0 } @@ -980,12 +973,12 @@ func (r *Lexer) AddError(e error) { } } -func (r *Lexer) AddSemanticError(err *LexerError) { - r.semanticErrors = append(r.semanticErrors, err) +func (r *Lexer) AddMultipleError(err *LexerError) { + r.multipleErrors = append(r.multipleErrors, err) } -func (r *Lexer) GetSemanticErrors() []*LexerError { - return r.semanticErrors +func (r *Lexer) GetMultipleErrors() []*LexerError { + return r.multipleErrors } // Interface fetches an interface{} analogous to the 'encoding/json' package. diff --git a/tests/errors_test.go b/tests/errors_test.go index ee1ea72..2575c22 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -6,30 +6,30 @@ import ( "github.com/mailru/easyjson/jlexer" ) -func TestSemanticErrorsInt(t *testing.T) { +func TestMultipleErrorsInt(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { - Data: []byte(`[1, 2, 3, "4", "5"]`), - ErrorNum: 2, + Data: []byte(`[1, 2, 3, "4", "5"]`), + Offsets: []int{10, 15}, }, { - Data: []byte(`[1, {"2":"3"}, 3, "4"]`), - ErrorNum: 2, + Data: []byte(`[1, {"2":"3"}, 3, "4"]`), + Offsets: []int{4, 18}, }, { - Data: []byte(`[1, "2", "3", "4", "5", "6"]`), - ErrorNum: 5, + Data: []byte(`[1, "2", "3", "4", "5", "6"]`), + Offsets: []int{4, 9, 14, 19, 24}, }, { - Data: []byte(`[1, 2, 3, 4, "5"]`), - ErrorNum: 1, + Data: []byte(`[1, 2, 3, 4, "5"]`), + Offsets: []int{13}, }, { - Data: []byte(`[{"1": "2"}]`), - ErrorNum: 1, + Data: []byte(`[{"1": "2"}]`), + Offsets: []int{1}, }, } { l := jlexer.Lexer{ @@ -41,29 +41,35 @@ func TestSemanticErrorsInt(t *testing.T) { v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsInt(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) - t.Errorf("%v", l.GetSemanticErrors()) - t.Errorf("%v", l.Error()) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsInt(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsInt(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } -func TestSemanticErrorsBool(t *testing.T) { +func TestMultipleErrorsBool(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { Data: []byte(`[true, false, true, false]`), }, { - Data: []byte(`["test", "value", "lol", "1"]`), - ErrorNum: 4, + Data: []byte(`["test", "value", "lol", "1"]`), + Offsets: []int{1, 9, 18, 25}, }, { - Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false]`), - ErrorNum: 2, + Data: []byte(`[true, 42, {"a":"b", "c":"d"}, false]`), + Offsets: []int{7, 11}, }, } { l := jlexer.Lexer{ @@ -74,32 +80,38 @@ func TestSemanticErrorsBool(t *testing.T) { var v ErrorBoolSlice v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsBool(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) - t.Errorf("%v", l.Error()) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsBool(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsBool(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } -func TestSemanticErrorsUint(t *testing.T) { +func TestMultipleErrorsUint(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { Data: []byte(`[42, 42, 42]`), }, { - Data: []byte(`[17, "42", 32]`), - ErrorNum: 1, + Data: []byte(`[17, "42", 32]`), + Offsets: []int{5}, }, { - Data: []byte(`["zz", "zz"]`), - ErrorNum: 2, + Data: []byte(`["zz", "zz"]`), + Offsets: []int{1, 7}, }, { - Data: []byte(`[{}, 42]`), - ErrorNum: 1, + Data: []byte(`[{}, 42]`), + Offsets: []int{1}, }, } { l := jlexer.Lexer{ @@ -110,43 +122,50 @@ func TestSemanticErrorsUint(t *testing.T) { var v ErrorUintSlice v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsUint(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsUint(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsUint(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } -func TestSemanticErrorsStruct(t *testing.T) { +func TestMultipleErrorsStruct(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { Data: []byte(`{"string": "test", "slice":[42, 42, 42], "int_slice":[1, 2, 3]}`), }, { - Data: []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`), - ErrorNum: 2, + Data: []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`), + Offsets: []int{11, 64}, }, { - Data: []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`), - ErrorNum: 3, + Data: []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`), + Offsets: []int{30, 61, 66}, }, { - Data: []byte(`{"string": "test", "slice": {}}`), - ErrorNum: 1, + Data: []byte(`{"string": "test", "slice": {}}`), + Offsets: []int{28}, }, { - Data: []byte(`{"slice":5, "string" : "test"}`), - ErrorNum: 1, + Data: []byte(`{"slice":5, "string" : "test"}`), + Offsets: []int{9}, }, { - Data: []byte(`{"slice" : "test", "string" : "test"}`), - ErrorNum: 1, + Data: []byte(`{"slice" : "test", "string" : "test"}`), + Offsets: []int{11}, }, { - Data: []byte(`{"slice": "", "string" : {}, "int":{}}`), - ErrorNum: 3, + Data: []byte(`{"slice": "", "string" : {}, "int":{}}`), + Offsets: []int{10, 25, 35}, }, } { l := jlexer.Lexer{ @@ -156,39 +175,46 @@ func TestSemanticErrorsStruct(t *testing.T) { var v ErrorStruct v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } } -func TestSemanticErrorsNestedStruct(t *testing.T) { +func TestMultipleErrorsNestedStruct(t *testing.T) { for i, test := range []struct { - Data []byte - ErrorNum int + Data []byte + Offsets []int }{ { Data: []byte(`{"error_struct":{}}`), }, { - Data: []byte(`{"error_struct":5}`), - ErrorNum: 1, + Data: []byte(`{"error_struct":5}`), + Offsets: []int{16}, }, { - Data: []byte(`{"error_struct":[]}`), - ErrorNum: 1, + Data: []byte(`{"error_struct":[]}`), + Offsets: []int{16}, }, { - Data: []byte(`{"error_struct":{"int":{}}}`), - ErrorNum: 1, + Data: []byte(`{"error_struct":{"int":{}}}`), + Offsets: []int{23}, }, { - Data: []byte(`{"error_struct":{"int_slice":{}}, "int":4}`), - ErrorNum: 1, + Data: []byte(`{"error_struct":{"int_slice":{}}, "int":4}`), + Offsets: []int{29}, }, { - Data: []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`), - ErrorNum: 3, + Data: []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`), + Offsets: []int{30, 38, 51}, }, } { l := jlexer.Lexer{ @@ -198,8 +224,15 @@ func TestSemanticErrorsNestedStruct(t *testing.T) { var v ErrorNestedStruct v.UnmarshalEasyJSON(&l) - if len(l.GetSemanticErrors()) != test.ErrorNum { - t.Errorf("[%d] TestSemanticErrorsNestedStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors())) + errors := l.GetMultipleErrors() + + if len(errors) != len(test.Offsets) { + t.Errorf("[%d] TestMultipleErrorsNestedStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors)) + } + for ii, e := range errors { + if e.Offset != test.Offsets[ii] { + t.Errorf("[%d] TestMultipleErrorsNestedStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset) + } } } }