Fixed tests, added comments

gen/decoder.go

@@ -118,7 +118,6 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field
 
 			fmt.Fprintln(g.out, ws+"    "+out+" = append("+out+", "+tmpVar+")")
 			fmt.Fprintln(g.out, ws+"    in.WantComma()")
-			fmt.Fprintln(g.out, ws+"    in.ConsumeSemantic()")
 			fmt.Fprintln(g.out, ws+"  }")
 			fmt.Fprintln(g.out, ws+"  in.Delim(']')")
 			fmt.Fprintln(g.out, ws+"}")
@@ -410,7 +409,6 @@ func (g *Generator) genStructDecoder(t reflect.Type) error {
 	fmt.Fprintln(g.out, "    default:")
 	fmt.Fprintln(g.out, "      in.SkipRecursive()")
 	fmt.Fprintln(g.out, "    }")
-	fmt.Fprintln(g.out, "    in.ConsumeSemantic()")
 	fmt.Fprintln(g.out, "    in.WantComma()")
 	fmt.Fprintln(g.out, "  }")
 	fmt.Fprintln(g.out, "  in.Delim('}')")

jlexer/lexer.go

@@ -49,14 +49,11 @@ type Lexer struct {
 	UseMultipleErrors bool          // If we want to use multiple errors.
 	fatalError        error         // Fatal error occured during lexing. It is usually a syntax error.
 	nowSem            bool          // If semantic error occured during parsing.
-	semanticErrors    []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors.
+	multipleErrors    []*LexerError // Semantic errors occured during lexing. Marshalling will be continued after finding this errors.
 }
 
 // fetchToken scans the input for the next token.
 func (r *Lexer) fetchToken() {
-	if r.nowSem {
-		return
-	}
 	r.token.kind = tokenUndef
 	r.start = r.pos
 
@@ -415,34 +412,30 @@ func (r *Lexer) errSyntax() {
 	r.errParse("syntax error")
 }
 
-func (r *Lexer) ConsumeSemantic() {
-	r.nowSem = false
-}
-
 func (r *Lexer) errSemantic(expected string) {
-	pos := r.pos
-	r.nowSem = true
+	r.pos = r.start
+	r.consume()
 	r.SkipRecursive()
-	if len(expected) != 0 {
-		if expected[0] == '[' {
-			r.token.delimValue = '['
-		}
-		if expected[0] == '{' {
-			r.token.delimValue = '{'
-		}
-		if expected[0] == ']' {
-			r.token.delimValue = ']'
-			return
-		}
-		if expected[0] == '}' {
-			r.token.delimValue = '}'
-			return
-		}
+	switch expected {
+	case "[":
+		r.token.delimValue = ']'
+		r.token.kind = tokenDelim
+	case "{":
+		r.token.delimValue = '}'
+		r.token.kind = tokenDelim
+	case "]":
+		r.token.delimValue = ']'
+		r.token.kind = tokenDelim
+		return
+	case "}":
+		r.token.delimValue = '}'
+		r.token.kind = tokenDelim
+		return
 	}
-	r.AddSemanticError(&LexerError{
-		Reason: "invalid token",
-		Offset: pos,
-		Data:   expected,
+	r.AddMultipleError(&LexerError{
+		Reason: fmt.Sprintf("expected %s", expected),
+		Offset: r.start,
+		Data:   string(r.Data[r.start:]),
 	})
 }
 
@@ -471,17 +464,17 @@ func (r *Lexer) Delim(c byte) {
 	if r.token.kind == tokenUndef && r.Ok() {
 		r.fetchToken()
 	}
+
 	if !r.Ok() || r.token.delimValue != c {
+		r.consume() // errInvalidToken can change token if UseMultipleErrors is enabled.
 		r.errInvalidToken(string([]byte{c}))
+	} else {
+		r.consume()
 	}
-	r.consume()
 }
 
 // IsDelim returns true if there was no scanning error and next token is the given delimiter.
 func (r *Lexer) IsDelim(c byte) bool {
-	if r.nowSem {
-		return true
-	}
 	if r.token.kind == tokenUndef && r.Ok() {
 		r.fetchToken()
 	}
@@ -686,7 +679,7 @@ func (r *Lexer) number() string {
 
 func (r *Lexer) Uint8() uint8 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -701,7 +694,7 @@ func (r *Lexer) Uint8() uint8 {
 
 func (r *Lexer) Uint16() uint16 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -716,7 +709,7 @@ func (r *Lexer) Uint16() uint16 {
 
 func (r *Lexer) Uint32() uint32 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -731,7 +724,7 @@ func (r *Lexer) Uint32() uint32 {
 
 func (r *Lexer) Uint64() uint64 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -750,7 +743,7 @@ func (r *Lexer) Uint() uint {
 
 func (r *Lexer) Int8() int8 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -765,7 +758,7 @@ func (r *Lexer) Int8() int8 {
 
 func (r *Lexer) Int16() int16 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -780,7 +773,7 @@ func (r *Lexer) Int16() int16 {
 
 func (r *Lexer) Int32() int32 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -795,7 +788,7 @@ func (r *Lexer) Int32() int32 {
 
 func (r *Lexer) Int64() int64 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -814,7 +807,7 @@ func (r *Lexer) Int() int {
 
 func (r *Lexer) Uint8Str() uint8 {
 	s := r.UnsafeString()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -829,7 +822,7 @@ func (r *Lexer) Uint8Str() uint8 {
 
 func (r *Lexer) Uint16Str() uint16 {
 	s := r.UnsafeString()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -844,7 +837,7 @@ func (r *Lexer) Uint16Str() uint16 {
 
 func (r *Lexer) Uint32Str() uint32 {
 	s := r.UnsafeString()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -859,7 +852,7 @@ func (r *Lexer) Uint32Str() uint32 {
 
 func (r *Lexer) Uint64Str() uint64 {
 	s := r.UnsafeString()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -878,7 +871,7 @@ func (r *Lexer) UintStr() uint {
 
 func (r *Lexer) Int8Str() int8 {
 	s := r.UnsafeString()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -893,7 +886,7 @@ func (r *Lexer) Int8Str() int8 {
 
 func (r *Lexer) Int16Str() int16 {
 	s := r.UnsafeString()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -908,7 +901,7 @@ func (r *Lexer) Int16Str() int16 {
 
 func (r *Lexer) Int32Str() int32 {
 	s := r.UnsafeString()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -923,7 +916,7 @@ func (r *Lexer) Int32Str() int32 {
 
 func (r *Lexer) Int64Str() int64 {
 	s := r.UnsafeString()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -942,7 +935,7 @@ func (r *Lexer) IntStr() int {
 
 func (r *Lexer) Float32() float32 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -957,7 +950,7 @@ func (r *Lexer) Float32() float32 {
 
 func (r *Lexer) Float64() float64 {
 	s := r.number()
-	if !r.Ok() || len(r.semanticErrors) != 0 {
+	if !r.Ok() || len(r.multipleErrors) != 0 {
 		return 0
 	}
 
@@ -980,12 +973,12 @@ func (r *Lexer) AddError(e error) {
 	}
 }
 
-func (r *Lexer) AddSemanticError(err *LexerError) {
-	r.semanticErrors = append(r.semanticErrors, err)
+func (r *Lexer) AddMultipleError(err *LexerError) {
+	r.multipleErrors = append(r.multipleErrors, err)
 }
 
-func (r *Lexer) GetSemanticErrors() []*LexerError {
-	return r.semanticErrors
+func (r *Lexer) GetMultipleErrors() []*LexerError {
+	return r.multipleErrors
 }
 
 // Interface fetches an interface{} analogous to the 'encoding/json' package.

tests/errors_test.go

@@ -6,30 +6,30 @@ import (
 	"github.com/mailru/easyjson/jlexer"
 )
 
-func TestSemanticErrorsInt(t *testing.T) {
+func TestMultipleErrorsInt(t *testing.T) {
 	for i, test := range []struct {
-		Data     []byte
-		ErrorNum int
+		Data    []byte
+		Offsets []int
 	}{
 		{
-			Data:     []byte(`[1, 2, 3, "4", "5"]`),
-			ErrorNum: 2,
+			Data:    []byte(`[1, 2, 3, "4", "5"]`),
+			Offsets: []int{10, 15},
 		},
 		{
-			Data:     []byte(`[1, {"2":"3"}, 3, "4"]`),
-			ErrorNum: 2,
+			Data:    []byte(`[1, {"2":"3"}, 3, "4"]`),
+			Offsets: []int{4, 18},
 		},
 		{
-			Data:     []byte(`[1, "2", "3", "4", "5", "6"]`),
-			ErrorNum: 5,
+			Data:    []byte(`[1, "2", "3", "4", "5", "6"]`),
+			Offsets: []int{4, 9, 14, 19, 24},
 		},
 		{
-			Data:     []byte(`[1, 2, 3, 4, "5"]`),
-			ErrorNum: 1,
+			Data:    []byte(`[1, 2, 3, 4, "5"]`),
+			Offsets: []int{13},
 		},
 		{
-			Data:     []byte(`[{"1": "2"}]`),
-			ErrorNum: 1,
+			Data:    []byte(`[{"1": "2"}]`),
+			Offsets: []int{1},
 		},
 	} {
 		l := jlexer.Lexer{
@@ -41,29 +41,35 @@ func TestSemanticErrorsInt(t *testing.T) {
 
 		v.UnmarshalEasyJSON(&l)
 
-		if len(l.GetSemanticErrors()) != test.ErrorNum {
-			t.Errorf("[%d] TestSemanticErrorsInt(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors()))
-			t.Errorf("%v", l.GetSemanticErrors())
-			t.Errorf("%v", l.Error())
+		errors := l.GetMultipleErrors()
+
+		if len(errors) != len(test.Offsets) {
+			t.Errorf("[%d] TestMultipleErrorsInt(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
+		}
+
+		for ii, e := range errors {
+			if e.Offset != test.Offsets[ii] {
+				t.Errorf("[%d] TestMultipleErrorsInt(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
+			}
 		}
 	}
 }
 
-func TestSemanticErrorsBool(t *testing.T) {
+func TestMultipleErrorsBool(t *testing.T) {
 	for i, test := range []struct {
-		Data     []byte
-		ErrorNum int
+		Data    []byte
+		Offsets []int
 	}{
 		{
 			Data: []byte(`[true, false, true, false]`),
 		},
 		{
-			Data:     []byte(`["test", "value", "lol", "1"]`),
-			ErrorNum: 4,
+			Data:    []byte(`["test", "value", "lol", "1"]`),
+			Offsets: []int{1, 9, 18, 25},
 		},
 		{
-			Data:     []byte(`[true, 42, {"a":"b", "c":"d"}, false]`),
-			ErrorNum: 2,
+			Data:    []byte(`[true, 42, {"a":"b", "c":"d"}, false]`),
+			Offsets: []int{7, 11},
 		},
 	} {
 		l := jlexer.Lexer{
@@ -74,32 +80,38 @@ func TestSemanticErrorsBool(t *testing.T) {
 		var v ErrorBoolSlice
 		v.UnmarshalEasyJSON(&l)
 
-		if len(l.GetSemanticErrors()) != test.ErrorNum {
-			t.Errorf("[%d] TestSemanticErrorsBool(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors()))
-			t.Errorf("%v", l.Error())
+		errors := l.GetMultipleErrors()
+
+		if len(errors) != len(test.Offsets) {
+			t.Errorf("[%d] TestMultipleErrorsBool(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
+		}
+		for ii, e := range errors {
+			if e.Offset != test.Offsets[ii] {
+				t.Errorf("[%d] TestMultipleErrorsBool(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
+			}
 		}
 	}
 }
 
-func TestSemanticErrorsUint(t *testing.T) {
+func TestMultipleErrorsUint(t *testing.T) {
 	for i, test := range []struct {
-		Data     []byte
-		ErrorNum int
+		Data    []byte
+		Offsets []int
 	}{
 		{
 			Data: []byte(`[42, 42, 42]`),
 		},
 		{
-			Data:     []byte(`[17, "42", 32]`),
-			ErrorNum: 1,
+			Data:    []byte(`[17, "42", 32]`),
+			Offsets: []int{5},
 		},
 		{
-			Data:     []byte(`["zz", "zz"]`),
-			ErrorNum: 2,
+			Data:    []byte(`["zz", "zz"]`),
+			Offsets: []int{1, 7},
 		},
 		{
-			Data:     []byte(`[{}, 42]`),
-			ErrorNum: 1,
+			Data:    []byte(`[{}, 42]`),
+			Offsets: []int{1},
 		},
 	} {
 		l := jlexer.Lexer{
@@ -110,43 +122,50 @@ func TestSemanticErrorsUint(t *testing.T) {
 		var v ErrorUintSlice
 		v.UnmarshalEasyJSON(&l)
 
-		if len(l.GetSemanticErrors()) != test.ErrorNum {
-			t.Errorf("[%d] TestSemanticErrorsUint(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors()))
+		errors := l.GetMultipleErrors()
+
+		if len(errors) != len(test.Offsets) {
+			t.Errorf("[%d] TestMultipleErrorsUint(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
+		}
+		for ii, e := range errors {
+			if e.Offset != test.Offsets[ii] {
+				t.Errorf("[%d] TestMultipleErrorsUint(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
+			}
 		}
 	}
 }
 
-func TestSemanticErrorsStruct(t *testing.T) {
+func TestMultipleErrorsStruct(t *testing.T) {
 	for i, test := range []struct {
-		Data     []byte
-		ErrorNum int
+		Data    []byte
+		Offsets []int
 	}{
 		{
 			Data: []byte(`{"string": "test", "slice":[42, 42, 42], "int_slice":[1, 2, 3]}`),
 		},
 		{
-			Data:     []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`),
-			ErrorNum: 2,
+			Data:    []byte(`{"string": {"test": "test"}, "slice":[42, 42, 42], "int_slice":["1", 2, 3]}`),
+			Offsets: []int{11, 64},
 		},
 		{
-			Data:     []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`),
-			ErrorNum: 3,
+			Data:    []byte(`{"slice": [42, 42], "string": {"test": "test"}, "int_slice":["1", "2", 3]}`),
+			Offsets: []int{30, 61, 66},
 		},
 		{
-			Data:     []byte(`{"string": "test", "slice": {}}`),
-			ErrorNum: 1,
+			Data:    []byte(`{"string": "test", "slice": {}}`),
+			Offsets: []int{28},
 		},
 		{
-			Data:     []byte(`{"slice":5, "string" : "test"}`),
-			ErrorNum: 1,
+			Data:    []byte(`{"slice":5, "string" : "test"}`),
+			Offsets: []int{9},
 		},
 		{
-			Data:     []byte(`{"slice" : "test", "string" : "test"}`),
-			ErrorNum: 1,
+			Data:    []byte(`{"slice" : "test", "string" : "test"}`),
+			Offsets: []int{11},
 		},
 		{
-			Data:     []byte(`{"slice": "", "string" : {}, "int":{}}`),
-			ErrorNum: 3,
+			Data:    []byte(`{"slice": "", "string" : {}, "int":{}}`),
+			Offsets: []int{10, 25, 35},
 		},
 	} {
 		l := jlexer.Lexer{
@@ -156,39 +175,46 @@ func TestSemanticErrorsStruct(t *testing.T) {
 		var v ErrorStruct
 		v.UnmarshalEasyJSON(&l)
 
-		if len(l.GetSemanticErrors()) != test.ErrorNum {
-			t.Errorf("[%d] TestSemanticErrorsStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors()))
+		errors := l.GetMultipleErrors()
+
+		if len(errors) != len(test.Offsets) {
+			t.Errorf("[%d] TestMultipleErrorsStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
+		}
+		for ii, e := range errors {
+			if e.Offset != test.Offsets[ii] {
+				t.Errorf("[%d] TestMultipleErrorsStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
+			}
 		}
 	}
 }
 
-func TestSemanticErrorsNestedStruct(t *testing.T) {
+func TestMultipleErrorsNestedStruct(t *testing.T) {
 	for i, test := range []struct {
-		Data     []byte
-		ErrorNum int
+		Data    []byte
+		Offsets []int
 	}{
 		{
 			Data: []byte(`{"error_struct":{}}`),
 		},
 		{
-			Data:     []byte(`{"error_struct":5}`),
-			ErrorNum: 1,
+			Data:    []byte(`{"error_struct":5}`),
+			Offsets: []int{16},
 		},
 		{
-			Data:     []byte(`{"error_struct":[]}`),
-			ErrorNum: 1,
+			Data:    []byte(`{"error_struct":[]}`),
+			Offsets: []int{16},
 		},
 		{
-			Data:     []byte(`{"error_struct":{"int":{}}}`),
-			ErrorNum: 1,
+			Data:    []byte(`{"error_struct":{"int":{}}}`),
+			Offsets: []int{23},
 		},
 		{
-			Data:     []byte(`{"error_struct":{"int_slice":{}}, "int":4}`),
-			ErrorNum: 1,
+			Data:    []byte(`{"error_struct":{"int_slice":{}}, "int":4}`),
+			Offsets: []int{29},
 		},
 		{
-			Data:     []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`),
-			ErrorNum: 3,
+			Data:    []byte(`{"error_struct":{"int_slice":["1", 2, "3"]}, "int":[]}`),
+			Offsets: []int{30, 38, 51},
 		},
 	} {
 		l := jlexer.Lexer{
@@ -198,8 +224,15 @@ func TestSemanticErrorsNestedStruct(t *testing.T) {
 		var v ErrorNestedStruct
 		v.UnmarshalEasyJSON(&l)
 
-		if len(l.GetSemanticErrors()) != test.ErrorNum {
-			t.Errorf("[%d] TestSemanticErrorsNestedStruct(): errornum: want: %d, got %d", i, test.ErrorNum, len(l.GetSemanticErrors()))
+		errors := l.GetMultipleErrors()
+
+		if len(errors) != len(test.Offsets) {
+			t.Errorf("[%d] TestMultipleErrorsNestedStruct(): errornum: want: %d, got %d", i, len(test.Offsets), len(errors))
+		}
+		for ii, e := range errors {
+			if e.Offset != test.Offsets[ii] {
+				t.Errorf("[%d] TestMultipleErrorsNestedStruct(): offset[%d]: want %d, got %d", i, ii, test.Offsets[ii], e.Offset)
+			}
 		}
 	}
 }