From f0a000e7a8e0b90a4dcf15bbd7c8a6c9c0e95f51 Mon Sep 17 00:00:00 2001 From: kirillx Date: Fri, 24 Apr 2020 20:26:02 +0300 Subject: [PATCH] Previous optimisation in findStringLen has broken unescaping of \\\" sequences (#284) * tests: don't ignore errors, verify them carefully * fix unescaping of \\\\\" and such sequences * tests: add Unmarshal test cases for escaped sequences --- Makefile | 2 + jlexer/lexer.go | 16 ++++-- tests/basic_test.go | 1 + tests/escaping.go | 11 +++++ tests/escaping_test.go | 90 ++++++++++++++++++++++++++++++++++ tests/intern_test.go | 10 +++- tests/members_escaping_test.go | 10 +++- tests/nocopy_test.go | 15 ++++-- 8 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 tests/escaping.go create mode 100644 tests/escaping_test.go diff --git a/Makefile b/Makefile index 92f3c43..85de543 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ generate: build ./tests/members_unescaped.go \ ./tests/intern.go \ ./tests/nocopy.go \ + ./tests/escaping.go \ bin/easyjson -all ./tests/data.go bin/easyjson -all ./tests/nothing.go @@ -46,6 +47,7 @@ generate: build bin/easyjson -disable_members_unescape ./tests/members_unescaped.go bin/easyjson ./tests/intern.go bin/easyjson ./tests/nocopy.go + bin/easyjson ./tests/escaping.go test: generate go test \ diff --git a/jlexer/lexer.go b/jlexer/lexer.go index db81db9..a42e9d6 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -253,6 +253,16 @@ func findStringLen(data []byte) (isValid bool, length int) { if idx == 0 || (idx > 0 && data[idx-1] != '\\') { return true, length + idx } + + // count \\\\\\\ sequences. even number of slashes means quote is not really escaped + cnt := 1 + for idx-cnt-1 >= 0 && data[idx-cnt-1] == '\\' { + cnt++ + } + if cnt%2 == 0 { + return true, length + idx + } + length += idx + 1 data = data[idx+1:] } @@ -325,7 +335,7 @@ func getu4(s []byte) rune { // decodeEscape processes a single escape sequence and returns number of bytes processed. func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) { if len(data) < 2 { - return 0, 0, fmt.Errorf("syntax error at %v", string(data)) + return 0, 0, errors.New("incorrect escape symbol \\ at the end of token") } c := data[1] @@ -345,7 +355,7 @@ func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) { case 'u': rr := getu4(data) if rr < 0 { - return 0, 0, errors.New("syntax error") + return 0, 0, errors.New("incorrectly escaped \\uXXXX sequence") } read := 6 @@ -361,7 +371,7 @@ func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) { return rr, read, nil } - return 0, 0, errors.New("syntax error") + return 0, 0, errors.New("incorrectly escaped bytes") } // fetchString scans a string literal token. diff --git a/tests/basic_test.go b/tests/basic_test.go index a2cd9ca..fe635ff 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -144,6 +144,7 @@ var testSpecialCases = []struct { {`"绿\ufffd茶"`, "绿\xc5茶"}, {`"тест\u2028"`, "тест\xE2\x80\xA8"}, {`"\\\r\n\t\""`, "\\\r\n\t\""}, + {`"text\\\""`, "text\\\""}, {`"ü"`, "ü"}, } diff --git a/tests/escaping.go b/tests/escaping.go new file mode 100644 index 0000000..da427af --- /dev/null +++ b/tests/escaping.go @@ -0,0 +1,11 @@ +package tests + +//easyjson:json +type EscStringStruct struct { + A string `json:"a"` +} + +//easyjson:json +type EscIntStruct struct { + A int `json:"a,string"` +} diff --git a/tests/escaping_test.go b/tests/escaping_test.go new file mode 100644 index 0000000..0491762 --- /dev/null +++ b/tests/escaping_test.go @@ -0,0 +1,90 @@ +package tests + +import ( + "reflect" + "testing" + + "github.com/mailru/easyjson" +) + +func TestStrFieldsUnescaping(t *testing.T) { + cases := []struct { + data string + exp EscStringStruct + }{ + { + data: `{}`, + exp: EscStringStruct{}, + }, + { + data: `{"a": "\""}`, + exp: EscStringStruct{A: `"`}, + }, + { + data: `{"a": "\\"}`, + exp: EscStringStruct{A: `\`}, + }, + { + data: `{"a": "\\\""}`, + exp: EscStringStruct{A: `\"`}, + }, + { + data: `{"a": "\\\\'"}`, + exp: EscStringStruct{A: `\\'`}, + }, + { + data: `{"a": "\t\\\nx\\\""}`, + exp: EscStringStruct{A: "\t\\\nx\\\""}, + }, + { + data: `{"a": "\r\n"}`, + exp: EscStringStruct{A: "\r\n"}, + }, + { + data: `{"a": "\r\n\u4e2D\u56fD\\\""}`, + exp: EscStringStruct{A: "\r\n中国\\\""}, + }, + } + + for i, c := range cases { + var val EscStringStruct + err := easyjson.Unmarshal([]byte(c.data), &val) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(val, c.exp) { + t.Errorf("[%d] TestStrFieldsUnescaping(): got=%q, exp=%q", i, val, c.exp) + } + } +} + +func TestIntFieldsUnescaping(t *testing.T) { + cases := []struct { + data string + exp EscIntStruct + }{ + { + data: `{}`, + exp: EscIntStruct{A: 0}, + }, + { + data: `{"a": "1"}`, + exp: EscIntStruct{A: 1}, + }, + { + data: `{"a": "\u0032"}`, + exp: EscIntStruct{A: 2}, + }, + } + + for i, c := range cases { + var val EscIntStruct + err := easyjson.Unmarshal([]byte(c.data), &val) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(val, c.exp) { + t.Errorf("[%d] TestIntFieldsUnescaping(): got=%v, exp=%v", i, val, c.exp) + } + } +} diff --git a/tests/intern_test.go b/tests/intern_test.go index f61b400..480f87c 100644 --- a/tests/intern_test.go +++ b/tests/intern_test.go @@ -12,7 +12,10 @@ func TestStringIntern(t *testing.T) { var i Intern allocsPerRun := testing.AllocsPerRun(1000, func() { i = Intern{} - easyjson.Unmarshal(data, &i) + err := easyjson.Unmarshal(data, &i) + if err != nil { + t.Error(err) + } if i.Field != "string interning test" { t.Fatalf("wrong value: %q", i.Field) } @@ -24,7 +27,10 @@ func TestStringIntern(t *testing.T) { var n NoIntern allocsPerRun = testing.AllocsPerRun(1000, func() { n = NoIntern{} - easyjson.Unmarshal(data, &n) + err := easyjson.Unmarshal(data, &n) + if err != nil { + t.Error(err) + } if n.Field != "string interning test" { t.Fatalf("wrong value: %q", n.Field) } diff --git a/tests/members_escaping_test.go b/tests/members_escaping_test.go index e385648..1995025 100644 --- a/tests/members_escaping_test.go +++ b/tests/members_escaping_test.go @@ -37,13 +37,19 @@ func TestMembersEscaping(t *testing.T) { for i, c := range cases { var esc MembersEscaped - easyjson.Unmarshal([]byte(c.data), &esc) + err := easyjson.Unmarshal([]byte(c.data), &esc) + if err != nil { + t.Error(err) + } if !reflect.DeepEqual(esc, c.esc) { t.Errorf("[%d] TestMembersEscaping(): got=%+v, exp=%+v", i, esc, c.esc) } var unesc MembersUnescaped - easyjson.Unmarshal([]byte(c.data), &unesc) + err = easyjson.Unmarshal([]byte(c.data), &unesc) + if err != nil { + t.Error(err) + } if !reflect.DeepEqual(unesc, c.unesc) { t.Errorf("[%d] TestMembersEscaping(): no-unescaping case: got=%+v, exp=%+v", i, esc, c.esc) } diff --git a/tests/nocopy_test.go b/tests/nocopy_test.go index 45592a2..05dccee 100644 --- a/tests/nocopy_test.go +++ b/tests/nocopy_test.go @@ -27,7 +27,10 @@ func TestNocopy(t *testing.T) { } res := NocopyStruct{} - easyjson.Unmarshal(data, &res) + err := easyjson.Unmarshal(data, &res) + if err != nil { + t.Error(err) + } if !reflect.DeepEqual(exp, res) { t.Errorf("TestNocopy(): got=%+v, exp=%+v", res, exp) } @@ -42,7 +45,10 @@ func TestNocopy(t *testing.T) { data = []byte(`{"b": "valueNoCopy"}`) res = NocopyStruct{} allocsPerRun := testing.AllocsPerRun(1000, func() { - easyjson.Unmarshal(data, &res) + err := easyjson.Unmarshal(data, &res) + if err != nil { + t.Error(err) + } if res.B != "valueNoCopy" { t.Fatalf("wrong value: %q", res.B) } @@ -53,7 +59,10 @@ func TestNocopy(t *testing.T) { data = []byte(`{"a": "valueNoCopy"}`) allocsPerRun = testing.AllocsPerRun(1000, func() { - easyjson.Unmarshal(data, &res) + err := easyjson.Unmarshal(data, &res) + if err != nil { + t.Error(err) + } if res.A != "valueNoCopy" { t.Fatalf("wrong value: %q", res.A) }