netbird/easyjson
0
0
Fork 0
mirror of https://github.com/netbirdio/easyjson.git synced 2026-05-22 18:44:42 -07:00
Code Issues Packages Projects Releases Wiki Activity

Previous optimisation in findStringLen has broken unescaping of \\\" sequences (#284)

Browse Source 
* tests: don't ignore errors, verify them carefully

* fix unescaping of \\\\\" and such sequences

* tests: add Unmarshal test cases for escaped sequences
This commit is contained in:
kirillx
2020-04-24 20:26:02 +03:00
committed by GitHub
parent 3b0c433891
commit f0a000e7a8
8 changed files with 145 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2
View File
@@ -25,6 +25,7 @@ generate: build
./tests/members_unescaped.go \
./tests/intern.go \
./tests/nocopy.go \
./tests/escaping.go \
bin/easyjson -all ./tests/data.go
bin/easyjson -all ./tests/nothing.go
@@ -46,6 +47,7 @@ generate: build
bin/easyjson -disable_members_unescape ./tests/members_unescaped.go
bin/easyjson ./tests/intern.go
bin/easyjson ./tests/nocopy.go
bin/easyjson ./tests/escaping.go
test: generate
go test \
jlexer/lexer.go
+13 -3
View File
@@ -253,6 +253,16 @@ func findStringLen(data []byte) (isValid bool, length int) {
if idx == 0 || (idx > 0 && data[idx-1] != '\\') {
return true, length + idx
}
// count \\\\\\\ sequences. even number of slashes means quote is not really escaped
cnt := 1
for idx-cnt-1 >= 0 && data[idx-cnt-1] == '\\' {
cnt++
}
if cnt%2 == 0 {
return true, length + idx
}
length += idx + 1
data = data[idx+1:]
}
@@ -325,7 +335,7 @@ func getu4(s []byte) rune {
// decodeEscape processes a single escape sequence and returns number of bytes processed.
func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
if len(data) < 2 {
return 0, 0, fmt.Errorf("syntax error at %v", string(data))
return 0, 0, errors.New("incorrect escape symbol \\ at the end of token")
}
c := data[1]
@@ -345,7 +355,7 @@ func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
case 'u':
rr := getu4(data)
if rr < 0 {
return 0, 0, errors.New("syntax error")
return 0, 0, errors.New("incorrectly escaped \\uXXXX sequence")
}
read := 6
@@ -361,7 +371,7 @@ func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
return rr, read, nil
}
return 0, 0, errors.New("syntax error")
return 0, 0, errors.New("incorrectly escaped bytes")
}
// fetchString scans a string literal token.
tests/basic_test.go
+1
View File
@@ -144,6 +144,7 @@ var testSpecialCases = []struct {
{`"绿\ufffd茶"`, "绿\xc5茶"},
{`"тест\u2028"`, "тест\xE2\x80\xA8"},
{`"\\\r\n\t\""`, "\\\r\n\t\""},
{`"text\\\""`, "text\\\""},
{`"ü"`, "ü"},
}
tests/escaping.go
+11
View File
@@ -0,0 +1,11 @@
package tests
//easyjson:json
type EscStringStruct struct {
A string `json:"a"`
}
//easyjson:json
type EscIntStruct struct {
A int `json:"a,string"`
}
tests/escaping_test.go
+90
View File
@@ -0,0 +1,90 @@
package tests
import (
"reflect"
"testing"
"github.com/mailru/easyjson"
)
func TestStrFieldsUnescaping(t *testing.T) {
cases := []struct {
data string
exp EscStringStruct
}{
{
data: `{}`,
exp: EscStringStruct{},
},
{
data: `{"a": "\""}`,
exp: EscStringStruct{A: `"`},
},
{
data: `{"a": "\\"}`,
exp: EscStringStruct{A: `\`},
},
{
data: `{"a": "\\\""}`,
exp: EscStringStruct{A: `\"`},
},
{
data: `{"a": "\\\\'"}`,
exp: EscStringStruct{A: `\\'`},
},
{
data: `{"a": "\t\\\nx\\\""}`,
exp: EscStringStruct{A: "\t\\\nx\\\""},
},
{
data: `{"a": "\r\n"}`,
exp: EscStringStruct{A: "\r\n"},
},
{
data: `{"a": "\r\n\u4e2D\u56fD\\\""}`,
exp: EscStringStruct{A: "\r\n中国\\\""},
},
}
for i, c := range cases {
var val EscStringStruct
err := easyjson.Unmarshal([]byte(c.data), &val)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, c.exp) {
t.Errorf("[%d] TestStrFieldsUnescaping(): got=%q, exp=%q", i, val, c.exp)
}
}
}
func TestIntFieldsUnescaping(t *testing.T) {
cases := []struct {
data string
exp EscIntStruct
}{
{
data: `{}`,
exp: EscIntStruct{A: 0},
},
{
data: `{"a": "1"}`,
exp: EscIntStruct{A: 1},
},
{
data: `{"a": "\u0032"}`,
exp: EscIntStruct{A: 2},
},
}
for i, c := range cases {
var val EscIntStruct
err := easyjson.Unmarshal([]byte(c.data), &val)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, c.exp) {
t.Errorf("[%d] TestIntFieldsUnescaping(): got=%v, exp=%v", i, val, c.exp)
}
}
}
tests/intern_test.go
+8 -2
View File
@@ -12,7 +12,10 @@ func TestStringIntern(t *testing.T) {
var i Intern
allocsPerRun := testing.AllocsPerRun(1000, func() {
i = Intern{}
easyjson.Unmarshal(data, &i)
err := easyjson.Unmarshal(data, &i)
if err != nil {
t.Error(err)
}
if i.Field != "string interning test" {
t.Fatalf("wrong value: %q", i.Field)
}
@@ -24,7 +27,10 @@ func TestStringIntern(t *testing.T) {
var n NoIntern
allocsPerRun = testing.AllocsPerRun(1000, func() {
n = NoIntern{}
easyjson.Unmarshal(data, &n)
err := easyjson.Unmarshal(data, &n)
if err != nil {
t.Error(err)
}
if n.Field != "string interning test" {
t.Fatalf("wrong value: %q", n.Field)
}
tests/members_escaping_test.go
+8 -2
View File
@@ -37,13 +37,19 @@ func TestMembersEscaping(t *testing.T) {
for i, c := range cases {
var esc MembersEscaped
easyjson.Unmarshal([]byte(c.data), &esc)
err := easyjson.Unmarshal([]byte(c.data), &esc)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(esc, c.esc) {
t.Errorf("[%d] TestMembersEscaping(): got=%+v, exp=%+v", i, esc, c.esc)
}
var unesc MembersUnescaped
easyjson.Unmarshal([]byte(c.data), &unesc)
err = easyjson.Unmarshal([]byte(c.data), &unesc)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(unesc, c.unesc) {
t.Errorf("[%d] TestMembersEscaping(): no-unescaping case: got=%+v, exp=%+v", i, esc, c.esc)
}
tests/nocopy_test.go
+12 -3
View File
@@ -27,7 +27,10 @@ func TestNocopy(t *testing.T) {
}
res := NocopyStruct{}
easyjson.Unmarshal(data, &res)
err := easyjson.Unmarshal(data, &res)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(exp, res) {
t.Errorf("TestNocopy(): got=%+v, exp=%+v", res, exp)
}
@@ -42,7 +45,10 @@ func TestNocopy(t *testing.T) {
data = []byte(`{"b": "valueNoCopy"}`)
res = NocopyStruct{}
allocsPerRun := testing.AllocsPerRun(1000, func() {
easyjson.Unmarshal(data, &res)
err := easyjson.Unmarshal(data, &res)
if err != nil {
t.Error(err)
}
if res.B != "valueNoCopy" {
t.Fatalf("wrong value: %q", res.B)
}
@@ -53,7 +59,10 @@ func TestNocopy(t *testing.T) {
data = []byte(`{"a": "valueNoCopy"}`)
allocsPerRun = testing.AllocsPerRun(1000, func() {
easyjson.Unmarshal(data, &res)
err := easyjson.Unmarshal(data, &res)
if err != nil {
t.Error(err)
}
if res.A != "valueNoCopy" {
t.Fatalf("wrong value: %q", res.A)
}