Merge pull request #47 from rvasily/master

correct encode broken utf

jwriter/writer.go

@@ -4,15 +4,15 @@ package jwriter
 import (
 	"io"
 	"strconv"
+	"unicode/utf8"
 
 	"github.com/mailru/easyjson/buffer"
 )
 
 // Writer is a JSON writer.
 type Writer struct {
-	EscapeLtGt bool
-	Error      error
-	Buffer     buffer.Buffer
+	Error  error
+	Buffer buffer.Buffer
 }
 
 // Size returns the size of the data that was written out.
@@ -198,10 +198,7 @@ func (w *Writer) Bool(v bool) {
 	}
 }
 
-func hex(c byte) byte {
-	const chars = "0123456789abcdef"
-	return chars[c&0xf]
-}
+const chars = "0123456789abcdef"
 
 func (w *Writer) String(s string) {
 	w.Buffer.AppendByte('"')
@@ -211,41 +208,65 @@ func (w *Writer) String(s string) {
 
 	p := 0 // last non-escape symbol
 
-	for i := 0; i < len(s); i++ {
-		c := s[i]
-		var escape byte
-		switch c {
-		case '\t':
-			escape = 't'
-		case '\r':
-			escape = 'r'
-		case '\n':
-			escape = 'n'
-		case '\\':
-			escape = '\\'
-		case '"':
-			escape = '"'
-		case '<', '>':
-			if !w.EscapeLtGt {
-				continue
+	for i := 0; i < len(s); {
+		// single-with character
+		if c := s[i]; c < utf8.RuneSelf {
+			var escape byte
+			switch c {
+			case '\t':
+				escape = 't'
+			case '\r':
+				escape = 'r'
+			case '\n':
+				escape = 'n'
+			case '\\':
+				escape = '\\'
+			case '"':
+				escape = '"'
+			case '<', '>':
+				// do nothing
+			default:
+				if c >= 0x20 {
+					// no escaping is required
+					i++
+					continue
+				}
 			}
-		default:
-			if c >= 0x20 {
-				// no escaping is required
-				continue
+			if escape != 0 {
+				w.Buffer.AppendString(s[p:i])
+				w.Buffer.AppendByte('\\')
+				w.Buffer.AppendByte(escape)
+			} else {
+				w.Buffer.AppendString(s[p:i])
+				w.Buffer.AppendString(`\u00`)
+				w.Buffer.AppendByte(chars[c>>4])
+				w.Buffer.AppendByte(chars[c&0xf])
 			}
+			i++
+			p = i
+			continue
 		}
-		if escape != 0 {
+
+		// broken utf
+		runeValue, runeWidth := utf8.DecodeRuneInString(s[i:])
+		if runeValue == utf8.RuneError && runeWidth == 1 {
 			w.Buffer.AppendString(s[p:i])
-			w.Buffer.AppendByte('\\')
-			w.Buffer.AppendByte(escape)
-		} else {
-			w.Buffer.AppendString(s[p:i])
-			w.Buffer.AppendString(`\u00`)
-			w.Buffer.AppendByte(hex(c >> 4))
-			w.Buffer.AppendByte(hex(c))
+			w.Buffer.AppendString(`\ufffd`)
+			i++
+			p = i
+			continue
 		}
-		p = i + 1
+
+		// jsonp stuff - tab separator and line separator
+		if runeValue == '\u2028' || runeValue == '\u2029' {
+			w.Buffer.AppendString(s[p:i])
+			w.Buffer.AppendString(`\u202`)
+			w.Buffer.AppendByte(chars[runeValue&0xf])
+			i += runeWidth
+			p = i
+			continue
+		}
+		i += runeWidth
 	}
 	w.Buffer.AppendString(s[p:])
 	w.Buffer.AppendByte('"')

tests/basic_test.go

@@ -105,24 +105,28 @@ func TestParseNull(t *testing.T) {
 	}
 }
 
-var testCasesEncodeLtGt = []struct {
-	Writer  *jwriter.Writer
-	Encoded string
+var testSpecialCases = []struct {
+	EncodedString string
+	Value string
 }{
-	{&jwriter.Writer{
-		EscapeLtGt: false,
-	}, encodeLtGtFalseWantString},
-	{&jwriter.Writer{
-		EscapeLtGt: true,
-	}, encodeLtGtTrueWantString},
+	{`"Username \u003cuser@example.com\u003e"`, `Username <user@example.com>`},
+	{`"Username\ufffd"`, "Username\xc5"},
+	{`"тестzтест"`, "тестzтест"},
+	{`"тест\ufffdтест"`, "тест\xc5тест"},
+	{`"绿茶"`, "绿茶"},
+	{`"绿\ufffd茶"`, "绿\xc5茶"},
+	{`"тест\u2028"`, "тест\xE2\x80\xA8"},
+	{`"\\\r\n\t\""`, "\\\r\n\t\""},
+	{`"ü"`, "ü"},
 }
 
-func TestEncodeLtGt(t *testing.T) {
-	for i, test := range testCasesEncodeLtGt {
-		test.Writer.String(encodeLtGtString)
-		got := string(test.Writer.Buffer.BuildBytes())
-		if got != test.Encoded {
-			t.Errorf("[%d] Encoded() = %+v; want %+v", i, got, test.Encoded)
+func TestSpecialCases(t *testing.T) {
+	for i, test := range testSpecialCases {
+		w := jwriter.Writer{}
+		w.String(test.Value)
+		got := string(w.Buffer.BuildBytes())
+		if got != test.EncodedString {
+			t.Errorf("[%d] Encoded() = %+v; want %+v", i, got, test.EncodedString)
 		}
 	}
-}
+}

tests/data.go

@@ -530,7 +530,3 @@ type RequiredOptionalStruct struct {
 	FirstName string `json:"first_name,required"`
 	Lastname  string `json:"last_name"`
 }
-
-var encodeLtGtString = `Username <user@example.com>`
-var encodeLtGtFalseWantString = `"Username <user@example.com>"`
-var encodeLtGtTrueWantString = `"Username \u003cuser@example.com\u003e"`