Bug 761877: Add correct UTF-8 handling to RESTRequest; r=gps

services/common/rest.js

@@ -370,10 +370,17 @@ RESTRequest.prototype = {
     response.body = "";
 
     // Define this here so that we don't have make a new one each time
-    // onDataAvailable() gets called.
-    this._inputStream = Cc["@mozilla.org/scriptableinputstream;1"]
+    // onDataAvailable() gets called. If the Content-Type specified a charset,
+    // make sure we use the correct converter stream, instead of a generic
+    // ScriptableInputStream (onDataAvailable will pick the right one).
+    if (channel.contentCharset) {
+      response.charset = channel.contentCharset;
+      this._converterStream = Cc["@mozilla.org/intl/converter-input-stream;1"]
+                                .createInstance(Ci.nsIConverterInputStream);
+    } else {
+      this._inputStream = Cc["@mozilla.org/scriptableinputstream;1"]
                           .createInstance(Ci.nsIScriptableInputStream);
-
+    }
     this.delayTimeout();
   },
 
@@ -435,9 +442,21 @@ RESTRequest.prototype = {
   },
 
   onDataAvailable: function onDataAvailable(req, cb, stream, off, count) {
-    this._inputStream.init(stream);
     try {
-      this.response.body += this._inputStream.read(count);
+      if (this._inputStream) {
+        this._inputStream.init(stream);
+        this.response.body += this._inputStream.read(count);
+      } else {
+        let str = {};
+        this._converterStream.init(
+          stream, this.response.charset, 0,
+          this._converterStream.DEFAULT_REPLACEMENT_CHARACTER
+        );
+        let num = this._converterStream.readString(count, str);
+        if (num != 0) {
+          this.response.body += str.value;
+        }
+      }
     } catch (ex) {
       this._log.warn("Exception thrown reading " + count +
                      " bytes from the channel.");

services/common/tests/unit/test_restrequest.js

@@ -159,6 +159,36 @@ add_test(function test_get() {
   });
 });
 
+/**
+ * Test HTTP GET with UTF-8 content, and custom Content-Type.
+ */
+add_test(function test_get_utf8() {
+  let response = "Hello World or Καλημέρα κόσμε or こんにちは 世界";
+  let contentType = "text/plain; charset=UTF-8";
+
+  let server = httpd_setup({"/resource": function(req, res) {
+    res.setStatusLine(req.httpVersion, 200, "OK");
+    res.setHeader("Content-Type", contentType);
+
+    let converter = Cc["@mozilla.org/intl/converter-output-stream;1"]
+                    .createInstance(Ci.nsIConverterOutputStream);
+    converter.init(res.bodyOutputStream, "UTF-8", 0, 0x0000);
+    converter.writeString(response);
+    converter.close();
+  }});
+
+  let request = new RESTRequest(TEST_RESOURCE_URL);
+  request.get(function(error) {
+    do_check_eq(error, null);
+
+    do_check_eq(request.response.status, 200);
+    do_check_eq(request.response.body, response);
+    do_check_eq(request.response.headers["content-type"], contentType);
+
+    server.stop(run_next_test);
+  });
+});
+
 /**
  * Test HTTP PUT with a simple string argument and default Content-Type.
  */