Bug 512637 - Use newlines instead of JSON collection for incremental parsing. r=thunder

Switch to newline mode when using a collection record handler, and look for newlines! Easy! Update test to provide newline-separated strings instead of JSON.

services/sync/modules/base_records/collection.js

@@ -127,63 +127,15 @@ Collection.prototype = {
     // Save this because onProgress is called with this as the ChannelListener
     let coll = this;
 
+    // Switch to newline separated records for incremental parsing
+    coll.setHeader("Accept", "application/newlines");
+
     this._onProgress = function() {
-      // Save some work by quitting early when there's no records
+      let newline;
-      if (this._data == "[]")
+      while ((newline = this._data.indexOf("\n")) > 0) {
-        return;
+        // Split the json record from the rest of the data
-
+        let json = this._data.slice(0, newline);
-      do {
+        this._data = this._data.slice(newline + 1);
-        // Strip off the the starting "[" or separating "," or trailing "]" if
-        // it wasn't stripped off from a previous progress update
-        let start = this._data[0];
-        if (start == "[" || start == "," || start == "]")
-          this._data = this._data.slice(1);
-
-        // Track various states of # open braces and ignore for strings
-        let json = "";
-        let braces = 1;
-        let ignore = false;
-        let escaped = false;
-        let length = this._data.length;
-
-        // Skip the first character, the "{", and try to find a json record
-        for (let i = 1; i < length; i++) {
-          let char = this._data[i];
-
-          // Opening a string makes us ignore all characters except close "
-          if (char == '"') {
-            if (!ignore)
-              ignore = true;
-            // It's a real close " if it's not escaped
-            else if (!escaped)
-              ignore = false;
-          }
-
-          // Track if an end quote might be escaped when processing strings
-          if (ignore) {
-            escaped = char == "\\" ? !escaped : false;
-
-            // Don't bother checking other characters when ignoring
-            continue;
-          }
-
-          // Increase the brace count on open {
-          if (char == "{")
-            braces++;
-          // Decrement brace count on close }
-          else if (char == "}" && --braces == 0) {
-            // Split the json record from the rest of the data
-            json = this._data.slice(0, i + 1);
-            this._data = this._data.slice(i + 1);
-
-            // Stop processing for now that we found one record
-            break;
-          }
-        }
-
-        // No valid record json found?
-        if (json.length == 0)
-          break;
 
         // Deserialize a record from json and give it to the callback
         let record = new coll._recordObj();
@@ -191,9 +143,7 @@ Collection.prototype = {
         record.baseURI = coll.uri;
         record.id = record.data.id;
         onRecord(record);
-
+      }
-      // Keep processing the data until we can't find a json record
-      } while (true);
 
       // Aggressively clean up the objects we created above so that the next set
       // of records have enough memory to decrypt, reconcile, apply, etc.

services/sync/tests/unit/test_collection_inc_get.js

@@ -9,7 +9,7 @@ function run_test() {
 
   _("Parse empty string payload as deleted");
   called = false;
-  stream._data = '[{"payload":""}]';
+  stream._data = '{"payload":""}\n';
   coll.recordHandler = function(rec) {
     called = true;
     _("Got record:", JSON.stringify(rec));
@@ -23,7 +23,7 @@ function run_test() {
 
   _("Parse record with payload");
   called = false;
-  stream._data = '[{"payload":"{\\"value\\":123}"}]';
+  stream._data = '{"payload":"{\\"value\\":123}"}\n';
   coll.recordHandler = function(rec) {
     called = true;
     _("Got record:", JSON.stringify(rec));
@@ -39,7 +39,7 @@ function run_test() {
   called = false;
   recCount = 0;
   sum = 0;
-  stream._data = '[{"payload":"{\\"value\\":100}"},{"payload":"{\\"value\\":10}"},{"payload":"{\\"value\\":1}"}]';
+  stream._data = '{"payload":"{\\"value\\":100}"}\n{"payload":"{\\"value\\":10}"}\n{"payload":"{\\"value\\":1}"}\n';
   coll.recordHandler = function(rec) {
     called = true;
     _("Got record:", JSON.stringify(rec));
@@ -76,7 +76,7 @@ function run_test() {
   called = false;
   recCount = 0;
   sum = 0;
-  stream._data = '[{"payl';
+  stream._data = '{"payl';
   coll.recordHandler = function(rec) {
     called = true;
     do_throw("shouldn't have gotten a record..");
@@ -92,7 +92,7 @@ function run_test() {
 
   _("adding more data enough for one record..");
   called = false;
-  stream._data += 'oad":"{\\"value\\":100}"},';
+  stream._data += 'oad":"{\\"value\\":100}"}\n';
   coll.recordHandler = function(rec) {
     called = true;
     _("Got record:", JSON.stringify(rec));
@@ -126,7 +126,7 @@ function run_test() {
 
   _("add data for two records..");
   called = false;
-  stream._data += '},{"payload":"{\\"value\\":1}"}';
+  stream._data += '}\n{"payload":"{\\"value\\":1}"}\n';
   coll.recordHandler = function(rec) {
     called = true;
     _("Got record:", JSON.stringify(rec));
@@ -155,9 +155,9 @@ function run_test() {
   do_check_true(called);
   _();
 
-  _("add ending array bracket");
+  _("add no extra data");
   called = false;
-  stream._data += ']';
+  stream._data += '';
   coll.recordHandler = function(rec) {
     called = true;
     do_throw("shouldn't have gotten a record..");
@@ -166,7 +166,7 @@ function run_test() {
   _("should still have 3 records with sum 111");
   do_check_eq(recCount, 3);
   do_check_eq(sum, 111);
-  _("should have consumed the last array bracket");
+  _("should have consumed nothing but still have nothing");
   do_check_eq(stream._data, "");
   do_check_false(called);
   _("\n");