Bug 854169 - OS.File.{read, writeAtomic} support for lz4. r=froydnj

toolkit/components/osfile/modules/osfile_async_front.jsm

@@ -763,18 +763,21 @@ File.makeDir = function makeDir(path, options) {
  *
  * @param {string} path The path to the file.
  * @param {number=} bytes Optionally, an upper bound to the number of bytes
- * to read.
+ * to read. DEPRECATED - please use options.bytes instead.
  * @param {JSON} options Additional options.
  * - {boolean} sequential A flag that triggers a population of the page cache
  * with data from a file so that subsequent reads from that file would not
  * block on disk I/O. If |true| or unspecified, inform the system that the
  * contents of the file will be read in order. Otherwise, make no such
  * assumption. |true| by default.
+ * - {number} bytes An upper bound to the number of bytes to read.
+ * - {string} compression If "lz4" and if the file is compressed using the lz4
+ * compression algorithm, decompress the file contents on the fly.
  *
  * @resolves {Uint8Array} A buffer holding the bytes
  * read from the file.
  */
-File.read = function read(path, bytes, options) {
+File.read = function read(path, bytes, options = {}) {
   let promise = Scheduler.post("read",
     [Type.path.toMsg(path), bytes, options], path);
   return promise.then(

toolkit/components/osfile/modules/osfile_async_worker.js

@@ -93,10 +93,11 @@ if (this.Components) {
      // instances of |OS.File.Error|)
      self.postMessage({fail: exports.OS.File.Error.toMsg(exn), id:id, durationMs: durationMs});
    } else {
-     LOG("Sending back regular error", exn, exn.stack, "id is", id);
      // Other exceptions do not, and should be propagated through DOM's
      // built-in mechanism for uncaught errors, although this mechanism
      // may lose interesting information.
+     LOG("Sending back regular error", exn, exn.stack, "id is", id);
+
      throw exn;
    }
   };

toolkit/components/osfile/modules/osfile_shared_front.jsm

@@ -16,7 +16,8 @@ if (typeof Components != "undefined") {
 
 let SharedAll =
   require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm");
-
+let Lz4 =
+  require("resource://gre/modules/workers/lz4.js");
 let LOG = SharedAll.LOG.bind(SharedAll, "Shared front-end");
 let clone = SharedAll.clone;
 
@@ -310,16 +311,29 @@ AbstractFile.normalizeOpenMode = function normalizeOpenMode(mode) {
  *
  * @param {string} path The path to the file.
  * @param {number=} bytes Optionally, an upper bound to the number of bytes
- * to read.
- * @param {JSON} options Optionally contains additional options.
+ * to read. DEPRECATED - please use options.bytes instead.
+ * @param {object=} options Optionally, an object with some of the following
+ * fields:
+ * - {number} bytes An upper bound to the number of bytes to read.
+ * - {string} compression If "lz4" and if the file is compressed using the lz4
+ *   compression algorithm, decompress the file contents on the fly.
  *
  * @return {Uint8Array} A buffer holding the bytes
  * and the number of bytes read from the file.
  */
 AbstractFile.read = function read(path, bytes, options = {}) {
+  if (bytes && typeof bytes == "object") {
+    options = bytes;
+    bytes = options.bytes || null;
+  }
   let file = exports.OS.File.open(path);
   try {
-    return file.read(bytes, options);
+    let buffer = file.read(bytes, options);
+    if (options.compression == "lz4") {
+      return Lz4.decompressFileContent(buffer, options);
+    } else {
+      return buffer;
+    }
   } finally {
     file.close();
   }
@@ -360,6 +374,10 @@ AbstractFile.read = function read(path, bytes, options = {}) {
  * if the system shuts down improperly (typically due to a kernel freeze
  * or a power failure) or if the device is disconnected before the buffer
  * is flushed, the file has more chances of not being corrupted.
+ * - {string} compression - If empty or unspecified, do not compress the file.
+ * If "lz4", compress the contents of the file atomically using lz4. For the
+ * time being, the container format is specific to Mozilla and cannot be read
+ * by means other than OS.File.read(..., { compression: "lz4"})
  *
  * @return {number} The number of bytes actually written.
  */
@@ -381,6 +399,12 @@ AbstractFile.writeAtomic =
     buffer = new TextEncoder(encoding).encode(buffer);
   }
 
+  if (options.compression == "lz4") {
+    buffer = Lz4.compressFileContent(buffer, options);
+    options = Object.create(options);
+    options.bytes = buffer.byteLength;
+  }
+
   let bytesWritten = 0;
 
   if (!options.tmpPath) {

toolkit/components/osfile/tests/xpcshell/test_compression.js

@@ -0,0 +1,52 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Components.utils.import("resource://gre/modules/osfile.jsm");
+
+function run_test() {
+  do_test_pending();
+  run_next_test();
+}
+
+add_task(function test_compress_lz4() {
+  let path = OS.Path.join(OS.Constants.Path.tmpDir, "compression.lz");
+  let array = new Uint8Array(1024);
+  for (let i = 0; i < array.byteLength; ++i) {
+    array[i] = i;
+  }
+
+  do_print("Writing data with lz4 compression");
+  let bytes = yield OS.File.writeAtomic(path, array, { compression: "lz4" });
+  do_print("Compressed " + array.byteLength + " bytes into " + bytes);
+
+  do_print("Reading back with lz4 decompression");
+  let decompressed = yield OS.File.read(path, { compression: "lz4" });
+  do_print("Decompressed into " + decompressed.byteLength + " bytes");
+  do_check_eq(Array.prototype.join.call(array), Array.prototype.join.call(decompressed));
+});
+
+add_task(function test_uncompressed() {
+  do_print("Writing data without compression");
+  let path = OS.Path.join(OS.Constants.Path.tmpDir, "no_compression.tmp");
+  let array = new Uint8Array(1024);
+  for (let i = 0; i < array.byteLength; ++i) {
+    array[i] = i;
+  }
+  let bytes = yield OS.File.writeAtomic(path, array); // No compression
+
+  let exn;
+  // Force decompression, reading should fail
+  try {
+    yield OS.File.read(path, { compression: "lz4" });
+  } catch (ex) {
+    exn = ex;
+  }
+  do_check_true(!!exn);
+  do_check_true(exn.message.indexOf("Invalid header") != -1);
+});
+
+add_task(function() {
+  do_test_finished();
+});

toolkit/components/osfile/tests/xpcshell/xpcshell.ini

@@ -24,3 +24,4 @@ tail =
 [test_shutdown.js]
 [test_unique.js]
 [test_open.js]
+[test_compression.js]