From 7b253e1d2bf5ec45435069dd5a775ab05c0eb795 Mon Sep 17 00:00:00 2001 From: David Rajchenbach-Teller Date: Thu, 9 Jan 2014 17:10:05 +0100 Subject: [PATCH] Bug 854169 - OS.File.{read, writeAtomic} support for lz4. r=froydnj --- .../osfile/modules/osfile_async_front.jsm | 7 ++- .../osfile/modules/osfile_async_worker.js | 3 +- .../osfile/modules/osfile_shared_front.jsm | 32 ++++++++++-- .../osfile/tests/xpcshell/test_compression.js | 52 +++++++++++++++++++ .../osfile/tests/xpcshell/xpcshell.ini | 1 + 5 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 toolkit/components/osfile/tests/xpcshell/test_compression.js diff --git a/toolkit/components/osfile/modules/osfile_async_front.jsm b/toolkit/components/osfile/modules/osfile_async_front.jsm index fbe1777f681..fa18d5b597d 100644 --- a/toolkit/components/osfile/modules/osfile_async_front.jsm +++ b/toolkit/components/osfile/modules/osfile_async_front.jsm @@ -763,18 +763,21 @@ File.makeDir = function makeDir(path, options) { * * @param {string} path The path to the file. * @param {number=} bytes Optionally, an upper bound to the number of bytes - * to read. + * to read. DEPRECATED - please use options.bytes instead. * @param {JSON} options Additional options. * - {boolean} sequential A flag that triggers a population of the page cache * with data from a file so that subsequent reads from that file would not * block on disk I/O. If |true| or unspecified, inform the system that the * contents of the file will be read in order. Otherwise, make no such * assumption. |true| by default. + * - {number} bytes An upper bound to the number of bytes to read. + * - {string} compression If "lz4" and if the file is compressed using the lz4 + * compression algorithm, decompress the file contents on the fly. * * @resolves {Uint8Array} A buffer holding the bytes * read from the file. */ -File.read = function read(path, bytes, options) { +File.read = function read(path, bytes, options = {}) { let promise = Scheduler.post("read", [Type.path.toMsg(path), bytes, options], path); return promise.then( diff --git a/toolkit/components/osfile/modules/osfile_async_worker.js b/toolkit/components/osfile/modules/osfile_async_worker.js index b1ae1e093a5..6f9cc0cd11a 100644 --- a/toolkit/components/osfile/modules/osfile_async_worker.js +++ b/toolkit/components/osfile/modules/osfile_async_worker.js @@ -93,10 +93,11 @@ if (this.Components) { // instances of |OS.File.Error|) self.postMessage({fail: exports.OS.File.Error.toMsg(exn), id:id, durationMs: durationMs}); } else { - LOG("Sending back regular error", exn, exn.stack, "id is", id); // Other exceptions do not, and should be propagated through DOM's // built-in mechanism for uncaught errors, although this mechanism // may lose interesting information. + LOG("Sending back regular error", exn, exn.stack, "id is", id); + throw exn; } }; diff --git a/toolkit/components/osfile/modules/osfile_shared_front.jsm b/toolkit/components/osfile/modules/osfile_shared_front.jsm index 3e1ef9a06fe..7d18bc1ebc0 100644 --- a/toolkit/components/osfile/modules/osfile_shared_front.jsm +++ b/toolkit/components/osfile/modules/osfile_shared_front.jsm @@ -16,7 +16,8 @@ if (typeof Components != "undefined") { let SharedAll = require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm"); - +let Lz4 = + require("resource://gre/modules/workers/lz4.js"); let LOG = SharedAll.LOG.bind(SharedAll, "Shared front-end"); let clone = SharedAll.clone; @@ -310,16 +311,29 @@ AbstractFile.normalizeOpenMode = function normalizeOpenMode(mode) { * * @param {string} path The path to the file. * @param {number=} bytes Optionally, an upper bound to the number of bytes - * to read. - * @param {JSON} options Optionally contains additional options. + * to read. DEPRECATED - please use options.bytes instead. + * @param {object=} options Optionally, an object with some of the following + * fields: + * - {number} bytes An upper bound to the number of bytes to read. + * - {string} compression If "lz4" and if the file is compressed using the lz4 + * compression algorithm, decompress the file contents on the fly. * * @return {Uint8Array} A buffer holding the bytes * and the number of bytes read from the file. */ AbstractFile.read = function read(path, bytes, options = {}) { + if (bytes && typeof bytes == "object") { + options = bytes; + bytes = options.bytes || null; + } let file = exports.OS.File.open(path); try { - return file.read(bytes, options); + let buffer = file.read(bytes, options); + if (options.compression == "lz4") { + return Lz4.decompressFileContent(buffer, options); + } else { + return buffer; + } } finally { file.close(); } @@ -360,6 +374,10 @@ AbstractFile.read = function read(path, bytes, options = {}) { * if the system shuts down improperly (typically due to a kernel freeze * or a power failure) or if the device is disconnected before the buffer * is flushed, the file has more chances of not being corrupted. + * - {string} compression - If empty or unspecified, do not compress the file. + * If "lz4", compress the contents of the file atomically using lz4. For the + * time being, the container format is specific to Mozilla and cannot be read + * by means other than OS.File.read(..., { compression: "lz4"}) * * @return {number} The number of bytes actually written. */ @@ -381,6 +399,12 @@ AbstractFile.writeAtomic = buffer = new TextEncoder(encoding).encode(buffer); } + if (options.compression == "lz4") { + buffer = Lz4.compressFileContent(buffer, options); + options = Object.create(options); + options.bytes = buffer.byteLength; + } + let bytesWritten = 0; if (!options.tmpPath) { diff --git a/toolkit/components/osfile/tests/xpcshell/test_compression.js b/toolkit/components/osfile/tests/xpcshell/test_compression.js new file mode 100644 index 00000000000..bc0167c52b7 --- /dev/null +++ b/toolkit/components/osfile/tests/xpcshell/test_compression.js @@ -0,0 +1,52 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Components.utils.import("resource://gre/modules/osfile.jsm"); + +function run_test() { + do_test_pending(); + run_next_test(); +} + +add_task(function test_compress_lz4() { + let path = OS.Path.join(OS.Constants.Path.tmpDir, "compression.lz"); + let array = new Uint8Array(1024); + for (let i = 0; i < array.byteLength; ++i) { + array[i] = i; + } + + do_print("Writing data with lz4 compression"); + let bytes = yield OS.File.writeAtomic(path, array, { compression: "lz4" }); + do_print("Compressed " + array.byteLength + " bytes into " + bytes); + + do_print("Reading back with lz4 decompression"); + let decompressed = yield OS.File.read(path, { compression: "lz4" }); + do_print("Decompressed into " + decompressed.byteLength + " bytes"); + do_check_eq(Array.prototype.join.call(array), Array.prototype.join.call(decompressed)); +}); + +add_task(function test_uncompressed() { + do_print("Writing data without compression"); + let path = OS.Path.join(OS.Constants.Path.tmpDir, "no_compression.tmp"); + let array = new Uint8Array(1024); + for (let i = 0; i < array.byteLength; ++i) { + array[i] = i; + } + let bytes = yield OS.File.writeAtomic(path, array); // No compression + + let exn; + // Force decompression, reading should fail + try { + yield OS.File.read(path, { compression: "lz4" }); + } catch (ex) { + exn = ex; + } + do_check_true(!!exn); + do_check_true(exn.message.indexOf("Invalid header") != -1); +}); + +add_task(function() { + do_test_finished(); +}); diff --git a/toolkit/components/osfile/tests/xpcshell/xpcshell.ini b/toolkit/components/osfile/tests/xpcshell/xpcshell.ini index 8e9825be762..0aed324a2d2 100644 --- a/toolkit/components/osfile/tests/xpcshell/xpcshell.ini +++ b/toolkit/components/osfile/tests/xpcshell/xpcshell.ini @@ -24,3 +24,4 @@ tail = [test_shutdown.js] [test_unique.js] [test_open.js] +[test_compression.js]