Bug 854169 - OS.File.{read, writeAtomic} support for lz4. r=froydnj
☠☠ backed out by 2d4fd5a493b1 ☠ ☠
authorDavid Rajchenbach-Teller <dteller@mozilla.com>
Fri, 08 Nov 2013 09:16:05 -0500
changeset 173995 bd7733e5d5cf2f5505639e18055889b57487a468
parent 173994 cbf7e38a49aa7ce0340636c7f0023d12182213a2
child 173996 70811e50312c92655097b778f12b78348936f613
push id445
push userffxbld
push dateMon, 10 Mar 2014 22:05:19 +0000
treeherdermozilla-release@dc38b741b04e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs854169
milestone28.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 854169 - OS.File.{read, writeAtomic} support for lz4. r=froydnj
toolkit/components/osfile/modules/osfile_async_front.jsm
toolkit/components/osfile/modules/osfile_async_worker.js
toolkit/components/osfile/modules/osfile_shared_front.jsm
toolkit/components/osfile/tests/xpcshell/test_compression.js
toolkit/components/osfile/tests/xpcshell/xpcshell.ini
--- a/toolkit/components/osfile/modules/osfile_async_front.jsm
+++ b/toolkit/components/osfile/modules/osfile_async_front.jsm
@@ -725,28 +725,31 @@ File.makeDir = function makeDir(path, op
     [Type.path.toMsg(path), options], path);
 };
 
 /**
  * Return the contents of a file
  *
  * @param {string} path The path to the file.
  * @param {number=} bytes Optionally, an upper bound to the number of bytes
- * to read.
+ * to read. DEPRECATED - please use options.bytes instead.
  * @param {JSON} options Additional options.
  * - {boolean} sequential A flag that triggers a population of the page cache
  * with data from a file so that subsequent reads from that file would not
  * block on disk I/O. If |true| or unspecified, inform the system that the
  * contents of the file will be read in order. Otherwise, make no such
  * assumption. |true| by default.
+ * - {number} bytes An upper bound to the number of bytes to read.
+ * - {string} compression If "lz4" and if the file is compressed using the lz4
+ * compression algorithm, decompress the file contents on the fly.
  *
  * @resolves {Uint8Array} A buffer holding the bytes
  * read from the file.
  */
-File.read = function read(path, bytes, options) {
+File.read = function read(path, bytes, options = {}) {
   let promise = Scheduler.post("read",
     [Type.path.toMsg(path), bytes, options], path);
   return promise.then(
     function onSuccess(data) {
       return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
     });
 };
 
--- a/toolkit/components/osfile/modules/osfile_async_worker.js
+++ b/toolkit/components/osfile/modules/osfile_async_worker.js
@@ -88,20 +88,21 @@ if (this.Components) {
      self.postMessage({StopIteration: true, id: id, durationMs: durationMs});
    } else if (exn instanceof exports.OS.File.Error) {
      LOG("Sending back OS.File error", exn, "id is", id);
      // Instances of OS.File.Error know how to serialize themselves
      // (deserialization ensures that we end up with OS-specific
      // instances of |OS.File.Error|)
      self.postMessage({fail: exports.OS.File.Error.toMsg(exn), id:id, durationMs: durationMs});
    } else {
-     LOG("Sending back regular error", exn, exn.stack, "id is", id);
      // Other exceptions do not, and should be propagated through DOM's
      // built-in mechanism for uncaught errors, although this mechanism
      // may lose interesting information.
+     LOG("Sending back regular error", exn, exn.stack, "id is", id);
+
      throw exn;
    }
   };
 
  /**
   * A data structure used to track opened resources
   */
   let ResourceTracker = function ResourceTracker() {
--- a/toolkit/components/osfile/modules/osfile_shared_front.jsm
+++ b/toolkit/components/osfile/modules/osfile_shared_front.jsm
@@ -11,17 +11,18 @@
 
 if (typeof Components != "undefined") {
   throw new Error("osfile_shared_front.jsm cannot be used from the main thread");
 }
 (function(exports) {
 
 let SharedAll =
   require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm");
-
+let Lz4 =
+  require("resource://gre/modules/workers/lz4.js");
 let LOG = SharedAll.LOG.bind(SharedAll, "Shared front-end");
 let clone = SharedAll.clone;
 
 /**
  * Code shared by implementations of File.
  *
  * @param {*} fd An OS-specific file handle.
  * @constructor
@@ -305,26 +306,39 @@ AbstractFile.normalizeOpenMode = functio
   return result;
 };
 
 /**
  * Return the contents of a file.
  *
  * @param {string} path The path to the file.
  * @param {number=} bytes Optionally, an upper bound to the number of bytes
- * to read.
- * @param {JSON} options Optionally contains additional options.
+ * to read. DEPRECATED - please use options.bytes instead.
+ * @param {object=} options Optionally, an object with some of the following
+ * fields:
+ * - {number} bytes An upper bound to the number of bytes to read.
+ * - {string} compression If "lz4" and if the file is compressed using the lz4
+ *   compression algorithm, decompress the file contents on the fly.
  *
  * @return {Uint8Array} A buffer holding the bytes
  * and the number of bytes read from the file.
  */
 AbstractFile.read = function read(path, bytes, options = {}) {
+  if (bytes && typeof bytes == "object") {
+    options = bytes;
+    bytes = options.bytes || null;
+  }
   let file = exports.OS.File.open(path);
   try {
-    return file.read(bytes, options);
+    let buffer = file.read(bytes, options);
+    if (options.compression == "lz4") {
+      return Lz4.decompressFileContent(buffer, options);
+    } else {
+      return buffer;
+    }
   } finally {
     file.close();
   }
 };
 
 /**
  * Write a file, atomically.
  *
@@ -355,16 +369,20 @@ AbstractFile.read = function read(path, 
  * exists at |path|.
  * - {bool} flush - If |false| or unspecified, return immediately once the
  * write is complete. If |true|, before writing, force the operating system
  * to write its internal disk buffers to the disk. This is considerably slower
  * (not just for the application but for the whole system) but also safer:
  * if the system shuts down improperly (typically due to a kernel freeze
  * or a power failure) or if the device is disconnected before the buffer
  * is flushed, the file has more chances of not being corrupted.
+ * - {string} compression - If empty or unspecified, do not compress the file.
+ * If "lz4", compress the contents of the file atomically using lz4. For the
+ * time being, the container format is specific to Mozilla and cannot be read
+ * by means other than OS.File.read(..., { compression: "lz4"})
  *
  * @return {number} The number of bytes actually written.
  */
 AbstractFile.writeAtomic =
      function writeAtomic(path, buffer, options = {}) {
 
   // Verify that path is defined and of the correct type
   if (typeof path != "string" || path == "") {
@@ -376,16 +394,22 @@ AbstractFile.writeAtomic =
   }
 
   if (typeof buffer == "string") {
     // Normalize buffer to a C buffer by encoding it
     let encoding = options.encoding || "utf-8";
     buffer = new TextEncoder(encoding).encode(buffer);
   }
 
+  if (options.compression == "lz4") {
+    buffer = Lz4.compressFileContent(buffer, options);
+    options = Object.create(options);
+    options.bytes = buffer.byteLength;
+  }
+
   let bytesWritten = 0;
 
   if (!options.tmpPath) {
     // Just write, without any renaming trick
     let dest = OS.File.open(path, {write: true, truncate: true});
     try {
       bytesWritten = dest.write(buffer, options);
       if (options.flush) {
new file mode 100644
--- /dev/null
+++ b/toolkit/components/osfile/tests/xpcshell/test_compression.js
@@ -0,0 +1,52 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Components.utils.import("resource://gre/modules/osfile.jsm");
+
+function run_test() {
+  do_test_pending();
+  run_next_test();
+}
+
+add_task(function test_compress_lz4() {
+  let path = OS.Path.join(OS.Constants.Path.tmpDir, "compression.lz");
+  let array = new Uint8Array(1024);
+  for (let i = 0; i < array.byteLength; ++i) {
+    array[i] = i;
+  }
+
+  do_print("Writing data with lz4 compression");
+  let bytes = yield OS.File.writeAtomic(path, array, { compression: "lz4" });
+  do_print("Compressed " + array.byteLength + " bytes into " + bytes);
+
+  do_print("Reading back with lz4 decompression");
+  let decompressed = yield OS.File.read(path, { compression: "lz4" });
+  do_print("Decompressed into " + decompressed.byteLength + " bytes");
+  do_check_eq(Array.prototype.join.call(array), Array.prototype.join.call(decompressed));
+});
+
+add_task(function test_uncompressed() {
+  do_print("Writing data without compression");
+  let path = OS.Path.join(OS.Constants.Path.tmpDir, "no_compression.tmp");
+  let array = new Uint8Array(1024);
+  for (let i = 0; i < array.byteLength; ++i) {
+    array[i] = i;
+  }
+  let bytes = yield OS.File.writeAtomic(path, array); // No compression
+
+  let exn;
+  // Force decompression, reading should fail
+  try {
+    yield OS.File.read(path, { compression: "lz4" });
+  } catch (ex) {
+    exn = ex;
+  }
+  do_check_true(!!exn);
+  do_check_true(exn.message.indexOf("Invalid header") != -1);
+});
+
+add_task(function() {
+  do_test_finished();
+});
--- a/toolkit/components/osfile/tests/xpcshell/xpcshell.ini
+++ b/toolkit/components/osfile/tests/xpcshell/xpcshell.ini
@@ -17,8 +17,9 @@ tail =
 [test_creationDate.js]
 [test_exception.js]
 [test_path_constants.js]
 [test_removeDir.js]
 [test_reset.js]
 [test_shutdown.js]
 [test_unique.js]
 [test_open.js]
+[test_compression.js]