Bug 854169 - Bindings to lz4 for chrome workers. r=froydnj
☠☠ backed out by 2d4fd5a493b1 ☠ ☠
authorDavid Rajchenbach-Teller <dteller@mozilla.com>
Fri, 08 Nov 2013 09:16:04 -0500
changeset 173993 c0f9bfe37d744f82b74c5d35edd8b568077570ea
parent 173992 224d0d4df43475781091c2d5f7de4adac9ea1a91
child 173994 cbf7e38a49aa7ce0340636c7f0023d12182213a2
push id445
push userffxbld
push dateMon, 10 Mar 2014 22:05:19 +0000
treeherdermozilla-release@dc38b741b04e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs854169
milestone28.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 854169 - Bindings to lz4 for chrome workers. r=froydnj
toolkit/components/workerlz4/lz4.cpp
toolkit/components/workerlz4/lz4.js
toolkit/components/workerlz4/lz4_internal.js
new file mode 100644
--- /dev/null
+++ b/toolkit/components/workerlz4/lz4.cpp
@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Compression.h"
+
+/**
+ * LZ4 is a very fast byte-wise compression algorithm.
+ *
+ * Compared to Google's Snappy it is faster to compress and decompress and
+ * generally produces output of about the same size.
+ *
+ * Compared to zlib it compresses at about 10x the speed, decompresses at about
+ * 4x the speed and produces output of about 1.5x the size.
+ *
+ */
+
+using namespace mozilla::Compression;
+
+/**
+ * Compresses 'inputSize' bytes from 'source' into 'dest'.
+ * Destination buffer must be already allocated,
+ * and must be sized to handle worst cases situations (input data not compressible)
+ * Worst case size evaluation is provided by function LZ4_compressBound()
+ *
+ * @param inputSize is the input size. Max supported value is ~1.9GB
+ * @param return the number of bytes written in buffer dest
+ */
+extern "C" MOZ_EXPORT size_t
+workerlz4_compress(const char* source, size_t inputSize, char* dest) {
+  return LZ4::compress(source, inputSize, dest);
+}
+
+/**
+ * If the source stream is malformed, the function will stop decoding
+ * and return a negative result, indicating the byte position of the
+ * faulty instruction
+ *
+ * This function never writes outside of provided buffers, and never
+ * modifies input buffer.
+ *
+ * note : destination buffer must be already allocated.
+ *        its size must be a minimum of 'outputSize' bytes.
+ * @param outputSize is the output size, therefore the original size
+ * @return true/false
+ */
+extern "C" MOZ_EXPORT int
+workerlz4_decompress(const char* source, size_t inputSize,
+                     char* dest, size_t maxOutputSize,
+                     size_t *bytesOutput) {
+  return LZ4::decompress(source, inputSize,
+                         dest, maxOutputSize,
+                         bytesOutput);
+}
+
+
+/*
+  Provides the maximum size that LZ4 may output in a "worst case"
+  scenario (input data not compressible) primarily useful for memory
+  allocation of output buffer.
+  note : this function is limited by "int" range (2^31-1)
+
+  @param inputSize is the input size. Max supported value is ~1.9GB
+  @return maximum output size in a "worst case" scenario
+*/
+extern "C" MOZ_EXPORT size_t
+workerlz4_maxCompressedSize(size_t inputSize)
+{
+  return LZ4::maxCompressedSize(inputSize);
+}
+
+
+
new file mode 100644
--- /dev/null
+++ b/toolkit/components/workerlz4/lz4.js
@@ -0,0 +1,136 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+if (typeof Components != "undefined") {
+  throw new Error("This file is meant to be loaded in a worker");
+}
+if (!module || !exports) {
+  throw new Error("Please load this module with require()");
+}
+
+const SharedAll = require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm");
+const Internals = require("resource://gre/modules/workers/lz4_internal.js");
+
+const MAGIC_NUMBER = new Uint8Array([109, 111, 122, 76, 122, 52, 48, 0]); // "mozLz4a\0"
+
+const BYTES_IN_SIZE_HEADER = ctypes.uint32_t.size;
+
+const HEADER_SIZE = MAGIC_NUMBER.byteLength + BYTES_IN_SIZE_HEADER;
+
+const EXPECTED_HEADER_TYPE = new ctypes.ArrayType(ctypes.uint8_t, HEADER_SIZE);
+const EXPECTED_SIZE_BUFFER_TYPE = new ctypes.ArrayType(ctypes.uint8_t, BYTES_IN_SIZE_HEADER);
+
+/**
+ * An error during (de)compression
+ *
+ * @param {string} operation The name of the operation ("compress", "decompress")
+ * @param {string} reason A reason to be used when matching errors. Must start
+ * with "because", e.g. "becauseInvalidContent".
+ * @param {string} message A human-readable message.
+ */
+function LZError(operation, reason, message) {
+  SharedAll.OSError.call(this);
+  this.operation = operation;
+  this[reason] = true;
+  this.message = message;
+}
+LZError.prototype = Object.create(SharedAll.OSError);
+LZError.prototype.toString = function toString() {
+  return this.message;
+};
+exports.Error = LZError;
+
+/**
+ * Compress a block to a form suitable for writing to disk.
+ *
+ * Compatibility note: For the moment, we are basing our code on lz4
+ * 1.3, which does not specify a *file* format. Therefore, we define
+ * our own format. Once lz4 defines a complete file format, we will
+ * migrate both |compressFileContent| and |decompressFileContent| to this file
+ * format. For backwards-compatibility, |decompressFileContent| will however
+ * keep the ability to decompress files provided with older versions of
+ * |compressFileContent|.
+ *
+ * Compressed files have the following layout:
+ *
+ * | MAGIC_NUMBER (8 bytes) | content size (uint32_t, little endian) | content, as obtained from lz4_compress |
+ *
+ * @param {TypedArray|void*} buffer The buffer to write to the disk.
+ * @param {object=} options An object that may contain the following fields:
+ *  - {number} bytes The number of bytes to read from |buffer|. If |buffer|
+ *    is an |ArrayBuffer|, |bytes| defaults to |buffer.byteLength|. If
+ *    |buffer| is a |void*|, |bytes| MUST be provided.
+ * @return {Uint8Array} An array of bytes suitable for being written to the
+ * disk.
+ */
+function compressFileContent(array, options = {}) {
+  // Prepare the output array
+  let inputBytes;
+  if (SharedAll.isTypedArray(array) && !(options && "bytes" in options)) {
+    inputBytes = array.byteLength;
+  } else if (options && options.bytes) {
+    inputBytes = options.bytes;
+  } else {
+    throw new TypeError("compressFileContent requires a size");
+  }
+  let maxCompressedSize = Internals.maxCompressedSize(inputBytes);
+  let outputArray = new Uint8Array(HEADER_SIZE + maxCompressedSize);
+
+  // Compress to output array
+  let payload = new Uint8Array(outputArray.buffer, outputArray.byteOffset + HEADER_SIZE);
+  let compressedSize = Internals.compress(array, inputBytes, payload);
+
+  // Add headers
+  outputArray.set(MAGIC_NUMBER);
+  let view = new DataView(outputArray.buffer);
+  view.setUint32(MAGIC_NUMBER.byteLength, inputBytes, true);
+
+  return new Uint8Array(outputArray.buffer, 0, HEADER_SIZE + compressedSize);
+}
+exports.compressFileContent = compressFileContent;
+
+function decompressFileContent(array, options = {}) {
+  let {ptr, bytes} = SharedAll.normalizeToPointer(array, options.bytes);
+  if (bytes < HEADER_SIZE) {
+    throw new LZError("decompress", "becauseLZNoHeader", "Buffer is too short (no header)");
+  }
+
+  // Read headers
+  let expectMagicNumber = ctypes.cast(ptr, EXPECTED_HEADER_TYPE.ptr).contents;
+  for (let i = 0; i < MAGIC_NUMBER.byteLength; ++i) {
+    if (expectMagicNumber[i] != MAGIC_NUMBER[i]) {
+      throw new LZError("decompress", "becauseLZWrongMagicNumber", "Invalid header (no magic number");
+    }
+  }
+
+  let sizeBuf =
+    ctypes.cast(
+      SharedAll.offsetBy(ptr, MAGIC_NUMBER.byteLength),
+      EXPECTED_SIZE_BUFFER_TYPE.ptr).contents;
+  let expectDecompressedSize =
+    sizeBuf[0] + (sizeBuf[1] << 8) + (sizeBuf[2] << 16) + (sizeBuf[3] << 24);
+  if (expectDecompressedSize == 0) {
+    // The underlying algorithm cannot handle a size of 0
+    return new Uint8Array(0);
+  }
+
+  // Prepare the input buffer
+  let inputPtr = SharedAll.offsetBy(ptr, HEADER_SIZE);
+
+  // Prepare the output buffer
+  let outputBuffer = new Uint8Array(expectDecompressedSize);
+  let decompressedBytes = (new SharedAll.Type.size_t.implementation(0));
+
+  // Decompress
+  let success = Internals.decompress(inputPtr, bytes - HEADER_SIZE,
+                                     outputBuffer, outputBuffer.byteLength,
+                                     decompressedBytes.address());
+  if (!success) {
+    throw new LZError("decompress", "becauseLZInvalidContent", "Invalid content:Decompression stopped at " + decompressedBytes.value);
+  }
+  return new Uint8Array(outputBuffer.buffer, outputBuffer.byteOffset, decompressedBytes.value);
+}
+exports.decompressFileContent = decompressFileContent;
new file mode 100644
--- /dev/null
+++ b/toolkit/components/workerlz4/lz4_internal.js
@@ -0,0 +1,58 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+if (typeof Components != "undefined") {
+  throw new Error("This file is meant to be loaded in a worker");
+}
+if (!module || !exports) {
+  throw new Error("Please load this module with require()");
+}
+
+let SharedAll = require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm");
+let libxul = ctypes.open(SharedAll.Constants.Path.libxul);
+let declareLazyFII = SharedAll.declareLazyFFI;
+let Type = SharedAll.Type;
+
+let Primitives = {};
+
+declareLazyFII(Primitives, "compress", libxul,
+  "workerlz4_compress",
+  null,
+  /*return*/ Type.size_t,
+  /*const source*/ Type.void_t.in_ptr,
+  /*inputSize*/ Type.size_t,
+  /*dest*/ Type.void_t.out_ptr
+);
+
+declareLazyFII(Primitives, "decompress", libxul,
+  "workerlz4_decompress",
+  null,
+  /*return*/ Type.int,
+  /*const source*/ Type.void_t.in_ptr,
+  /*inputSize*/ Type.size_t,
+  /*dest*/ Type.void_t.out_ptr,
+  /*maxOutputSize*/ Type.size_t,
+  /*actualOutputSize*/ Type.size_t.out_ptr
+);
+
+declareLazyFII(Primitives, "maxCompressedSize", libxul,
+  "workerlz4_maxCompressedSize",
+  null,
+  /*return*/ Type.size_t,
+  /*inputSize*/ Type.size_t
+);
+
+module.exports = {
+  get compress() {
+    return Primitives.compress;
+  },
+  get decompress() {
+    return Primitives.decompress;
+  },
+  get maxCompressedSize() {
+    return Primitives.maxCompressedSize;
+  }
+};