Bug 641212 - use lzma compression for application update. Part 2 - add xe-embedded decompression support to the updater. r=mhowell, , a=app_update_lzma
authorRobert Strong <robert.bugzilla@gmail.com>
Sun, 30 Jul 2017 23:27:00 -0700
changeset 420621 692137868c6b6f3ea237c8e4cd1dc246d5f29088
parent 420620 89a4dd1ba384127da48a2804f59f6edf61835645
child 420622 8fbee1d5b7481487774b82e67a8e5b4f1b06faf6
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmhowell, app_update_lzma
bugs641212
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 641212 - use lzma compression for application update. Part 2 - add xe-embedded decompression support to the updater. r=mhowell, , a=app_update_lzma Changes to the libmar host utilities (mar and signmar) are backwards compatible with the previous versions of mar and signmar
modules/libmar/src/mar.h
modules/libmar/src/mar_extract.c
modules/libmar/src/mar_read.c
toolkit/mozapps/update/common/errors.h
toolkit/mozapps/update/updater/archivereader.cpp
toolkit/mozapps/update/updater/updater-common.build
--- a/modules/libmar/src/mar.h
+++ b/modules/libmar/src/mar.h
@@ -103,17 +103,17 @@ int mar_enum_items(MarFile *mar, MarItem
  * @param mar       The MAR file to read.
  * @param item      The MAR item to read.
  * @param offset    The byte offset relative to the start of the item.
  * @param buf       A pointer to a buffer to copy the data into.
  * @param bufsize   The length of the buffer to copy the data into.
  * @return          The number of bytes written or a negative value if an
  *                  error occurs.
  */
-int mar_read(MarFile *mar, const MarItem *item, int offset, char *buf,
+int mar_read(MarFile *mar, const MarItem *item, int offset, uint8_t *buf,
              int bufsize);
 
 /**
  * Create a MAR file from a set of files.
  * @param dest      The path to the file to create.  This path must be
  *                  compatible with fopen.
  * @param numfiles  The number of files to store in the archive.
  * @param files     The list of null-terminated file paths.  Each file
--- a/modules/libmar/src/mar_extract.c
+++ b/modules/libmar/src/mar_extract.c
@@ -32,17 +32,17 @@ static int mar_ensure_parent_dir(const c
 #endif
     *slash = '/';
   }
   return 0;
 }
 
 static int mar_test_callback(MarFile *mar, const MarItem *item, void *unused) {
   FILE *fp;
-  char buf[BLOCKSIZE];
+  uint8_t buf[BLOCKSIZE];
   int fd, len, offset = 0;
 
   if (mar_ensure_parent_dir(item->name))
     return -1;
 
 #ifdef XP_WIN
   fd = _open(item->name, _O_BINARY|_O_CREAT|_O_TRUNC|_O_WRONLY, item->flags);
 #else
--- a/modules/libmar/src/mar_read.c
+++ b/modules/libmar/src/mar_read.c
@@ -511,17 +511,17 @@ int mar_enum_items(MarFile *mar, MarItem
         return rv;
       item = item->next;
     }
   }
 
   return 0;
 }
 
-int mar_read(MarFile *mar, const MarItem *item, int offset, char *buf,
+int mar_read(MarFile *mar, const MarItem *item, int offset, uint8_t *buf,
              int bufsize) {
   int nr;
 
   if (offset == (int) item->length)
     return 0;
   if (offset > (int) item->length)
     return -1;
 
--- a/toolkit/mozapps/update/common/errors.h
+++ b/toolkit/mozapps/update/common/errors.h
@@ -52,17 +52,17 @@
 #define SERVICE_UPDATER_NOT_FIXED_DRIVE 31
 #define SERVICE_COULD_NOT_LOCK_UPDATER 32
 #define SERVICE_INSTALLDIR_ERROR 33
 
 #define NO_INSTALLDIR_ERROR 34
 #define WRITE_ERROR_ACCESS_DENIED 35
 // #define WRITE_ERROR_SHARING_VIOLATION 36 // Replaced with errors 46-48
 #define WRITE_ERROR_CALLBACK_APP 37
-#define UNEXPECTED_BZIP_ERROR 39
+#define UNEXPECTED_XZ_ERROR 39
 #define UNEXPECTED_MAR_ERROR 40
 #define UNEXPECTED_BSPATCH_ERROR 41
 #define UNEXPECTED_FILE_OPERATION_ERROR 42
 // #define FILESYSTEM_MOUNT_READWRITE_ERROR 43 // Removed support for gonk
 #define DELETE_ERROR_EXPECTED_DIR 46
 #define DELETE_ERROR_EXPECTED_FILE 47
 #define RENAME_ERROR_EXPECTED_FILE 48
 
--- a/toolkit/mozapps/update/updater/archivereader.cpp
+++ b/toolkit/mozapps/update/updater/archivereader.cpp
@@ -9,16 +9,18 @@
 #include <fcntl.h>
 #include "bzlib.h"
 #include "archivereader.h"
 #include "errors.h"
 #ifdef XP_WIN
 #include "nsAlgorithm.h" // Needed by nsVersionComparator.cpp
 #include "updatehelper.h"
 #endif
+#define XZ_USE_CRC64
+#include "xz.h"
 
 // These are generated at compile time based on the DER file for the channel
 // being used
 #ifdef MOZ_VERIFY_MAR_SIGNATURE
 #ifdef TEST_UPDATER
 #include "../xpcshellCert.h"
 #else
 #include "primaryCert.h"
@@ -31,20 +33,20 @@
 #undef UPDATER_NO_STRING_GLUE_STL
 
 #if defined(XP_UNIX)
 # include <sys/types.h>
 #elif defined(XP_WIN)
 # include <io.h>
 #endif
 
-static int inbuf_size  = 262144;
-static int outbuf_size = 262144;
-static char *inbuf  = nullptr;
-static char *outbuf = nullptr;
+static size_t inbuf_size  = 262144;
+static size_t outbuf_size = 262144;
+static uint8_t *inbuf  = nullptr;
+static uint8_t *outbuf = nullptr;
 
 /**
  * Performs a verification on the opened MAR file with the passed in
  * certificate name ID and type ID.
  *
  * @param  archive   The MAR file to verify the signature on.
  * @param  certData  The certificate data.
  * @return OK on success, CERT_VERIFY_ERROR on failure.
@@ -177,45 +179,48 @@ ArchiveReader::VerifyProductInformation(
 
 int
 ArchiveReader::Open(const NS_tchar *path)
 {
   if (mArchive)
     Close();
 
   if (!inbuf) {
-    inbuf = (char *)malloc(inbuf_size);
+    inbuf = (uint8_t *)malloc(inbuf_size);
     if (!inbuf) {
       // Try again with a smaller buffer.
       inbuf_size = 1024;
-      inbuf = (char *)malloc(inbuf_size);
+      inbuf = (uint8_t *)malloc(inbuf_size);
       if (!inbuf)
         return ARCHIVE_READER_MEM_ERROR;
     }
   }
 
   if (!outbuf) {
-    outbuf = (char *)malloc(outbuf_size);
+    outbuf = (uint8_t *)malloc(outbuf_size);
     if (!outbuf) {
       // Try again with a smaller buffer.
       outbuf_size = 1024;
-      outbuf = (char *)malloc(outbuf_size);
+      outbuf = (uint8_t *)malloc(outbuf_size);
       if (!outbuf)
         return ARCHIVE_READER_MEM_ERROR;
     }
   }
 
 #ifdef XP_WIN
   mArchive = mar_wopen(path);
 #else
   mArchive = mar_open(path);
 #endif
   if (!mArchive)
     return READ_ERROR;
 
+  xz_crc32_init();
+  xz_crc64_init();
+
   return OK;
 }
 
 void
 ArchiveReader::Close()
 {
   if (mArchive) {
     mar_close(mArchive);
@@ -268,57 +273,77 @@ ArchiveReader::ExtractFileToStream(const
   return ExtractItemToStream(item, fp);
 }
 
 int
 ArchiveReader::ExtractItemToStream(const MarItem *item, FILE *fp)
 {
   /* decompress the data chunk by chunk */
 
-  bz_stream strm;
-  int offset, inlen, outlen, ret = OK;
+  int offset, inlen, ret = OK;
+  struct xz_buf strm = { 0 };
+  enum xz_ret xz_rv = XZ_OK;
 
-  memset(&strm, 0, sizeof(strm));
-  if (BZ2_bzDecompressInit(&strm, 0, 0) != BZ_OK)
-    return UNEXPECTED_BZIP_ERROR;
+  struct xz_dec * dec = xz_dec_init(XZ_DYNALLOC, 64 * 1024 * 1024);
+  if (!dec) {
+    return UNEXPECTED_XZ_ERROR;
+  }
+
+  strm.in = inbuf;
+  strm.in_pos = 0;
+  strm.in_size = 0;
+  strm.out = outbuf;
+  strm.out_pos = 0;
+  strm.out_size = outbuf_size;
 
   offset = 0;
   for (;;) {
     if (!item->length) {
       ret = UNEXPECTED_MAR_ERROR;
       break;
     }
 
-    if (offset < (int) item->length && strm.avail_in == 0) {
+    if (offset < (int) item->length && strm.in_pos == strm.in_size) {
       inlen = mar_read(mArchive, item, offset, inbuf, inbuf_size);
-      if (inlen <= 0)
-        return READ_ERROR;
+      if (inlen <= 0) {
+        ret = READ_ERROR;
+        break;
+      }
       offset += inlen;
-      strm.next_in = inbuf;
-      strm.avail_in = inlen;
+      strm.in_size = inlen;
+      strm.in_pos = 0;
     }
 
-    strm.next_out = outbuf;
-    strm.avail_out = outbuf_size;
+    xz_rv = xz_dec_run(dec, &strm);
+
+    if (strm.out_pos == outbuf_size) {
+      if (fwrite(outbuf, 1, strm.out_pos, fp) != strm.out_pos) {
+        ret = WRITE_ERROR_EXTRACT;
+        break;
+      }
 
-    ret = BZ2_bzDecompress(&strm);
-    if (ret != BZ_OK && ret != BZ_STREAM_END) {
-      ret = UNEXPECTED_BZIP_ERROR;
+      strm.out_pos = 0;
+    }
+
+    if (xz_rv == XZ_OK) {
+      // There is still more data to decompress.
+      continue;
+    }
+
+    // The return value of xz_dec_run is not XZ_OK and if it isn't XZ_STREAM_END
+    // an error has occured.
+    if (xz_rv != XZ_STREAM_END) {
+      ret = UNEXPECTED_XZ_ERROR;
       break;
     }
 
-    outlen = outbuf_size - strm.avail_out;
-    if (outlen) {
-      if (fwrite(outbuf, outlen, 1, fp) != 1) {
-        ret = WRITE_ERROR_EXTRACT;
-        break;
-      }
+    // Write out the remainder of the decompressed data. In the case of
+    // strm.out_pos == 0 this is needed to create empty files included in the
+    // mar file.
+    if (fwrite(outbuf, 1, strm.out_pos, fp) != strm.out_pos) {
+      ret = WRITE_ERROR_EXTRACT;
     }
-
-    if (ret == BZ_STREAM_END) {
-      ret = OK;
-      break;
-    }
+    break;
   }
 
-  BZ2_bzDecompressEnd(&strm);
+  xz_dec_end(dec);
   return ret;
 }
--- a/toolkit/mozapps/update/updater/updater-common.build
+++ b/toolkit/mozapps/update/updater/updater-common.build
@@ -53,16 +53,17 @@ elif CONFIG['OS_ARCH'] == 'Linux' and CO
     OS_LIBS += CONFIG['NSPR_LIBS']
 else:
     USE_LIBS += [
         'updatecommon',
     ]
 
 USE_LIBS += [
     'mar',
+    'xz-embedded',
 ]
 
 if CONFIG['MOZ_SYSTEM_BZ2']:
     OS_LIBS += CONFIG['MOZ_BZ2_LIBS']
 else:
     USE_LIBS += [
         'bz2',
     ]