Bug 1385780 - Update mar file generation scripts for lzma. r=bhearsum, r=rail, a=app_update_lzma
authorRobert Strong <robert.bugzilla@gmail.com>
Mon, 31 Jul 2017 10:34:22 -0700
changeset 420672 f632eede0f19b8d81ee2bb0de48c2cc996d5906f
parent 420671 1be0c1da06076f85c69cd8a9d244e0164ec544d9
child 420673 bbd46c077793343b5d14f30ad8cafe515b831d5d
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbhearsum, rail, app_update_lzma
bugs1385780
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1385780 - Update mar file generation scripts for lzma. r=bhearsum, r=rail, a=app_update_lzma Update scripts to support both lzma and bzip2 Update unused python script to support lzma. This also adds python 3.0 support to the script while still supporting pythin 2.7 Update test scripts to support lzma
python/mozbuild/mozbuild/repackaging/mar.py
tools/update-packaging/common.sh
tools/update-packaging/make_full_update.sh
tools/update-packaging/make_incremental_update.sh
tools/update-packaging/make_incremental_updates.py
tools/update-packaging/test/catmanifest.sh
tools/update-packaging/test/common.sh
tools/update-packaging/test/diffmar.sh
tools/update-packaging/test/from-mac/Contents/Resources/chrome.manifest
tools/update-packaging/test/from/chrome.manifest
tools/update-packaging/test/make_full_update.sh
tools/update-packaging/test/to-mac/Contents/Resources/chrome.manifest
tools/update-packaging/test/to/chrome.manifest
tools/update-packaging/unwrap_full_update.pl
--- a/python/mozbuild/mozbuild/repackaging/mar.py
+++ b/python/mozbuild/mozbuild/repackaging/mar.py
@@ -42,16 +42,22 @@ def repackage_mar(topsrcdir, package, ma
                             "(eg: 'firefox'), not: %s" % toplevel_dirs)
         ffxdir = mozpath.join(tmpdir, toplevel_dirs.pop())
 
         make_full_update = mozpath.join(topsrcdir, 'tools/update-packaging/make_full_update.sh')
 
         env = os.environ.copy()
         env['MOZ_FULL_PRODUCT_VERSION'] = get_application_ini_value(tmpdir, 'App', 'Version')
         env['MAR'] = mozpath.normpath(mar)
+        # The Windows build systems have xz installed but it isn't in the path
+        # like it is on Linux and Mac OS X so just use the XZ env var so the mar
+        # generation scripts can find it.
+        xz_path = mozpath.join(topsrcdir, 'xz/xz.exe')
+        if os.path.exists(xz_path):
+            env['XZ'] = mozpath.normpath(xz_path)
 
         cmd = [make_full_update, output, ffxdir]
         if sys.platform == 'win32':
             # make_full_update.sh is a bash script, and Windows needs to
             # explicitly call out the shell to execute the script from Python.
             cmd.insert(0, env['MOZILLABUILD'] + '/msys/bin/bash.exe')
         subprocess.check_call(cmd, env=env)
 
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -6,18 +6,23 @@
 #
 # Code shared by update packaging scripts.
 # Author: Darin Fisher
 #
 
 # -----------------------------------------------------------------------------
 # By default just assume that these tools exist on our path
 MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
+if [[ -z "${MAR_OLD_FORMAT}" ]]; then
+  XZ=${XZ:-xz}
+else
+  MAR_OLD_FORMAT=1
+  BZIP2=${BZIP2:-bzip2}
+fi
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
   echo "$*" 1>&2
 }
 
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -88,29 +88,38 @@ for ((i=0; $i<$num_files; i=$i+1)); do
       make_add_instruction "$f" "$updatemanifestv2" "" 1
     fi
   else
     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
   fi
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
-  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  if [[ -n $MAR_OLD_FORMAT ]]; then
+    $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  else
+    $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
+  fi
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
 # Append remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
 append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+if [[ -n $MAR_OLD_FORMAT ]]; then
+  $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
+  $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+else
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
+fi
 
 mar_command="$MAR"
 if [[ -n $MOZ_PRODUCT_VERSION ]]
 then
   mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
 fi
 mar_command="$mar_command -C \"$workdir\" -c output.mar"
 eval "$mar_command $targetfiles"
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -164,27 +164,35 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
   f="${oldfiles[$i]}"
 
   # If this file exists in the new directory as well, then check if it differs.
   if [ -f "$newdir/$f" ]; then
 
     if check_for_add_if_not_update "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      if [[ -n $MAR_OLD_FORMAT ]]; then
+        $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      else
+        $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
+      fi
       copy_perm "$newdir/$f" "$workdir/$f"
       make_add_if_not_instruction "$f" "$updatemanifestv3"
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if check_for_forced_update "$requested_forced_updates" "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      if [[ -n $MAR_OLD_FORMAT ]]; then
+        $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      else
+        $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
+      fi
       copy_perm "$newdir/$f" "$workdir/$f"
       make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
       # Compute both the compressed binary diff and the compressed file, and
@@ -197,36 +205,59 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
       # compute avoidance
       #
       # An example of MBSDIFF_HOOK env variable could look like this:
       # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
       # where myscript.sh has the following usage:
       # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
       #   PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
       #
-      # Note: patches are bzipped stashed in funsize to gain more speed
+      # Note: patches are bzipped or xz stashed in funsize to gain more speed
 
       # if service is not enabled then default to old behavior
       if [ -z "$MBSDIFF_HOOK" ]; then
         $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-        $BZIP2 -z9 "$workdir/$f.patch"
+        if [[ -n $MAR_OLD_FORMAT ]]; then
+          $BZIP2 -z9 "$workdir/$f.patch"
+        else
+          $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
+        fi
       else
         # if service enabled then check patch existence for retrieval
-        if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
-          notice "file \"$f\" found in funsize, diffing skipped"
+        if [[ -n $MAR_OLD_FORMAT ]]; then
+          if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
+            notice "file \"$f\" found in funsize, diffing skipped"
+          else
+            # if not found already - compute it and cache it for future use
+            $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
+            $BZIP2 -z9 "$workdir/$f.patch"
+            $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
+          fi
         else
-          # if not found already - compute it and cache it for future use
-          $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-          $BZIP2 -z9 "$workdir/$f.patch"
-          $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
+          if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
+            notice "file \"$f\" found in funsize, diffing skipped"
+          else
+            # if not found already - compute it and cache it for future use
+            $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
+            $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
+            $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
+          fi
         fi
       fi
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      if [[ -n $MAR_OLD_FORMAT ]]; then
+        $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      else
+        $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
+      fi
       copy_perm "$newdir/$f" "$workdir/$f"
-      patchfile="$workdir/$f.patch.bz2"
+      if [[ -n $MAR_OLD_FORMAT ]]; then
+        patchfile="$workdir/$f.patch.bz2"
+      else
+        patchfile="$workdir/$f.patch.xz"
+      fi
       patchsize=$(get_file_size "$patchfile")
       fullsize=$(get_file_size "$workdir/$f")
 
       if [ $patchsize -lt $fullsize ]; then
         make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
         mv -f "$patchfile" "$workdir/$f.patch"
         rm -f "$workdir/$f"
         archivefiles="$archivefiles \"$f.patch\""
@@ -257,17 +288,21 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); d
     if [ "$f" = "${oldfiles[j]}" ]; then
       continue 2
     fi
   done
 
   dir=$(dirname "$workdir/$f")
   mkdir -p "$dir"
 
-  $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+  if [[ -n $MAR_OLD_FORMAT ]]; then
+    $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+  else
+    $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
+  fi
   copy_perm "$newdir/$f" "$workdir/$f"
 
   if check_for_add_if_not_update "$f"; then
     make_add_if_not_instruction "$f" "$updatemanifestv3"
   else
     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
   fi
 
@@ -298,18 +333,23 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
   # If this dir doesn't exist in the new directory remove it.
   if [ ! -d "$newdir/$f" ]; then
     notice "      rmdir $f/"
     echo "rmdir \"$f/\"" >> $updatemanifestv2
     echo "rmdir \"$f/\"" >> $updatemanifestv3
   fi
 done
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+if [[ -n $MAR_OLD_FORMAT ]]; then
+  $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
+  $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+else
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
+fi
 
 mar_command="$MAR"
 if [[ -n $MOZ_PRODUCT_VERSION ]]
 then
   mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
 fi
 if [[ -n $MOZ_CHANNEL_ID ]]
 then
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -1,25 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import shutil
-import sha
+import hashlib
 from os.path import join, getsize
 from stat import *
 import re
 import sys
 import getopt
 import time
 import datetime
-import bz2
 import string
 import tempfile
+import io
 
 class PatchInfo:
     """ Represents the meta-data associated with a patch
         work_dir = working dir where files are stored for this patch
         archive_files = list of files to include in this patch
         manifestv2 = set of manifest version 2 patch instructions
         manifestv3 = set of manifest version 3 patch instructions
         file_exclusion_list =
@@ -41,94 +41,94 @@ class PatchInfo:
             add-if instruction that will add the file if the parent directory
             of the file exists.  This was ported from
             mozilla/tools/update-packaging/common.sh's make_add_instruction.
         """
         m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
         if m:
             # Directory immediately following extensions is used for the test
             testdir = m.group(1)
-            print '     add-if "'+testdir+'" "'+filename+'"'
+            print('     add-if "'+testdir+'" "'+filename+'"')
             self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
             self.manifestv3.append('add-if "'+testdir+'" "'+filename+'"')
         else:
-            print '        add "'+filename+'"'
+            print('        add "'+filename+'"')
             self.manifestv2.append('add "'+filename+'"')
             self.manifestv3.append('add "'+filename+'"')
 
     def append_add_if_not_instruction(self, filename):
         """ Appends an add-if-not instruction to the version 3 manifest for this patch.
             This was ported from mozilla/tools/update-packaging/common.sh's
             make_add_if_not_instruction.
         """
-        print ' add-if-not "'+filename+'" "'+filename+'"'
+        print(' add-if-not "'+filename+'" "'+filename+'"')
         self.manifestv3.append('add-if-not "'+filename+'" "'+filename+'"')
 
     def append_patch_instruction(self, filename, patchname):
         """ Appends a patch instruction for this patch.
 
             filename = file to patch
             patchname = patchfile to apply to file
 
             if filename starts with distribution/extensions/.*/ this will add a
             patch-if instruction that will patch the file if the parent
             directory of the file exists. This was ported from
             mozilla/tools/update-packaging/common.sh's make_patch_instruction.
         """
         m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
         if m:
             testdir = m.group(1)
-            print '   patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"'
+            print('   patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
             self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
             self.manifestv3.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
         else:
-            print '      patch "'+patchname+'" "'+filename+'"'
+            print('      patch "'+patchname+'" "'+filename+'"')
             self.manifestv2.append('patch "'+patchname+'" "'+filename+'"')
             self.manifestv3.append('patch "'+patchname+'" "'+filename+'"')
 
     def append_remove_instruction(self, filename):
         """ Appends an remove instruction for this patch.
             This was ported from
             mozilla/tools/update-packaging/common.sh/make_remove_instruction
         """
         if filename.endswith("/"):
-            print '      rmdir "'+filename+'"'
+            print('      rmdir "'+filename+'"')
             self.manifestv2.append('rmdir "'+filename+'"')
             self.manifestv3.append('rmdir "'+filename+'"')
         elif filename.endswith("/*"):
             filename = filename[:-1]
-            print '    rmrfdir "'+filename+'"'
+            print('    rmrfdir "'+filename+'"')
             self.manifestv2.append('rmrfdir "'+filename+'"')
             self.manifestv3.append('rmrfdir "'+filename+'"')
         else:
-            print '     remove "'+filename+'"'
+            print('     remove "'+filename+'"')
             self.manifestv2.append('remove "'+filename+'"')
             self.manifestv3.append('remove "'+filename+'"')
 
     def create_manifest_files(self):
         """ Create the v2 manifest file in the root of the work_dir """
         manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
         manifest_file = open(manifest_file_path, "wb")
-        manifest_file.writelines("type \"partial\"\n")
-        manifest_file.writelines(string.join(self.manifestv2, '\n'))
-        manifest_file.writelines("\n")
+        manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
+        manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv2).encode('ascii')))
+        manifest_file.writelines(io.BytesIO(b"\n"))
         manifest_file.close()
 
-        bzip_file(manifest_file_path)
+        xz_file(manifest_file_path)
         self.archive_files.append('"updatev2.manifest"')
 
         """ Create the v3 manifest file in the root of the work_dir """
         manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
         manifest_file = open(manifest_file_path, "wb")
-        manifest_file.writelines("type \"partial\"\n")
-        manifest_file.writelines(string.join(self.manifestv3, '\n'))
-        manifest_file.writelines("\n")
+        manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
+        manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv3).encode('ascii')))
+        manifest_file.writelines(io.BytesIO(b"\n"))
         manifest_file.close()
 
-        bzip_file(manifest_file_path)
+        xz_file(manifest_file_path)
         self.archive_files.append('"updatev3.manifest"')
 
     def build_marfile_entry_hash(self, root_path):
         """ Iterates through the root_path, creating a MarFileEntry for each file
             and directory in that path.  Excludes any filenames in the file_exclusion_list
         """
         mar_entry_hash = {}
         filename_set = set()
@@ -171,89 +171,89 @@ class MarFileEntry:
         self.abs_path=os.path.join(root,name)
         self.sha_cache=None
 
     def __str__(self):
         return 'Name: %s FullPath: %s' %(self.name,self.abs_path)
 
     def calc_file_sha_digest(self, filename):
         """ Returns sha digest of given filename"""
-        file_content = open(filename, 'r').read()
-        return sha.new(file_content).digest()
+        file_content = open(filename, 'rb').read()
+        return hashlib.sha1(file_content).digest()
 
     def sha(self):
         """ Returns sha digest of file repreesnted by this _marfile_entry"""
         if not self.sha_cache:
             self.sha_cache=self.calc_file_sha_digest(self.abs_path)
         return self.sha_cache
 
 def exec_shell_cmd(cmd):
     """Execs shell cmd and raises an exception if the cmd fails"""
     if (os.system(cmd)):
-        raise Exception, "cmd failed "+cmd
+        raise Exception("cmd failed "+cmd)
 
 
 def copy_file(src_file_abs_path, dst_file_abs_path):
     """ Copies src to dst creating any parent dirs required in dst first """
     dst_file_dir=os.path.dirname(dst_file_abs_path)
     if not os.path.exists(dst_file_dir):
          os.makedirs(dst_file_dir)
     # Copy the file over
     shutil.copy2(src_file_abs_path, dst_file_abs_path)
 
-def bzip_file(filename):
-    """ Bzip's the file in place.  The original file is replaced with a bzip'd version of itself
+def xz_file(filename):
+    """ XZ compresses the file in place.  The original file is replaced with the xz compressed version of itself
         assumes the path is absolute"""
-    exec_shell_cmd('bzip2 -z9 "' + filename+'"')
-    os.rename(filename+".bz2",filename)
+    exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename+'"')
+    os.rename(filename+".xz",filename)
 
-def bunzip_file(filename):
-    """ Bzip's the file in palce.   The original file is replaced with a bunzip'd version of itself.
-        doesn't matter if the filename ends in .bz2 or not"""
-    if not filename.endswith(".bz2"):
-        os.rename(filename, filename+".bz2")
-        filename=filename+".bz2"
-    exec_shell_cmd('bzip2 -d "' + filename+'"')
+def xzunzip_file(filename):
+    """ xz decompresses the file in palce.  The original file is replaced with a xz decompressed version of itself.
+        doesn't matter if the filename ends in .xz or not"""
+    if not filename.endswith(".xz"):
+        os.rename(filename, filename+".xz")
+        filename=filename+".xz"
+    exec_shell_cmd('xz -d "' + filename+'"')
 
 
 def extract_mar(filename, work_dir):
     """ Extracts the marfile intot he work_dir
         assumes work_dir already exists otherwise will throw osError"""
-    print "Extracting "+filename+" to "+work_dir
+    print("Extracting "+filename+" to "+work_dir)
     saved_path = os.getcwd()
     try:
         os.chdir(work_dir)
         exec_shell_cmd("mar -x "+filename)
     finally:
         os.chdir(saved_path)
 
 def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info):
     """ Creates the partial patch file and manifest entry for the pair of files passed in
     """
     if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas:
-        print 'diffing "'+from_marfile_entry.name+'\"'
+        print('diffing "'+from_marfile_entry.name+'\"')
         #bunzip to/from
-        bunzip_file(from_marfile_entry.abs_path)
-        bunzip_file(to_marfile_entry.abs_path)
+        xzunzip_file(from_marfile_entry.abs_path)
+        xzunzip_file(to_marfile_entry.abs_path)
 
         # The patch file will be created in the working directory with the
         # name of the file in the mar + .patch
         patch_file_abs_path = os.path.join(patch_info.work_dir,from_marfile_entry.name+".patch")
         patch_file_dir=os.path.dirname(patch_file_abs_path)
         if not os.path.exists(patch_file_dir):
             os.makedirs(patch_file_dir)
 
-        # Create bzip'd patch file
+        # Create xz compressed patch file
         exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
-        bzip_file(patch_file_abs_path)
+        xz_file(patch_file_abs_path)
 
-        # Create bzip's full file
+        # Create xz compressed full file
         full_file_abs_path =  os.path.join(patch_info.work_dir, to_marfile_entry.name)
         shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
-        bzip_file(full_file_abs_path)
+        xz_file(full_file_abs_path)
 
         if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
             # Patch is smaller than file.  Remove the file and add patch to manifest
             os.remove(full_file_abs_path)
             file_in_manifest_name = from_marfile_entry.name+".patch"
             file_in_manifest_abspath = patch_file_abs_path
             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
         else:
@@ -299,23 +299,27 @@ def process_explicit_remove_files(dir_pa
     found in that file to the patch_info"""
 
     # Windows and linux have this file at the root of the dir
     list_file_path = os.path.join(dir_path, "removed-files")
     if not os.path.exists(list_file_path):
         list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files")
 
     if (os.path.exists(list_file_path)):
-        list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
+        fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
+        os.close(fd)
+        exec_shell_cmd('xz -k -d --stdout "' + list_file_path + '" > "'+tmppath+'"')
+        list_file = open(tmppath)
 
         lines = []
         for line in list_file:
             lines.append(line.strip())
+
         list_file.close()
-
+        os.remove(tmppath)
         lines.sort(reverse=True)
         for line in lines:
             # Exclude any blank and comment lines.
             if line and not line.startswith("#"):
                 # Python on windows uses \ for path separators and the update
                 # manifests expects / for path separators on all platforms.
                 line = line.replace("\\", "/")
                 patch_info.append_remove_instruction(line)
@@ -334,49 +338,49 @@ def create_partial_patch(from_dir_path, 
     if "precomplete" in to_file_set:
         forced_list.append("precomplete")
     elif "Contents/Resources/precomplete" in to_file_set:
         forced_list.append("Contents/Resources/precomplete")
     # The check with \ file separators allows tests for Mac to run on Windows
     elif "Contents\Resources\precomplete" in to_file_set:
         forced_list.append("Contents\Resources\precomplete")
     else:
-        raise Exception, "missing precomplete file in: "+to_dir_path
+        raise Exception("missing precomplete file in: "+to_dir_path)
 
     if "removed-files" in to_file_set:
         forced_list.append("removed-files")
     elif "Contents/Resources/removed-files" in to_file_set:
         forced_list.append("Contents/Resources/removed-files")
     # The check with \ file separators allows tests for Mac to run on Windows
     elif "Contents\Resources\\removed-files" in to_file_set:
         forced_list.append("Contents\Resources\\removed-files")
     else:
-        raise Exception, "missing removed-files file in: "+to_dir_path
+        raise Exception("missing removed-files file in: "+to_dir_path)
 
     if "chrome.manifest" in to_file_set:
         forced_list.append("chrome.manifest")
     elif "Contents/Resources/chrome.manifest" in to_file_set:
         forced_list.append("Contents/Resources/chrome.manifest")
     # The check with \ file separators allows tests for Mac to run on Windows
     elif "Contents\Resources\\chrome.manifest" in to_file_set:
         forced_list.append("Contents\Resources\\chrome.manifest")
     else:
-        raise Exception, "missing chrome.manifest file in: "+to_dir_path
+        raise Exception("missing chrome.manifest file in: "+to_dir_path)
 
     # Files which exist in both sets need to be patched
     patch_filenames = list(from_file_set.intersection(to_file_set))
     patch_filenames.sort(reverse=True)
     for filename in patch_filenames:
         from_marfile_entry = from_dir_hash[filename]
         to_marfile_entry = to_dir_hash[filename]
         if os.path.basename(filename) in add_if_not_list:
             # This filename is in the add if not list, explicitly add-if-not
             create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
         elif filename in forced_list:
-            print 'Forcing "'+filename+'"'
+            print('Forcing "'+filename+'"')
             # This filename is in the forced list, explicitly add
             create_add_patch_for_file(to_dir_hash[filename], patch_info)
         else:
           if from_marfile_entry.sha() != to_marfile_entry.sha():
               # Not the same - calculate a patch
               create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info)
 
     # files in to_dir not in from_dir need to added
@@ -401,77 +405,84 @@ def create_partial_patch(from_dir_path, 
     remove_dirnames.sort(reverse=True)
     for dirname in remove_dirnames:
         patch_info.append_remove_instruction(from_dir_hash[dirname].name)
 
     # Construct the Manifest files
     patch_info.create_manifest_files()
 
     # And construct the mar
-    mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
+    mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+' '.join(patch_info.archive_files)
     exec_shell_cmd(mar_cmd)
 
     # Copy mar to final destination
     patch_file_dir = os.path.split(patch_filename)[0]
     if not os.path.exists(patch_file_dir):
         os.makedirs(patch_file_dir)
     shutil.copy2(os.path.join(patch_info.work_dir,"output.mar"), patch_filename)
 
     return patch_filename
 
 def usage():
-    print "-h for help"
-    print "-f for patchlist_file"
+    print("-h for help")
+    print("-f for patchlist_file")
 
 def get_buildid(work_dir):
     """ extracts buildid from MAR
     """
     ini = '%s/application.ini' % work_dir
     if not os.path.exists(ini):
         ini = '%s/Contents/Resources/application.ini' % work_dir
         if not os.path.exists(ini):
-            print 'WARNING: application.ini not found, cannot find build ID'
+            print('WARNING: application.ini not found, cannot find build ID')
             return ''
 
-    file = bz2.BZ2File(ini)
+    fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
+    os.close(fd)
+    exec_shell_cmd('xz -k -d --stdout "' + ini + '" > "'+tmppath+'"')
+    file = open(tmppath)
     for line in file:
         if line.find('BuildID') == 0:
+            file.close()
+            os.remove(tmppath)
             return line.strip().split('=')[1]
-    print 'WARNING: cannot find build ID in application.ini'
+    print('WARNING: cannot find build ID in application.ini')
+    file.close()
+    os.remove(tmppath)
     return ''
 
 def decode_filename(filepath):
     """ Breaks filename/dir structure into component parts based on regex
         for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
         Or linux-i686/en-US/firefox-3.0b3.complete.mar
         Returns dict with keys product, version, locale, platform, type
     """
     try:
       m = re.search(
         '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',
       os.path.basename(filepath))
       return m.groupdict()
-    except Exception, exc:
+    except Exception(exc):
       try:
         m = re.search(
           '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',
         filepath)
         return m.groupdict()
       except:
         raise Exception("could not parse filepath %s: %s" % (filepath, exc))
 
 def create_partial_patches(patches):
     """ Given the patches generates a set of partial patches"""
     shas = {}
 
     work_dir_root = None
     metadata = []
     try:
         work_dir_root = tempfile.mkdtemp('-fastmode', 'tmp', os.getcwd())
-        print "Building patches using work dir: %s" % (work_dir_root)
+        print("Building patches using work dir: %s" % (work_dir_root))
 
         # Iterate through every patch set in the patch file
         patch_num = 1
         for patch in patches:
             startTime = time.time()
 
             from_filename,to_filename,patch_filename,forced_updates = patch.split(",")
             from_filename,to_filename,patch_filename = os.path.abspath(from_filename),os.path.abspath(to_filename),os.path.abspath(patch_filename)
@@ -481,33 +492,33 @@ def create_partial_patches(patches):
             os.mkdir(work_dir)
 
             # Extract from mar into from dir
             work_dir_from =  os.path.join(work_dir,"from");
             os.mkdir(work_dir_from)
             extract_mar(from_filename,work_dir_from)
             from_decoded = decode_filename(from_filename)
             from_buildid = get_buildid(work_dir_from)
-            from_shasum = sha.sha(open(from_filename).read()).hexdigest()
+            from_shasum = hashlib.sha1(open(from_filename, "rb").read()).hexdigest()
             from_size = str(os.path.getsize(to_filename))
 
             # Extract to mar into to dir
             work_dir_to =  os.path.join(work_dir,"to")
             os.mkdir(work_dir_to)
             extract_mar(to_filename, work_dir_to)
             to_decoded = decode_filename(from_filename)
             to_buildid = get_buildid(work_dir_to)
-            to_shasum = sha.sha(open(to_filename).read()).hexdigest()
+            to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest()
             to_size = str(os.path.getsize(to_filename))
 
             mar_extract_time = time.time()
 
             partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest'],[]),forced_updates,['channel-prefs.js','update-settings.ini'])
             partial_buildid = to_buildid
-            partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
+            partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest()
             partial_size = str(os.path.getsize(partial_filename))
 
             metadata.append({
              'to_filename': os.path.basename(to_filename),
              'from_filename': os.path.basename(from_filename),
              'partial_filename': os.path.basename(partial_filename),
              'to_buildid':to_buildid,
              'from_buildid':from_buildid,
@@ -517,17 +528,17 @@ def create_partial_patches(patches):
              'to_size':to_size,
              'from_size':from_size,
              'partial_size':partial_size,
              'to_version':to_decoded['version'],
              'from_version':from_decoded['version'],
              'locale':from_decoded['locale'],
              'platform':from_decoded['platform'],
             })
-            print "done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),str(len(patches)),mar_extract_time-startTime,time.time()-mar_extract_time,time.time()-startTime)
+            print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),str(len(patches)),mar_extract_time-startTime,time.time()-mar_extract_time,time.time()-startTime))
             patch_num += 1
         return metadata
     finally:
         # If we fail or get a ctrl-c during run be sure to clean up temp dir
         if (work_dir_root and os.path.exists(work_dir_root)):
             shutil.rmtree(work_dir_root)
 
 def main(argv):
--- a/tools/update-packaging/test/catmanifest.sh
+++ b/tools/update-packaging/test/catmanifest.sh
@@ -4,11 +4,11 @@
 mar="$1"
 workdir="/tmp/catmanifest"
 
 rm -rf "$workdir"
 mkdir -p "$workdir"
 cp "$1" "$workdir"
 cd "$workdir"
 mar -x "$1"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
+mv updatev2.manifest updatev2.manifest.xz
+xz -d updatev2.manifest.xz
 cat updatev2.manifest
--- a/tools/update-packaging/test/common.sh
+++ b/tools/update-packaging/test/common.sh
@@ -7,17 +7,17 @@
 # Code shared by update packaging scripts.
 # Author: Darin Fisher
 #
 # In here to use the local common.sh to allow the full mars to have unfiltered files
 
 # -----------------------------------------------------------------------------
 # By default just assume that these tools exist on our path
 MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
+XZ=${XZ:-xz}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
   echo "$*" 1>&2
 }
--- a/tools/update-packaging/test/diffmar.sh
+++ b/tools/update-packaging/test/diffmar.sh
@@ -25,27 +25,27 @@ mkdir -p "$fromdir"
 mkdir -p "$todir"
 
 cp "$1" "$fromdir"
 cp "$2" "$todir"
 
 cd "$fromdir"
 mar -x "$1"
 rm "$1"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
-mv updatev3.manifest updatev3.manifest.bz2
-bzip2 -d updatev3.manifest.bz2
+mv updatev2.manifest updatev2.manifest.xz
+xz -d updatev2.manifest.xz
+mv updatev3.manifest updatev3.manifest.xz
+xz -d updatev3.manifest.xz
 ls $lsargs > files.txt
 
 cd "$todir"
 mar -x "$2"
 rm "$2"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
-mv updatev3.manifest updatev3.manifest.bz2
-bzip2 -d updatev3.manifest.bz2
+mv updatev2.manifest updatev2.manifest.xz
+xz -d updatev2.manifest.xz
+mv updatev3.manifest updatev3.manifest.xz
+xz -d updatev3.manifest.xz
 ls $lsargs > files.txt
 
 echo "diffing $fromdir and $todir"
 echo "on linux shell sort and python sort return different results"
 echo "which can cause differences in the manifest files"
 diff -ru "$fromdir" "$todir"
new file mode 100644
new file mode 100644
--- a/tools/update-packaging/test/make_full_update.sh
+++ b/tools/update-packaging/test/make_full_update.sh
@@ -89,29 +89,29 @@ for ((i=0; $i<$num_files; i=$i+1)); do
       make_add_instruction "$f" "$updatemanifestv2" "" 1
     fi
   else
     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
   fi
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
-  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
 # Append remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
 append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
+$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
 
 notice ""
new file mode 100644
new file mode 100644
--- a/tools/update-packaging/unwrap_full_update.pl
+++ b/tools/update-packaging/unwrap_full_update.pl
@@ -8,30 +8,42 @@
 # Author: Benjamin Smedberg
 #
 
 # -----------------------------------------------------------------------------
 # By default just assume that these tools exist on our path
 
 use Getopt::Std;
 
-my ($MAR, $BZIP2, $archive, @marentries, @marfiles);
+my ($MAR, $XZ, $BZIP2, $MAR_OLD_FORMAT, $archive, @marentries, @marfiles);
 
 if (defined($ENV{"MAR"})) {
     $MAR = $ENV{"MAR"};
 }
 else {
     $MAR = "mar";
 }
 
-if (defined($ENV{"BZIP2"})) {
-    $BZIP2 = $ENV{"BZIP2"};
+if (defined($ENV{"MAR_OLD_FORMAT"})) {
+    $MAR_OLD_FORMAT = 1;
+    if (defined($ENV{"BZIP2"})) {
+        $BZIP2 = $ENV{"BZIP2"};
+    }
+    else {
+        $BZIP2 = "bzip2";
+    }
 }
 else {
-    $BZIP2 = "bzip2";
+    $MAR_OLD_FORMAT = 0;
+    if (defined($ENV{"XZ"})) {
+        $XZ = $ENV{"XZ"};
+    }
+    else {
+        $XZ = "xz";
+    }
 }
 
 sub print_usage
 {
     print "Usage: unwrap_full_update.pl [OPTIONS] ARCHIVE\n\n";
     print "The contents of ARCHIVE will be unpacked into the current directory.\n\n";
     print "Options:\n";
     print "  -h show this help text\n";
@@ -54,14 +66,21 @@ shift @marentries;
 
 system("$MAR -x \"$archive\"") == 0 || die "Couldn't run $MAR -x";
 
 foreach (@marentries) {
     tr/\n\r//d;
     my @splits = split(/\t/,$_);
     my $file = $splits[2];
 
-    system("mv \"$file\" \"$file.bz2\"") == 0 ||
-      die "Couldn't mv \"$file\"";
-    system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
-      die "Couldn't decompress \"$file\"";
+    if ($MAR_OLD_FORMAT == 1) {
+      system("mv \"$file\" \"$file.bz2\"") == 0 ||
+        die "Couldn't mv \"$file\"";
+      system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
+        die "Couldn't decompress \"$file\"";
+    }
+    else {
+      system("mv \"$file\" \"$file.xz\"") == 0 ||
+        die "Couldn't mv \"$file\"";
+      system("\"$XZ\" -d \"$file.xz\"") == 0 ||
+        die "Couldn't decompress \"$file\"";
+    }
 }
-