1. mar packaging and build changes - Bug 386760 - directory removal with software update. r=nthomas, r=khuey
authorRobert Strong <robert.bugzilla@gmail.com>
Mon, 11 Apr 2011 21:23:18 -0700
changeset 67955 4ceb3de77444dd4c97b9ec6f21a45785f7793ed5
parent 67954 3ac2e9681421f3348f482832579dbb66be9fc431
child 67956 860d857e899db6f7512a7a873c47391bdfafa16b
push id19456
push userrstrong@mozilla.com
push dateTue, 12 Apr 2011 04:24:46 +0000
treeherdermozilla-central@b4dcf0feefd1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnthomas, khuey
bugs1, 386760
milestone2.2a1pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
1. mar packaging and build changes - Bug 386760 - directory removal with software update. r=nthomas, r=khuey
config/createprecomplete.py
toolkit/mozapps/installer/packager.mk
tools/update-packaging/common.sh
tools/update-packaging/make_full_update.sh
tools/update-packaging/make_incremental_update.sh
tools/update-packaging/make_incremental_updates.py
tools/update-packaging/test/buildrefmars.sh
tools/update-packaging/test/catmanifest.sh
tools/update-packaging/test/common.sh
tools/update-packaging/test/diffmar.sh
tools/update-packaging/test/from/precomplete
tools/update-packaging/test/make_full_update.sh
tools/update-packaging/test/runtests.sh
tools/update-packaging/test/to-mac/precomplete
tools/update-packaging/test/to/precomplete
tools/update-packaging/test_make_incremental_updates.py
new file mode 100644
--- /dev/null
+++ b/config/createprecomplete.py
@@ -0,0 +1,64 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+# Creates the precomplete file containing the remove, remove-cc, and rmdir
+# application update instructions which is used to remove files and directories
+# that are no longer present in a complete update. The current working directory
+# is used for the location to enumerate and to create the precomplete file.
+
+import sys
+import os
+
+def get_build_entries(root_path):
+    """ Iterates through the root_path, creating a list for each file and
+        directory. Excludes any path starting with extensions or distribution.
+    """
+    rel_file_path_set = set()
+    rel_dir_path_set = set()
+    for root, dirs, files in os.walk(root_path):
+        for file_name in files:
+            parent_dir_rel_path = root[len(root_path)+1:]
+            rel_path_file = os.path.join(parent_dir_rel_path, file_name)
+            rel_path_file = rel_path_file.replace("\\", "/")
+            if not (rel_path_file.startswith("distribution/") or
+                    rel_path_file.startswith("extensions/")):
+                rel_file_path_set.add(rel_path_file)
+
+        for dir_name in dirs:
+            parent_dir_rel_path = root[len(root_path)+1:]
+            rel_path_dir = os.path.join(parent_dir_rel_path, dir_name)
+            rel_path_dir = rel_path_dir.replace("\\", "/")+"/"
+            if not (rel_path_dir.startswith("distribution/") or
+                    rel_path_dir.startswith("extensions/")):
+                rel_dir_path_set.add(rel_path_dir)
+
+    rel_file_path_list = list(rel_file_path_set)
+    rel_file_path_list.sort(reverse=True)
+    rel_dir_path_list = list(rel_dir_path_set)
+    rel_dir_path_list.sort(reverse=True)
+
+    return rel_file_path_list, rel_dir_path_list
+
+def generate_precomplete():
+    """ Creates the precomplete file containing the remove, remove-cc, and rmdir
+        application update instructions. The current working directory is used
+        for the location to enumerate and to create the precomplete file.
+    """
+    root_path = os.getcwd()
+    rel_file_path_list, rel_dir_path_list = get_build_entries(root_path)
+    precomplete_file_path = os.path.join(root_path,"precomplete")
+    # open in binary mode to prevent OS specific line endings.
+    precomplete_file = open(precomplete_file_path, "wb")
+    for rel_file_path in rel_file_path_list:
+        if rel_file_path.endswith("channel-prefs.js"):
+            precomplete_file.writelines("remove-cc \""+rel_file_path+"\"\n")
+        else:
+            precomplete_file.writelines("remove \""+rel_file_path+"\"\n")
+
+    for rel_dir_path in rel_dir_path_list:
+        precomplete_file.writelines("rmdir \""+rel_dir_path+"\"\n")
+
+    precomplete_file.close()
+
+if __name__ == "__main__":
+    generate_precomplete()
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -18,16 +18,17 @@
 # Netscape Communications Corporation.
 # Portions created by the Initial Developer are Copyright (C) 1998
 # the Initial Developer. All Rights Reserved.
 #
 # Contributor(s):
 #   Benjamin Smedberg <bsmedberg@covad.net>
 #   Arthur Wiebe <artooro@gmail.com>
 #   Mark Mentovai <mark@moxienet.com>
+#   Robert Strong <robert.bugzilla@gmail.com>
 #
 # Alternatively, the contents of this file may be used under the terms of
 # either of the GNU General Public License Version 2 or later (the "GPL"),
 # or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 # in which case the provisions of the GPL or the LGPL are applicable instead
 # of those above. If you wish to allow use of your version of this file only
 # under the terms of either the GPL or the LGPL, and not to allow others to
 # use your version of this file under the terms of the MPL, indicate your
@@ -373,16 +374,18 @@ INNER_UNMAKE_PACKAGE	= \
 SDK_SUFFIX = .tar.bz2
 SDK = $(MOZ_PKG_APPNAME)-$(MOZ_PKG_VERSION).$(AB_CD).mac-$(TARGET_CPU).sdk$(SDK_SUFFIX)
 ifeq ($(MOZ_APP_NAME),xulrunner)
 SDK = $(SDK_PATH)$(MOZ_APP_NAME)-$(MOZ_PKG_VERSION).$(AB_CD).mac-$(TARGET_CPU).sdk$(SDK_SUFFIX)
 endif
 MAKE_SDK = $(CREATE_FINAL_TAR) - $(MOZ_APP_NAME)-sdk | bzip2 -vf > $(SDK)
 endif
 
+CREATE_PRECOMPLETE = $(PYTHON) $(MOZILLA_DIR)/config/createprecomplete.py
+
 ifdef MOZ_OMNIJAR
 GENERATE_CACHE ?= true
 
 OMNIJAR_FILES	= \
   chrome \
   chrome.manifest \
   components/*.js \
   components/*.xpt \
@@ -414,20 +417,21 @@ PACK_OMNIJAR	= \
   printf "manifest components/binary.manifest\n" > chrome.manifest
 UNPACK_OMNIJAR	= \
   $(OPTIMIZE_JARS_CMD) --deoptimize $(_ABS_DIST)/jarlog/ ./ ./ && \
   unzip -o omni.jar && \
   rm -f components/binary.manifest && \
   sed -e 's/^\#binary-component/binary-component/' components/components.manifest > components.manifest && \
   mv components.manifest components
 
-MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && $(INNER_MAKE_PACKAGE)
+MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && \
+	              (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE)
 UNMAKE_PACKAGE	= $(INNER_UNMAKE_PACKAGE) && (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(UNPACK_OMNIJAR))
 else
-MAKE_PACKAGE	= $(INNER_MAKE_PACKAGE)
+MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE)
 UNMAKE_PACKAGE	= $(INNER_UNMAKE_PACKAGE)
 endif
 
 # dummy macro if we don't have PSM built
 SIGN_NSS		=
 ifneq (1_,$(if $(CROSS_COMPILE),1,0)_$(UNIVERSAL_BINARY))
 ifdef MOZ_PSM
 SIGN_NSS		= @echo signing nss libraries;
@@ -560,16 +564,17 @@ ifndef MOZ_PKG_MANIFEST
 endif
 	@rm -rf $(DEPTH)/installer-stage $(DIST)/xpt
 	@echo "Staging installer files..."
 	@$(NSINSTALL) -D $(DEPTH)/installer-stage/core
 ifdef MOZ_OMNIJAR
 	@(cd $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR))
 endif
 	@cp -av $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH)/. $(DEPTH)/installer-stage/core
+	@(cd $(DEPTH)/installer-stage/core && $(CREATE_PRECOMPLETE))
 ifdef MOZ_OPTIONAL_PKG_LIST
 	@$(NSINSTALL) -D $(DEPTH)/installer-stage/optional
 	$(call PACKAGER_COPY, "$(call core_abspath,$(DIST))",\
 	  "$(call core_abspath,$(DEPTH)/installer-stage/optional)", \
 	  "$(MOZ_PKG_MANIFEST)", "$(PKGCP_OS)", 1, 0, 1 \
 	  $(foreach pkg,$(MOZ_OPTIONAL_PKG_LIST),$(PKG_ARG)) )
 	@cd $(DEPTH)/installer-stage/optional/extensions; find -maxdepth 1 -mindepth 1 -exec rm -r ../../core/extensions/{} \;
 endif
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -9,17 +9,17 @@
 MAR=${MAR:-mar}
 BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
-  echo $* 1>&2
+  echo "$*" 1>&2
 }
 
 get_file_size() {
   info=($(ls -ln "$1"))
   echo ${info[4]}
 }
 
 copy_perm() {
@@ -30,83 +30,141 @@ copy_perm() {
     chmod 0755 "$target"
   else
     chmod 0644 "$target"
   fi
 }
 
 make_add_instruction() {
   f="$1"
+
+  # Used to log to the console
+  if [ $2 ]; then
+    forced=" (forced)"
+  else
+    forced=
+  fi
+
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "     add-if: $f$forced"
     echo "add-if \"$testdir\" \"$f\""
   else
+    notice "        add: $f$forced"
     echo "add \"$f\""
   fi
 }
 
 make_patch_instruction() {
   f="$1"
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   is_search_plugin=$(echo "$f" | grep -c 'searchplugins/.*')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "   patch-if: $f"
     echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
   elif [ $is_search_plugin = "1" ]; then
+    notice "   patch-if: $f"
     echo "patch-if \"$f\" \"$f.patch\" \"$f\""
   else
+    notice "      patch: $f"
     echo "patch \"$f.patch\" \"$f\""
   fi
 }
 
 append_remove_instructions() {
   dir="$1"
+  filev1="$2"
+  filev2="$3"
   if [ -f "$dir/removed-files" ]; then
     prefix=
     listfile="$dir/removed-files"
   elif [ -f "$dir/Contents/MacOS/removed-files" ]; then
     prefix=Contents/MacOS/
     listfile="$dir/Contents/MacOS/removed-files"
   fi
   if [ -n "$listfile" ]; then
     # Map spaces to pipes so that we correctly handle filenames with spaces.
-    files=($(cat "$listfile" | tr " " "|"))  
+    files=($(cat "$listfile" | tr " " "|"  | sort -r))
     num_files=${#files[*]}
     for ((i=0; $i<$num_files; i=$i+1)); do
-      # Trim whitespace (including trailing carriage returns)
-      f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r')
-      # Exclude any blank lines or any lines ending with a slash, which indicate
-      # directories.  The updater doesn't know how to remove entire directories.
+      # Map pipes back to whitespace and remove carriage returns
+      f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
+      # Trim whitespace
+      f=$(echo $f)
+      # Exclude blank lines.
       if [ -n "$f" ]; then
-        if [ $(echo "$f" | grep -c '\/$') = 0 ]; then
-          echo "remove \"$prefix$f\""
-        else
-          notice "ignoring remove instruction for directory: $f"
+        # Exclude comments
+        if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
+          # Normalize the path to the root of the Mac OS X bundle if necessary
+          fixedprefix="$prefix"
+          if [ $prefix ]; then
+            if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then
+              if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then
+                f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::')
+                fixedprefix=""
+              else
+                f=$(echo $f | sed -e 's:^\.\.\/::')
+                fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::')
+              fi
+            fi
+          fi
+          if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
+            notice "      rmdir: $fixedprefix$f"
+            echo "rmdir \"$fixedprefix$f\"" >> $filev2
+          elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
+            # Remove the *
+            f=$(echo "$f" | sed -e 's:\*$::')
+            notice "    rmrfdir: $fixedprefix$f"
+            echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
+          else
+            notice "     remove: $fixedprefix$f"
+            echo "remove \"$fixedprefix$f\"" >> $filev1
+            echo "remove \"$fixedprefix$f\"" >> $filev2
+          fi
         fi
       fi
     done
   fi
 }
 
 # List all files in the current directory, stripping leading "./"
 # Skip the channel-prefs.js file as it should not be included in any
 # generated MAR files (see bug 306077). Pass a variable name and it will be
 # filled as an array.
 list_files() {
   count=0
 
   find . -type f \
     ! -name "channel-prefs.js" \
     ! -name "update.manifest" \
+    ! -name "updatev2.manifest" \
+    ! -name "temp-dirlist" \
     ! -name "temp-filelist" \
     | sed 's/\.\/\(.*\)/\1/' \
-    | sort > "temp-filelist"
+    | sort -r > "temp-filelist"
   while read file; do
     eval "${1}[$count]=\"$file\""
     (( count++ ))
   done < "temp-filelist"
   rm "temp-filelist"
 }
+
+# List all directories in the current directory, stripping leading "./"
+list_dirs() {
+  count=0
+
+  find . -type d \
+    ! -name "." \
+    ! -name ".." \
+    | sed 's/\.\/\(.*\)/\1/' \
+    | sort -r > "temp-dirlist"
+  while read dir; do
+    eval "${1}[$count]=\"$dir\""
+    (( count++ ))
+  done < "temp-dirlist"
+  rm "temp-dirlist"
+}
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -27,59 +27,102 @@ if [ $1 = -h ]; then
   notice ""
   exit 1
 fi
 
 # -----------------------------------------------------------------------------
 
 archive="$1"
 targetdir="$2"
+# Prevent the workdir from being inside the targetdir so it isn't included in
+# the update mar.
+if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
+  # Remove the /
+  targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
+fi
 workdir="$targetdir.work"
-manifest="$workdir/update.manifest"
-targetfiles="update.manifest"
+updatemanifestv1="$workdir/update.manifest"
+updatemanifestv2="$workdir/updatev2.manifest"
+targetfiles="update.manifest updatev2.manifest"
 
 mkdir -p "$workdir"
 
+# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
+# Info.plist so it launches the right architecture, bug 600098
+
 # Generate a list of all files in the target directory.
 pushd "$targetdir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
-# On Mac, force a top-level file so that OS X reloads the Info.plist
-# and launches the right architecture for the OS version, bug 600098
-if [[ -d Contents ]]; then
-  touch force_plist_reload
+if [ ! -f "precomplete" ]; then
+  notice "precomplete file is missing!"
+  exit 1
 fi
 
 list_files files
 
+# Files that should be added on channel change
+ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
+
 popd
 
-> $manifest
+notice ""
+notice "Adding file add instructions to file 'update.manifest'"
+> $updatemanifestv1
 
 num_files=${#files[*]}
 
 for ((i=0; $i<$num_files; i=$i+1)); do
   f="${files[$i]}"
 
-  notice "processing $f"
-
-  make_add_instruction "$f" >> $manifest
+  make_add_instruction "$f" >> $updatemanifestv1
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
   $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
+# Add the type of update to the beginning of and cat the contents of the version
+# 1 update manifest to the version 2 update manifest.
+> $updatemanifestv2
+notice ""
+notice "Adding type instruction to file 'updatev2.manifest'"
+notice "       type: complete"
+echo "type \"complete\"" >> $updatemanifestv2
+
+notice ""
+notice "Adding file ADD instructions for channel change to file 'updatev2.manifest'"
+for f in $ccfiles; do
+  notice "     add-cc: $f"
+  echo "add-cc \"$f\"" >> $updatemanifestv2
+  dir=$(dirname "$f")
+  mkdir -p "$workdir/$dir"
+  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  copy_perm "$targetdir/$f" "$workdir/$f"
+
+  targetfiles="$targetfiles \"$f\""
+done
+
+notice ""
+notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
+cat $updatemanifestv1 >> $updatemanifestv2
+
 # Append remove instructions for any dead files.
-append_remove_instructions "$targetdir" >> $manifest
+notice ""
+notice "Adding file and directory remove instructions from file 'removed-files'"
+append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2"
 
-$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
+$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
+$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
+
+notice ""
+notice "Finished"
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -21,16 +21,21 @@ print_usage() {
 }
 
 check_for_forced_update() {
   force_list="$1"
   forced_file_chk="$2"
 
   local f
 
+  if [ "$forced_file_chk" = "precomplete" ]; then
+    ## "true" *giggle*
+    return 0;
+  fi
+
   if [ "${forced_file_chk##*.}" = "chk" ]
   then
     ## "true" *giggle*
     return 0;
   fi
 
   for f in $force_list; do
     #echo comparing $forced_file_chk to $f
@@ -65,122 +70,200 @@ done
 # -----------------------------------------------------------------------------
 
 let arg_start=$OPTIND-1
 shift $arg_start
 
 archive="$1"
 olddir="$2"
 newdir="$3"
+# Prevent the workdir from being inside the targetdir so it isn't included in
+# the update mar.
+if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
+  # Remove the /
+  newdir=$(echo "$newdir" | sed -e 's:\/$::')
+fi
 workdir="$newdir.work"
-manifest="$workdir/update.manifest"
-archivefiles="update.manifest"
+updatemanifestv1="$workdir/update.manifest"
+updatemanifestv2="$workdir/updatev2.manifest"
+archivefiles="update.manifest updatev2.manifest"
 
 mkdir -p "$workdir"
 
+# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
+# Info.plist so it launches the right architecture, bug 600098
+
 # Generate a list of all files in the target directory.
 pushd "$olddir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
 list_files oldfiles
+list_dirs olddirs
 
 popd
 
 pushd "$newdir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
+if [ ! -f "precomplete" ]; then
+  notice "precomplete file is missing!"
+  exit 1
+fi
+
+list_dirs newdirs
 list_files newfiles
 
+# Files that should be added on channel change
+ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
+
 popd
 
-> $manifest
+notice ""
+notice "Adding file patch and add instructions to file 'update.manifest'"
+> $updatemanifestv1
 
 num_oldfiles=${#oldfiles[*]}
+remove_array=
+num_removes=0
 
 for ((i=0; $i<$num_oldfiles; i=$i+1)); do
   f="${oldfiles[$i]}"
 
   # This file is created by Talkback, so we can ignore it
   if [ "$f" = "readme.txt" ]; then
     continue 1
   fi
 
+  # removed-files is excluded by make_incremental_updates.py so it is excluded
+  # here for consistency.
+  if [ `basename $f` = "removed-files" ]; then
+    continue 1
+  fi
+
   # If this file exists in the new directory as well, then check if it differs.
   if [ -f "$newdir/$f" ]; then
 
     if check_for_forced_update "$requested_forced_updates" "$f"; then
-      echo 1>&2 "  FORCING UPDATE for file '$f'..."
-      # The full workdir may not exist yet, so create it if necessary. 
+      # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
       $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
-      make_add_instruction "$f" >> $manifest
+      make_add_instruction "$f" 1 >> $updatemanifestv1
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
       # Compute both the compressed binary diff and the compressed file, and
       # compare the sizes.  Then choose the smaller of the two to package.
-      echo "  diffing $f"
       dir=$(dirname "$workdir/$f")
       mkdir -p "$dir"
       $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
       $BZIP2 -z9 "$workdir/$f.patch"
       $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
       patchfile="$workdir/$f.patch.bz2"
       patchsize=$(get_file_size "$patchfile")
       fullsize=$(get_file_size "$workdir/$f")
 
-      if [ $patchsize -lt $fullsize -a "$f" != "removed-files" ]; then
-        make_patch_instruction "$f" >> $manifest
+      if [ $patchsize -lt $fullsize ]; then
+        make_patch_instruction "$f" >> $updatemanifestv1
         mv -f "$patchfile" "$workdir/$f.patch"
         rm -f "$workdir/$f"
         archivefiles="$archivefiles \"$f.patch\""
       else
-        make_add_instruction "$f" >> $manifest
+        make_add_instruction "$f" >> $updatemanifestv1
         rm -f "$patchfile"
         archivefiles="$archivefiles \"$f\""
       fi
     fi
   else
-    echo "remove \"$f\"" >> $manifest
+    # remove instructions are added after add / patch instructions for
+    # consistency with make_incremental_updates.py
+    remove_array[$num_removes]=$f
+    (( num_removes++ ))
   fi
 done
 
-# Now, we just need to worry about newly added files
+# Newly added files
+notice ""
+notice "Adding file add instructions to file 'update.manifest'"
 num_newfiles=${#newfiles[*]}
 
 for ((i=0; $i<$num_newfiles; i=$i+1)); do
   f="${newfiles[$i]}"
 
+  # removed-files is excluded by make_incremental_updates.py so it is excluded
+  # here for consistency.
+  if [ `basename $f` = "removed-files" ]; then
+    continue 1
+  fi
+
   # If we've already tested this file, then skip it
   for ((j=0; $j<$num_oldfiles; j=$j+1)); do
     if [ "$f" = "${oldfiles[j]}" ]; then
       continue 2
     fi
   done
-  
+
   dir=$(dirname "$workdir/$f")
   mkdir -p "$dir"
 
   $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
   copy_perm "$newdir/$f" "$workdir/$f"
 
-  make_add_instruction "$f" >> "$manifest"
+  make_add_instruction "$f" >> "$updatemanifestv1"
   archivefiles="$archivefiles \"$f\""
 done
 
+notice ""
+notice "Adding file remove instructions to file 'update.manifest'"
+for ((i=0; $i<$num_removes; i=$i+1)); do
+  f="${remove_array[$i]}"
+  notice "     remove: $f"
+  echo "remove \"$f\"" >> $updatemanifestv1
+done
+
+# Add the type of update to the beginning of and cat the contents of the version
+# 1 update manifest to the version 2 update manifest.
+notice ""
+notice "Adding type instruction to file 'updatev2.manifest'"
+> $updatemanifestv2
+notice "       type: partial"
+echo "type \"partial\"" >> $updatemanifestv2
+
+notice ""
+notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
+cat $updatemanifestv1 >> $updatemanifestv2
+
 # Append remove instructions for any dead files.
-append_remove_instructions "$newdir" >> $manifest
+notice ""
+notice "Adding file and directory remove instructions from file 'removed-files'"
+append_remove_instructions "$newdir" "$updatemanifestv1" "$updatemanifestv2"
+
+notice ""
+notice "Adding directory remove instructions for directories that no longer exist"
+num_olddirs=${#olddirs[*]}
 
-$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
+for ((i=0; $i<$num_olddirs; i=$i+1)); do
+  f="${olddirs[$i]}"
+  # If this dir doesn't exist in the new directory remove it.
+  if [ ! -d "$newdir/$f" ]; then
+    notice "      rmdir: $f/"
+    echo "rmdir \"$f/\"" >> $updatemanifestv2
+  fi
+done
+
+$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
+$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
 
 eval "$MAR -C \"$workdir\" -c output.mar $archivefiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
+
+notice ""
+notice "Finished"
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -1,129 +1,166 @@
 import os
 import shutil
 import sha
-import sha
 from os.path import join, getsize
 from stat import *
 import re
 import sys
 import getopt
 import time
 import datetime
 import bz2
 import string
 import tempfile
 
 class PatchInfo:
     """ Represents the meta-data associated with a patch
         work_dir = working dir where files are stored for this patch
         archive_files = list of files to include in this patch
-        manifest = set of patch instructions
+        manifestv1 = set of manifest version 1 patch instructions
+        manifestv2 = set of manifest version 2 patch instructions
         file_exclusion_list = 
         files to exclude from this patch. names without slashes will be
                               excluded anywhere in the directory hiearchy.   names with slashes
                               will only be excluded at that exact path
         """
     def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
         self.work_dir=work_dir
         self.archive_files=[]
-        self.manifest=[]
+        self.manifestv1=[]
+        self.manifestv2=[]
         self.file_exclusion_list=file_exclusion_list
         self.path_exclusion_list=path_exclusion_list
         
     def append_add_instruction(self, filename):
         """ Appends an add instruction for this patch.   
             if the filename starts with extensions/ adds an add-if instruction
             to test the existence of the subdirectory.  This was ported from
             mozilla/tools/update-packaging/common.sh/make_add_instruction
         """
         if filename.startswith("extensions/"):
             # Directory immediately following extensions is used for the test
             testdir = "extensions/"+filename.split("/")[1]
-            self.manifest.append('add-if "'+testdir+'" "'+filename+'"')
+            self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"')
+            self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
         elif filename.startswith("Contents/MacOS/extensions/"):
             testdir = "Contents/MacOS/extensions/"+filename.split("/")[3]
-            self.manifest.append('add-if "'+testdir+'" "'+filename+'"')
+            self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"')
+            self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
         else:
-            self.manifest.append('add "'+filename+'"')
+            self.manifestv1.append('add "'+filename+'"')
+            self.manifestv2.append('add "'+filename+'"')
            
     def append_patch_instruction(self, filename, patchname):
         """ Appends an patch instruction for this patch.   
             
             filename = file to patch
             patchname = patchfile to apply to file
             
             if the filename starts with extensions/ adds a patch-if instruction
             to test the existence of the subdirectory.  
             if the filename starts with searchplugins/ add a add-if instruction for the filename
             This was ported from
             mozilla/tools/update-packaging/common.sh/make_patch_instruction
         """
         if filename.startswith("extensions/"):
             testdir = "extensions/"+filename.split("/")[1]
-            self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
         elif filename.startswith("Contents/MacOS/extensions/"):
             testdir = "Contents/MacOS/extensions/"+filename.split("/")[3]
-            self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
         elif (filename.startswith("searchplugins/") or
              filename.startswith("Contents/MacOS/searchplugins/")):
-            self.manifest.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv1.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv2.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
         else:
-            self.manifest.append('patch "'+patchname+'" "'+filename+'"')
+            self.manifestv1.append('patch "'+patchname+'" "'+filename+'"')
+            self.manifestv2.append('patch "'+patchname+'" "'+filename+'"')
                 
     def append_remove_instruction(self, filename):
         """ Appends an remove instruction for this patch.   
             This was ported from
             mozilla/tools/update-packaging/common.sh/make_remove_instruction
         """
-        self.manifest.append('remove "'+filename+'"')
+        if filename.endswith("/"):
+            self.manifestv2.append('rmdir "'+filename+'"')
+        elif filename.endswith("/*"):
+            filename = filename[:-1]
+            self.manifestv2.append('rmrfdir "'+filename+'"')
+        else:
+            self.manifestv1.append('remove "'+filename+'"')
+            self.manifestv2.append('remove "'+filename+'"')
 
-    def create_manifest_file(self):
-        """ Createst the manifest file into to the root of the work_dir """
+    def create_manifest_files(self):
+        """ Createst the v1 manifest file in the root of the work_dir """
         manifest_file_path = os.path.join(self.work_dir,"update.manifest")
-        manifest_file = open(manifest_file_path, "w")
-        manifest_file.writelines(string.join(self.manifest, '\n'))
+        manifest_file = open(manifest_file_path, "wb")
+        manifest_file.writelines(string.join(self.manifestv1, '\n'))
         manifest_file.writelines("\n")
         manifest_file.close()    
 
         bzip_file(manifest_file_path)
         self.archive_files.append('"update.manifest"')
 
+        """ Createst the v2 manifest file in the root of the work_dir """
+        manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
+        manifest_file = open(manifest_file_path, "wb")
+        manifest_file.writelines("type \"partial\"\n")
+        manifest_file.writelines(string.join(self.manifestv2, '\n'))
+        manifest_file.writelines("\n")
+        manifest_file.close()
+
+        bzip_file(manifest_file_path)
+        self.archive_files.append('"updatev2.manifest"')
 
     def build_marfile_entry_hash(self, root_path):
-         """ Iterates through the root_path, creating a MarFileEntry for each file
-             in that path.  Excluseds any filenames in the file_exclusion_list
-         """
-         mar_entry_hash = {}
-         filename_set = set()
-         for root, dirs, files in os.walk(root_path):
-             for name in files:
-                 # filename is relative path from root directory
-                 partial_path = root[len(root_path)+1:]
-                 if name not in self.file_exclusion_list:
-                     filename = os.path.join(partial_path, name)
-                     if "/"+filename not in self.path_exclusion_list:
-                         mar_entry_hash[filename]=MarFileEntry(root_path, filename)
-                         filename_set.add(filename)
-         return mar_entry_hash, filename_set
+        """ Iterates through the root_path, creating a MarFileEntry for each file
+            and directory in that path.  Excludes any filenames in the file_exclusion_list
+        """
+        mar_entry_hash = {}
+        filename_set = set()
+        dirname_set = set()
+        for root, dirs, files in os.walk(root_path):
+            for name in files:
+                # filename is the relative path from root directory
+                partial_path = root[len(root_path)+1:]
+                if name not in self.file_exclusion_list:
+                    filename = os.path.join(partial_path, name)
+                    if "/"+filename not in self.path_exclusion_list:
+                        mar_entry_hash[filename]=MarFileEntry(root_path, filename)
+                        filename_set.add(filename)
+
+            for name in dirs:
+                # dirname is the relative path from root directory
+                partial_path = root[len(root_path)+1:]
+                if name not in self.file_exclusion_list:
+                    dirname = os.path.join(partial_path, name)
+                    if "/"+dirname not in self.path_exclusion_list:
+                        dirname = dirname+"/"
+                        mar_entry_hash[dirname]=MarFileEntry(root_path, dirname)
+                        dirname_set.add(dirname)
+
+        return mar_entry_hash, filename_set, dirname_set
  
        
 class MarFileEntry:
     """Represents a file inside a Mozilla Archive Format (MAR)
         abs_path = abspath to the the file
         name =  relative path within the mar.  e.g.
           foo.mar/dir/bar.txt extracted into /tmp/foo:
             abs_path=/tmp/foo/dir/bar.txt
             name = dir/bar.txt
     """ 
     def __init__(self, root, name):
         """root = path the the top of the mar
            name = relative path within the mar"""
-        self.name=name
+        self.name=name.replace("\\", "/")
         self.abs_path=os.path.join(root,name)
         self.sha_cache=None
 
     def __str__(self):
         return 'Name: %s FullPath: %s' %(self.name,self.abs_path)
 
     def calc_file_sha_digest(self, filename):        
         """ Returns sha digest of given filename"""
@@ -214,20 +251,20 @@ def create_partial_patch_for_file(from_m
             os.remove(patch_file_abs_path)
             file_in_manifest_name = from_marfile_entry.name
             file_in_manifest_abspath = full_file_abs_path
             patch_info.append_add_instruction(file_in_manifest_name)
             
         shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath)
         patch_info.archive_files.append('"'+file_in_manifest_name+'"')        
     else:
-        print "skipping diff: " + from_marfile_entry.name
         filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()]
         # We've already calculated the patch for this pair of files.   
         if (filename.endswith(".patch")):
+            print "skipping diff: " + from_marfile_entry.name
             # Patch was smaller than file - add patch instruction to manifest
             file_in_manifest_name = to_marfile_entry.name+'.patch';
             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
         else:
             # File was smaller than file - add file to manifest
             file_in_manifest_name = to_marfile_entry.name
             patch_info.append_add_instruction(file_in_manifest_name)                
         # Copy the pre-calculated file into our new patch work aread
@@ -245,72 +282,88 @@ def process_explicit_remove_files(dir_pa
     """ Looks for a 'removed-files' file in the dir_path.  If the removed-files does not exist
     this will throw.  If found adds the removed-files
     found in that file to the patch_info"""
 
     # Windows and linux have this file at the root of the dir
     list_file_path = os.path.join(dir_path, "removed-files")
     prefix=""
     if not os.path.exists(list_file_path):
-        # Mac has is in Contents/MacOS/
+        # On Mac removed-files contains relative paths from Contents/MacOS/
         prefix= "Contents/MacOS"
         list_file_path = os.path.join(dir_path, prefix+"/removed-files")
 
     if (os.path.exists(list_file_path)):
         list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
 
+        lines = []
         for line in list_file:
-            line = line.strip()
-            # Exclude any blank lines or any lines ending with a slash, which indicate
-            # directories.  The updater doesn't know how to remove entire directories.
-            if line and not line.endswith("/"): 
-                patch_info.append_remove_instruction(os.path.join(prefix,line))
+            lines.append(line.strip())
+
+        lines.sort(reverse=True)
+        for line in lines:
+            # Exclude any blank and comment lines.
+            if line and not line.startswith("#"):
+                # Python on windows uses \ for path separators and the update
+                # manifests expects / for path separators on all platforms.
+                line=os.path.join(prefix,line).replace("\\", "/")
+                patch_info.append_remove_instruction(line)
 
 def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates):
     """ Builds a partial patch by comparing the files in from_dir_path to thoes of to_dir_path"""
     # Cannocolize the paths for safey
     from_dir_path = os.path.abspath(from_dir_path)
     to_dir_path = os.path.abspath(to_dir_path)
-    # First create a hashtable of the from  and to directories
-    from_dir_hash,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
-    to_dir_hash,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
+    # Create a hashtable of the from  and to directories
+    from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
+    to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
+    # Require that the precomplete file is included in the to complete update
+    if "precomplete" not in to_file_set:
+        raise Exception, "missing precomplete file in: "+to_dir_path
     # Create a list of the forced updates 
     forced_list = forced_updates.strip().split('|')
-    
+    forced_list.append("precomplete")
+
     # Files which exist in both sets need to be patched
-    patch_filenames = list(from_dir_set.intersection(to_dir_set))
-    patch_filenames.sort()
+    patch_filenames = list(from_file_set.intersection(to_file_set))
+    patch_filenames.sort(reverse=True)
     for filename in patch_filenames:
         from_marfile_entry = from_dir_hash[filename]
         to_marfile_entry = to_dir_hash[filename]
         if filename in forced_list:
             print "Forcing "+ filename
             # This filename is in the forced list, explicitly add
-       	    create_add_patch_for_file(to_dir_hash[filename], patch_info)
+            create_add_patch_for_file(to_dir_hash[filename], patch_info)
         else: 
           if from_marfile_entry.sha() != to_marfile_entry.sha():
               # Not the same - calculate a patch
               create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info)
 
+    # files in to_dir not in from_dir need to added
+    add_filenames = list(to_file_set - from_file_set)
+    add_filenames.sort(reverse=True)
+    for filename in add_filenames:
+        create_add_patch_for_file(to_dir_hash[filename], patch_info)
+
     # files in from_dir not in to_dir need to be removed
-    remove_filenames = list(from_dir_set - to_dir_set)
-    remove_filenames.sort()
+    remove_filenames = list(from_file_set - to_file_set)
+    remove_filenames.sort(reverse=True)
     for filename in remove_filenames:
         patch_info.append_remove_instruction(from_dir_hash[filename].name)
 
-    # files in to_dir not in from_dir need to added
-    add_filenames = list(to_dir_set - from_dir_set)
-    add_filenames.sort()
-    for filename in add_filenames:
-        create_add_patch_for_file(to_dir_hash[filename], patch_info)
-
     process_explicit_remove_files(to_dir_path, patch_info)
     
-    # Construct Manifest file
-    patch_info.create_manifest_file()
+    # directories in from_dir not in to_dir need to be removed
+    remove_dirnames = list(from_dir_set - to_dir_set)
+    remove_dirnames.sort(reverse=True)
+    for dirname in remove_dirnames:
+        patch_info.append_remove_instruction(from_dir_hash[dirname].name)
+
+    # Construct the Manifest files
+    patch_info.create_manifest_files()
     
     # And construct the mar
     mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')    
     exec_shell_cmd(mar_cmd)
 
     # Copy mar to final destination
     patch_file_dir = os.path.split(patch_filename)[0]
     if not os.path.exists(patch_file_dir):
@@ -397,17 +450,17 @@ def create_partial_patches(patches):
             extract_mar(to_filename, work_dir_to)
             to_decoded = decode_filename(from_filename)
             to_buildid = get_buildid(work_dir_to, to_decoded['platform'])
             to_shasum = sha.sha(open(to_filename).read()).hexdigest()
             to_size = str(os.path.getsize(to_filename))
 
             mar_extract_time = time.time()
 
-            partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt']),forced_updates)
+            partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt']),forced_updates)
             partial_buildid = to_buildid
             partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
             partial_size = str(os.path.getsize(partial_filename))
 
             metadata.append({
              'to_filename': os.path.basename(to_filename),
              'from_filename': os.path.basename(from_filename),
              'partial_filename': os.path.basename(partial_filename),
--- a/tools/update-packaging/test/buildrefmars.sh
+++ b/tools/update-packaging/test/buildrefmars.sh
@@ -1,16 +1,26 @@
 #!/bin/bash
 # Builds all the reference mars
 
-rm ref.mar
-rm ref-mac.mar
+if [ -f "ref.mar" ]; then
+  rm "ref.mar"
+fi
+if [ -f "ref-mac.mar" ]; then
+  rm "ref-mac.mar"
+fi
 
  ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to
  ../make_incremental_update.sh ref-mac.mar `pwd`/from `pwd`/to-mac
 
-rm product-1.0.lang.platform.complete.mar
-rm product-2.0.lang.platform.complete.mar
-rm product-2.0.lang.mac.complete.mar
+if [ -f "product-1.0.lang.platform.complete.mar" ]; then
+  rm "product-1.0.lang.platform.complete.mar"
+fi
+if [ -f "product-2.0.lang.platform.complete.mar" ]; then
+  rm "product-2.0.lang.platform.complete.mar"
+fi
+if [ -f "product-2.0.lang.mac.complete.mar" ]; then
+  rm "product-2.0.lang.mac.complete.mar"
+fi
 
-./make_full_update.sh product-1.0.lang.platform.complete.mar `pwd`/from
-./make_full_update.sh product-2.0.lang.platform.complete.mar `pwd`/to 
-./make_full_update.sh product-2.0.lang.mac.complete.mar `pwd`/to-mac 
+./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from"
+./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to"
+./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac"
--- a/tools/update-packaging/test/catmanifest.sh
+++ b/tools/update-packaging/test/catmanifest.sh
@@ -6,9 +6,9 @@ workdir="/tmp/catmanifest"
 
 rm -rf "$workdir"
 mkdir -p "$workdir"
 cp "$1" "$workdir"
 cd "$workdir"
 mar -x "$1"
 mv update.manifest update.manifest.bz2
 bzip2 -d update.manifest.bz2
-cat update.manifest
\ No newline at end of file
+cat update.manifest
--- a/tools/update-packaging/test/common.sh
+++ b/tools/update-packaging/test/common.sh
@@ -10,17 +10,17 @@
 MAR=${MAR:-mar}
 BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
-  echo $* 1>&2
+  echo "$*" 1>&2
 }
 
 get_file_size() {
   info=($(ls -ln "$1"))
   echo ${info[4]}
 }
 
 copy_perm() {
@@ -31,82 +31,137 @@ copy_perm() {
     chmod 0755 "$target"
   else
     chmod 0644 "$target"
   fi
 }
 
 make_add_instruction() {
   f="$1"
+
+  # Used to log to the console
+  if [ $2 ]; then
+    forced=" (forced)"
+  else
+    forced=
+  fi
+
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "     add-if: $f$forced"
     echo "add-if \"$testdir\" \"$f\""
   else
+    notice "        add: $f$forced"
     echo "add \"$f\""
   fi
 }
 
 make_patch_instruction() {
   f="$1"
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   is_search_plugin=$(echo "$f" | grep -c 'searchplugins/.*')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "   patch-if: $f"
     echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
   elif [ $is_search_plugin = "1" ]; then
+    notice "   patch-if: $f"
     echo "patch-if \"$f\" \"$f.patch\" \"$f\""
   else
+    notice "      patch: $f"
     echo "patch \"$f.patch\" \"$f\""
   fi
 }
 
 append_remove_instructions() {
   dir="$1"
+  filev1="$2"
+  filev2="$3"
   if [ -f "$dir/removed-files" ]; then
     prefix=
     listfile="$dir/removed-files"
   elif [ -f "$dir/Contents/MacOS/removed-files" ]; then
     prefix=Contents/MacOS/
     listfile="$dir/Contents/MacOS/removed-files"
   fi
   if [ -n "$listfile" ]; then
     # Map spaces to pipes so that we correctly handle filenames with spaces.
-    files=($(cat "$listfile" | tr " " "|"))  
+    files=($(cat "$listfile" | tr " " "|"  | sort -r))
     num_files=${#files[*]}
     for ((i=0; $i<$num_files; i=$i+1)); do
-      # Trim whitespace (including trailing carriage returns)
-      f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r')
-      # Exclude any blank lines or any lines ending with a slash, which indicate
-      # directories.  The updater doesn't know how to remove entire directories.
+      # Map pipes back to whitespace and remove carriage returns
+      f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
+      # Trim whitespace
+      f=$(echo $f)
+      # Exclude blank lines.
       if [ -n "$f" ]; then
-        if [ $(echo "$f" | grep -c '\/$') = 0 ]; then
-          echo "remove \"$prefix$f\""
-        else
-          notice "ignoring remove instruction for directory: $f"
+        # Exclude comments
+        if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
+          # Normalize the path to the root of the Mac OS X bundle if necessary
+          fixedprefix="$prefix"
+          if [ $prefix ]; then
+            if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then
+              if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then
+                f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::')
+                fixedprefix=""
+              else
+                f=$(echo $f | sed -e 's:^\.\.\/::')
+                fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::')
+              fi
+            fi
+          fi
+          if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
+            notice "      rmdir: $fixedprefix$f"
+            echo "rmdir \"$fixedprefix$f\"" >> $filev2
+          elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
+            # Remove the *
+            f=$(echo "$f" | sed -e 's:\*$::')
+            notice "    rmrfdir: $fixedprefix$f"
+            echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
+          else
+            notice "     remove: $fixedprefix$f"
+            echo "remove \"$fixedprefix$f\"" >> $filev1
+            echo "remove \"$fixedprefix$f\"" >> $filev2
+          fi
         fi
       fi
     done
   fi
 }
 
 # List all files in the current directory, stripping leading "./"
 # Skip the channel-prefs.js file as it should not be included in any
 # generated MAR files (see bug 306077). Pass a variable name and it will be
 # filled as an array.
 list_files() {
   count=0
 
-  # Schrep - removed the exclusion cases here to allow for generation
-  # of testing mars
+  # Removed the exclusion cases here to allow for generation of testing mars
   find . -type f \
     | sed 's/\.\/\(.*\)/\1/' \
-    | sort > "$workdir/temp-filelist"
+    | sort -r > "$workdir/temp-filelist"
   while read file; do
     eval "${1}[$count]=\"$file\""
     (( count++ ))
   done < "$workdir/temp-filelist"
   rm "$workdir/temp-filelist"
 }
+
+# List all directories in the current directory, stripping leading "./"
+list_dirs() {
+  count=0
+
+  find . -type d \
+    ! -name "." \
+    ! -name ".." \
+    | sed 's/\.\/\(.*\)/\1/' \
+    | sort -r > "$workdir/temp-dirlist"
+  while read dir; do
+    eval "${1}[$count]=\"$dir\""
+    (( count++ ))
+  done < "$workdir/temp-dirlist"
+  rm "$workdir/temp-dirlist"
+}
--- a/tools/update-packaging/test/diffmar.sh
+++ b/tools/update-packaging/test/diffmar.sh
@@ -2,34 +2,49 @@
 # Compares two mars
 
 marA="$1"
 marB="$2"
 workdir="/tmp/diffmar"
 fromdir="$workdir/0"
 todir="$workdir/1"
 
+# On Windows, creation time can be off by a second or more between the files in
+# the fromdir and todir due to them being extracted synchronously so use
+# time-style and exclude seconds from the creation time.
+lsargs="-algR"
+unamestr=`uname`
+if [ ! "$unamestr" = 'Darwin' ]; then
+  unamestr=`uname -o`
+  if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then
+     lsargs="-algR --time-style=+%Y-%m-%d-%H:%M"
+  fi
+fi
+
 rm -rf "$workdir"
 mkdir -p "$fromdir"
 mkdir -p "$todir"
 
 cp "$1" "$fromdir"
 cp "$2" "$todir"
 
 cd "$fromdir"
 mar -x "$1"
 rm "$1"
 mv update.manifest update.manifest.bz2
 bzip2 -d update.manifest.bz2
-ls -algR > files.txt
-# Sort the manifest so we don't get any diffs for ordering
-#cat update.manifest | sort > update.manifest
+mv updatev2.manifest updatev2.manifest.bz2
+bzip2 -d updatev2.manifest.bz2
+ls $lsargs > files.txt
 
 cd "$todir"
 mar -x "$2"
 rm "$2"
 mv update.manifest update.manifest.bz2
 bzip2 -d update.manifest.bz2
-# Sort the manifest so we don't get any diffs for ordering
-#cat update.manifest | sort > update.manifest
-ls -algR > files.txt
+mv updatev2.manifest updatev2.manifest.bz2
+bzip2 -d updatev2.manifest.bz2
+ls $lsargs > files.txt
 
-diff -r "$fromdir" "$todir"
\ No newline at end of file
+echo "diffing $fromdir and $todir"
+echo "on linux shell sort and python sort return different results"
+echo "which cause differences in the manifest files"
+diff -ru "$fromdir" "$todir"
new file mode 100644
--- /dev/null
+++ b/tools/update-packaging/test/from/precomplete
@@ -0,0 +1,21 @@
+remove {foodir/update.manifest
+remove {foodir/same.txt
+remove {foodir/same.bin
+remove {foodir/removed.txt
+remove {foodir/readme.txt
+remove {foodir/force.txt
+remove {foodir/diff-patch-larger-than-file.txt
+remove-cc {foodir/channel-prefs.js
+remove update.manifest
+remove searchplugins/diff/diff-patch-larger-than-file.txt
+remove same.txt
+remove same.bin
+remove removed.txt
+remove readme.txt
+remove precomplete
+remove force.txt
+remove diff-patch-larger-than-file.txt
+remove-cc channel-prefs.js
+rmdir {foodir/
+rmdir searchplugins/diff/
+rmdir searchplugins/
--- a/tools/update-packaging/test/make_full_update.sh
+++ b/tools/update-packaging/test/make_full_update.sh
@@ -28,53 +28,102 @@ if [ $1 = -h ]; then
   notice ""
   exit 1
 fi
 
 # -----------------------------------------------------------------------------
 
 archive="$1"
 targetdir="$2"
+# Prevent the workdir from being inside the targetdir so it isn't included in
+# the update mar.
+if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
+  # Remove the /
+  targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
+fi
 workdir="$targetdir.work"
-manifest="$workdir/update.manifest"
-targetfiles="update.manifest"
+updatemanifestv1="$workdir/update.manifest"
+updatemanifestv2="$workdir/updatev2.manifest"
+targetfiles="update.manifest updatev2.manifest"
 
 mkdir -p "$workdir"
 
+# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
+# Info.plist so it launches the right architecture, bug 600098
+
 # Generate a list of all files in the target directory.
 pushd "$targetdir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
+if [ ! -f "precomplete" ]; then
+  notice "precomplete file is missing!"
+  exit 1
+fi
+
 list_files files
 
+# Files that should be added on channel change
+ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
+
 popd
 
-> $manifest
+notice ""
+notice "Adding file add instructions to file 'update.manifest'"
+> $updatemanifestv1
 
 num_files=${#files[*]}
 
 for ((i=0; $i<$num_files; i=$i+1)); do
   f="${files[$i]}"
 
-  notice "processing $f"
-
-  make_add_instruction "$f" >> $manifest
+  make_add_instruction "$f" >> $updatemanifestv1
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
   $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
+# Add the type of update to the beginning of and cat the contents of the version
+# 1 update manifest to the version 2 update manifest.
+> $updatemanifestv2
+notice ""
+notice "Adding type instruction to file 'updatev2.manifest'"
+notice "       type: complete"
+echo "type \"complete\"" >> $updatemanifestv2
+
+notice ""
+notice "Adding file add on channel change instructions to file 'updatev2.manifest'"
+for f in $ccfiles; do
+  notice "     add-cc: $f"
+  echo "add-cc \"$f\"" >> $updatemanifestv2
+  dir=$(dirname "$f")
+  mkdir -p "$workdir/$dir"
+  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  copy_perm "$targetdir/$f" "$workdir/$f"
+
+  targetfiles="$targetfiles \"$f\""
+done
+
+notice ""
+notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
+cat $updatemanifestv1 >> $updatemanifestv2
+
 # Append remove instructions for any dead files.
-append_remove_instructions "$targetdir" >> $manifest
+notice ""
+notice "Adding file and directory remove instructions from file 'removed-files'"
+append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2"
 
-$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
+$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
+$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
+
+notice ""
+notice "Finished"
--- a/tools/update-packaging/test/runtests.sh
+++ b/tools/update-packaging/test/runtests.sh
@@ -2,18 +2,8 @@
 
 echo "testing make_incremental_updates.py"
 python ../make_incremental_updates.py -f testpatchfile.txt
 
 echo "diffing ref.mar and test.mar"
 ./diffmar.sh ref.mar test.mar
 echo "diffing ref-mac.mar and test-mac.mar"
 ./diffmar.sh ref-mac.mar test-mac.mar
-
-
-echo "testing make_incremental_updates_mar.py"
-python ../make_incremental_updates_mar.py -f testpatchfile.txt
-
-echo "diffing ref.mar and test.mar"
-./diffmar.sh ref.mar test.mar
-echo "diffing ref-mac.mar and test-mac.mar"
-./diffmar.sh ref-mac.mar test-mac.mar
-
new file mode 100644
--- /dev/null
+++ b/tools/update-packaging/test/to-mac/precomplete
@@ -0,0 +1,4 @@
+remove precomplete
+remove Contents/MacOS/removed-files
+rmdir Contents/MacOS/
+rmdir Contents/
new file mode 100644
--- /dev/null
+++ b/tools/update-packaging/test/to/precomplete
@@ -0,0 +1,21 @@
+remove {foodir/update.manifest
+remove {foodir/same.txt
+remove {foodir/same.bin
+remove {foodir/readme.txt
+remove {foodir/force.txt
+remove {foodir/diff-patch-larger-than-file.txt
+remove-cc {foodir/channel-prefs.js
+remove update.manifest
+remove searchplugins/diff/diff-patch-larger-than-file.txt
+remove same.txt
+remove same.bin
+remove removed-files
+remove readme.txt
+remove precomplete
+remove force.txt
+remove diff-patch-larger-than-file.txt
+remove-cc channel-prefs.js
+rmdir {foodir/
+rmdir searchplugins/diff/
+rmdir searchplugins/added/
+rmdir searchplugins/
--- a/tools/update-packaging/test_make_incremental_updates.py
+++ b/tools/update-packaging/test_make_incremental_updates.py
@@ -2,46 +2,47 @@
 
 import unittest
 import make_incremental_updates as mkup
 from make_incremental_updates import PatchInfo, MarFileEntry
 
 class TestPatchInfo(unittest.TestCase):
     def setUp(self):
         self.work_dir = 'work_dir'
-        self.file_exclusion_list = ['channel-prefs.js','update.manifest','removed-files']
+        self.file_exclusion_list = ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files']
         self.path_exclusion_list = ['/readme.txt']
         self.patch_info = PatchInfo(self.work_dir, self.file_exclusion_list, self.path_exclusion_list)
 
     def testPatchInfo(self):
         self.assertEquals(self.work_dir, self.patch_info.work_dir)
         self.assertEquals([], self.patch_info.archive_files)
-        self.assertEquals([], self.patch_info.manifest)
+        self.assertEquals([], self.patch_info.manifestv1)
+        self.assertEquals([], self.patch_info.manifestv2)
         self.assertEquals(self.file_exclusion_list, self.patch_info.file_exclusion_list)
         self.assertEquals(self.path_exclusion_list, self.patch_info.path_exclusion_list)
 
     def test_append_add_instruction(self):
         self.patch_info.append_add_instruction('file.test')
-        self.assertEquals(['add "file.test"'], self.patch_info.manifest)
+        self.assertEquals(['add "file.test"'], self.patch_info.manifestv1)
 
     def test_append_patch_instruction(self):
         self.patch_info.append_patch_instruction('file.test', 'patchname')
-        self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifest)
+        self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv1)
 
     """ FIXME touches the filesystem, need refactoring
     def test_append_remove_instruction(self):
         self.patch_info.append_remove_instruction('file.test')
-        self.assertEquals(['remove "file.test"'], self.patch_info.manifest)
+        self.assertEquals(['remove "file.test"'], self.patch_info.manifestv1)
 
     def test_create_manifest_file(self):
         self.patch_info.create_manifest_file()
     """
 
     def test_build_marfile_entry_hash(self):
-        self.assertEquals(({}, set([])), self.patch_info.build_marfile_entry_hash('root_path'))
+        self.assertEquals(({}, set([]), set([])), self.patch_info.build_marfile_entry_hash('root_path'))
 
 """ FIXME touches the filesystem, need refactoring
 class TestMarFileEntry(unittest.TestCase):
     def setUp(self):
         root_path = '.'
         self.filename = 'file.test'
         f = open(self.filename, 'w')
         f.write('Test data\n')
@@ -61,17 +62,17 @@ class TestMarFileEntry(unittest.TestCase
         f.close()
         sha = self.mar_file_entry.sha()
         self.assertEquals(goodSha, sha)
 """
 
 class TestMakeIncrementalUpdates(unittest.TestCase):
     def setUp(self):
         work_dir = '.'
-        self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt'])
+        self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt'])
         root_path = '/'
         filename = 'test.file'
         self.mar_file_entry = MarFileEntry(root_path, filename)
 
     """ FIXME makes direct shell calls, need refactoring
     def test_exec_shell_cmd(self):
         mkup.exec_shell_cmd('echo test')