1. mar packaging and build changes - Bug 386760 - directory removal with software update. r=nthomas, r=khuey
authorRobert Strong <robert.bugzilla@gmail.com>
Mon, 11 Apr 2011 21:23:18 -0700
changeset 67955 4ceb3de77444dd4c97b9ec6f21a45785f7793ed5
parent 67954 3ac2e9681421f3348f482832579dbb66be9fc431
child 67956 860d857e899db6f7512a7a873c47391bdfafa16b
push idunknown
push userunknown
push dateunknown
reviewersnthomas, khuey
bugs1, 386760
milestone2.2a1pre
1. mar packaging and build changes - Bug 386760 - directory removal with software update. r=nthomas, r=khuey
config/createprecomplete.py
toolkit/mozapps/installer/packager.mk
tools/update-packaging/common.sh
tools/update-packaging/make_full_update.sh
tools/update-packaging/make_incremental_update.sh
tools/update-packaging/make_incremental_updates.py
tools/update-packaging/test/buildrefmars.sh
tools/update-packaging/test/catmanifest.sh
tools/update-packaging/test/common.sh
tools/update-packaging/test/diffmar.sh
tools/update-packaging/test/from/precomplete
tools/update-packaging/test/make_full_update.sh
tools/update-packaging/test/runtests.sh
tools/update-packaging/test/to-mac/precomplete
tools/update-packaging/test/to/precomplete
tools/update-packaging/test_make_incremental_updates.py
new file mode 100644
--- /dev/null
+++ b/config/createprecomplete.py
@@ -0,0 +1,64 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+# Creates the precomplete file containing the remove, remove-cc, and rmdir
+# application update instructions which is used to remove files and directories
+# that are no longer present in a complete update. The current working directory
+# is used for the location to enumerate and to create the precomplete file.
+
+import sys
+import os
+
+def get_build_entries(root_path):
+    """ Iterates through the root_path, creating a list for each file and
+        directory. Excludes any path starting with extensions or distribution.
+    """
+    rel_file_path_set = set()
+    rel_dir_path_set = set()
+    for root, dirs, files in os.walk(root_path):
+        for file_name in files:
+            parent_dir_rel_path = root[len(root_path)+1:]
+            rel_path_file = os.path.join(parent_dir_rel_path, file_name)
+            rel_path_file = rel_path_file.replace("\\", "/")
+            if not (rel_path_file.startswith("distribution/") or
+                    rel_path_file.startswith("extensions/")):
+                rel_file_path_set.add(rel_path_file)
+
+        for dir_name in dirs:
+            parent_dir_rel_path = root[len(root_path)+1:]
+            rel_path_dir = os.path.join(parent_dir_rel_path, dir_name)
+            rel_path_dir = rel_path_dir.replace("\\", "/")+"/"
+            if not (rel_path_dir.startswith("distribution/") or
+                    rel_path_dir.startswith("extensions/")):
+                rel_dir_path_set.add(rel_path_dir)
+
+    rel_file_path_list = list(rel_file_path_set)
+    rel_file_path_list.sort(reverse=True)
+    rel_dir_path_list = list(rel_dir_path_set)
+    rel_dir_path_list.sort(reverse=True)
+
+    return rel_file_path_list, rel_dir_path_list
+
+def generate_precomplete():
+    """ Creates the precomplete file containing the remove, remove-cc, and rmdir
+        application update instructions. The current working directory is used
+        for the location to enumerate and to create the precomplete file.
+    """
+    root_path = os.getcwd()
+    rel_file_path_list, rel_dir_path_list = get_build_entries(root_path)
+    precomplete_file_path = os.path.join(root_path,"precomplete")
+    # open in binary mode to prevent OS specific line endings.
+    precomplete_file = open(precomplete_file_path, "wb")
+    for rel_file_path in rel_file_path_list:
+        if rel_file_path.endswith("channel-prefs.js"):
+            precomplete_file.writelines("remove-cc \""+rel_file_path+"\"\n")
+        else:
+            precomplete_file.writelines("remove \""+rel_file_path+"\"\n")
+
+    for rel_dir_path in rel_dir_path_list:
+        precomplete_file.writelines("rmdir \""+rel_dir_path+"\"\n")
+
+    precomplete_file.close()
+
+if __name__ == "__main__":
+    generate_precomplete()
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -18,16 +18,17 @@
 # Netscape Communications Corporation.
 # Portions created by the Initial Developer are Copyright (C) 1998
 # the Initial Developer. All Rights Reserved.
 #
 # Contributor(s):
 #   Benjamin Smedberg <bsmedberg@covad.net>
 #   Arthur Wiebe <artooro@gmail.com>
 #   Mark Mentovai <mark@moxienet.com>
+#   Robert Strong <robert.bugzilla@gmail.com>
 #
 # Alternatively, the contents of this file may be used under the terms of
 # either of the GNU General Public License Version 2 or later (the "GPL"),
 # or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 # in which case the provisions of the GPL or the LGPL are applicable instead
 # of those above. If you wish to allow use of your version of this file only
 # under the terms of either the GPL or the LGPL, and not to allow others to
 # use your version of this file under the terms of the MPL, indicate your
@@ -373,16 +374,18 @@ INNER_UNMAKE_PACKAGE	= \
 SDK_SUFFIX = .tar.bz2
 SDK = $(MOZ_PKG_APPNAME)-$(MOZ_PKG_VERSION).$(AB_CD).mac-$(TARGET_CPU).sdk$(SDK_SUFFIX)
 ifeq ($(MOZ_APP_NAME),xulrunner)
 SDK = $(SDK_PATH)$(MOZ_APP_NAME)-$(MOZ_PKG_VERSION).$(AB_CD).mac-$(TARGET_CPU).sdk$(SDK_SUFFIX)
 endif
 MAKE_SDK = $(CREATE_FINAL_TAR) - $(MOZ_APP_NAME)-sdk | bzip2 -vf > $(SDK)
 endif
 
+CREATE_PRECOMPLETE = $(PYTHON) $(MOZILLA_DIR)/config/createprecomplete.py
+
 ifdef MOZ_OMNIJAR
 GENERATE_CACHE ?= true
 
 OMNIJAR_FILES	= \
   chrome \
   chrome.manifest \
   components/*.js \
   components/*.xpt \
@@ -414,20 +417,21 @@ PACK_OMNIJAR	= \
   printf "manifest components/binary.manifest\n" > chrome.manifest
 UNPACK_OMNIJAR	= \
   $(OPTIMIZE_JARS_CMD) --deoptimize $(_ABS_DIST)/jarlog/ ./ ./ && \
   unzip -o omni.jar && \
   rm -f components/binary.manifest && \
   sed -e 's/^\#binary-component/binary-component/' components/components.manifest > components.manifest && \
   mv components.manifest components
 
-MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && $(INNER_MAKE_PACKAGE)
+MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && \
+	              (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE)
 UNMAKE_PACKAGE	= $(INNER_UNMAKE_PACKAGE) && (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(UNPACK_OMNIJAR))
 else
-MAKE_PACKAGE	= $(INNER_MAKE_PACKAGE)
+MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE)
 UNMAKE_PACKAGE	= $(INNER_UNMAKE_PACKAGE)
 endif
 
 # dummy macro if we don't have PSM built
 SIGN_NSS		=
 ifneq (1_,$(if $(CROSS_COMPILE),1,0)_$(UNIVERSAL_BINARY))
 ifdef MOZ_PSM
 SIGN_NSS		= @echo signing nss libraries;
@@ -560,16 +564,17 @@ ifndef MOZ_PKG_MANIFEST
 endif
 	@rm -rf $(DEPTH)/installer-stage $(DIST)/xpt
 	@echo "Staging installer files..."
 	@$(NSINSTALL) -D $(DEPTH)/installer-stage/core
 ifdef MOZ_OMNIJAR
 	@(cd $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR))
 endif
 	@cp -av $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH)/. $(DEPTH)/installer-stage/core
+	@(cd $(DEPTH)/installer-stage/core && $(CREATE_PRECOMPLETE))
 ifdef MOZ_OPTIONAL_PKG_LIST
 	@$(NSINSTALL) -D $(DEPTH)/installer-stage/optional
 	$(call PACKAGER_COPY, "$(call core_abspath,$(DIST))",\
 	  "$(call core_abspath,$(DEPTH)/installer-stage/optional)", \
 	  "$(MOZ_PKG_MANIFEST)", "$(PKGCP_OS)", 1, 0, 1 \
 	  $(foreach pkg,$(MOZ_OPTIONAL_PKG_LIST),$(PKG_ARG)) )
 	@cd $(DEPTH)/installer-stage/optional/extensions; find -maxdepth 1 -mindepth 1 -exec rm -r ../../core/extensions/{} \;
 endif
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -9,17 +9,17 @@
 MAR=${MAR:-mar}
 BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
-  echo $* 1>&2
+  echo "$*" 1>&2
 }
 
 get_file_size() {
   info=($(ls -ln "$1"))
   echo ${info[4]}
 }
 
 copy_perm() {
@@ -30,83 +30,141 @@ copy_perm() {
     chmod 0755 "$target"
   else
     chmod 0644 "$target"
   fi
 }
 
 make_add_instruction() {
   f="$1"
+
+  # Used to log to the console
+  if [ $2 ]; then
+    forced=" (forced)"
+  else
+    forced=
+  fi
+
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "     add-if: $f$forced"
     echo "add-if \"$testdir\" \"$f\""
   else
+    notice "        add: $f$forced"
     echo "add \"$f\""
   fi
 }
 
 make_patch_instruction() {
   f="$1"
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   is_search_plugin=$(echo "$f" | grep -c 'searchplugins/.*')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "   patch-if: $f"
     echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
   elif [ $is_search_plugin = "1" ]; then
+    notice "   patch-if: $f"
     echo "patch-if \"$f\" \"$f.patch\" \"$f\""
   else
+    notice "      patch: $f"
     echo "patch \"$f.patch\" \"$f\""
   fi
 }
 
 append_remove_instructions() {
   dir="$1"
+  filev1="$2"
+  filev2="$3"
   if [ -f "$dir/removed-files" ]; then
     prefix=
     listfile="$dir/removed-files"
   elif [ -f "$dir/Contents/MacOS/removed-files" ]; then
     prefix=Contents/MacOS/
     listfile="$dir/Contents/MacOS/removed-files"
   fi
   if [ -n "$listfile" ]; then
     # Map spaces to pipes so that we correctly handle filenames with spaces.
-    files=($(cat "$listfile" | tr " " "|"))  
+    files=($(cat "$listfile" | tr " " "|"  | sort -r))
     num_files=${#files[*]}
     for ((i=0; $i<$num_files; i=$i+1)); do
-      # Trim whitespace (including trailing carriage returns)
-      f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r')
-      # Exclude any blank lines or any lines ending with a slash, which indicate
-      # directories.  The updater doesn't know how to remove entire directories.
+      # Map pipes back to whitespace and remove carriage returns
+      f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
+      # Trim whitespace
+      f=$(echo $f)
+      # Exclude blank lines.
       if [ -n "$f" ]; then
-        if [ $(echo "$f" | grep -c '\/$') = 0 ]; then
-          echo "remove \"$prefix$f\""
-        else
-          notice "ignoring remove instruction for directory: $f"
+        # Exclude comments
+        if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
+          # Normalize the path to the root of the Mac OS X bundle if necessary
+          fixedprefix="$prefix"
+          if [ $prefix ]; then
+            if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then
+              if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then
+                f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::')
+                fixedprefix=""
+              else
+                f=$(echo $f | sed -e 's:^\.\.\/::')
+                fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::')
+              fi
+            fi
+          fi
+          if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
+            notice "      rmdir: $fixedprefix$f"
+            echo "rmdir \"$fixedprefix$f\"" >> $filev2
+          elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
+            # Remove the *
+            f=$(echo "$f" | sed -e 's:\*$::')
+            notice "    rmrfdir: $fixedprefix$f"
+            echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
+          else
+            notice "     remove: $fixedprefix$f"
+            echo "remove \"$fixedprefix$f\"" >> $filev1
+            echo "remove \"$fixedprefix$f\"" >> $filev2
+          fi
         fi
       fi
     done
   fi
 }
 
 # List all files in the current directory, stripping leading "./"
 # Skip the channel-prefs.js file as it should not be included in any
 # generated MAR files (see bug 306077). Pass a variable name and it will be
 # filled as an array.
 list_files() {
   count=0
 
   find . -type f \
     ! -name "channel-prefs.js" \
     ! -name "update.manifest" \
+    ! -name "updatev2.manifest" \
+    ! -name "temp-dirlist" \
     ! -name "temp-filelist" \
     | sed 's/\.\/\(.*\)/\1/' \
-    | sort > "temp-filelist"
+    | sort -r > "temp-filelist"
   while read file; do
     eval "${1}[$count]=\"$file\""
     (( count++ ))
   done < "temp-filelist"
   rm "temp-filelist"
 }
+
+# List all directories in the current directory, stripping leading "./"
+list_dirs() {
+  count=0
+
+  find . -type d \
+    ! -name "." \
+    ! -name ".." \
+    | sed 's/\.\/\(.*\)/\1/' \
+    | sort -r > "temp-dirlist"
+  while read dir; do
+    eval "${1}[$count]=\"$dir\""
+    (( count++ ))
+  done < "temp-dirlist"
+  rm "temp-dirlist"
+}
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -27,59 +27,102 @@ if [ $1 = -h ]; then
   notice ""
   exit 1
 fi
 
 # -----------------------------------------------------------------------------
 
 archive="$1"
 targetdir="$2"
+# Prevent the workdir from being inside the targetdir so it isn't included in
+# the update mar.
+if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
+  # Remove the /
+  targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
+fi
 workdir="$targetdir.work"
-manifest="$workdir/update.manifest"
-targetfiles="update.manifest"
+updatemanifestv1="$workdir/update.manifest"
+updatemanifestv2="$workdir/updatev2.manifest"
+targetfiles="update.manifest updatev2.manifest"
 
 mkdir -p "$workdir"
 
+# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
+# Info.plist so it launches the right architecture, bug 600098
+
 # Generate a list of all files in the target directory.
 pushd "$targetdir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
-# On Mac, force a top-level file so that OS X reloads the Info.plist
-# and launches the right architecture for the OS version, bug 600098
-if [[ -d Contents ]]; then
-  touch force_plist_reload
+if [ ! -f "precomplete" ]; then
+  notice "precomplete file is missing!"
+  exit 1
 fi
 
 list_files files
 
+# Files that should be added on channel change
+ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
+
 popd
 
-> $manifest
+notice ""
+notice "Adding file add instructions to file 'update.manifest'"
+> $updatemanifestv1
 
 num_files=${#files[*]}
 
 for ((i=0; $i<$num_files; i=$i+1)); do
   f="${files[$i]}"
 
-  notice "processing $f"
-
-  make_add_instruction "$f" >> $manifest
+  make_add_instruction "$f" >> $updatemanifestv1
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
   $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
+# Add the type of update to the beginning of and cat the contents of the version
+# 1 update manifest to the version 2 update manifest.
+> $updatemanifestv2
+notice ""
+notice "Adding type instruction to file 'updatev2.manifest'"
+notice "       type: complete"
+echo "type \"complete\"" >> $updatemanifestv2
+
+notice ""
+notice "Adding file ADD instructions for channel change to file 'updatev2.manifest'"
+for f in $ccfiles; do
+  notice "     add-cc: $f"
+  echo "add-cc \"$f\"" >> $updatemanifestv2
+  dir=$(dirname "$f")
+  mkdir -p "$workdir/$dir"
+  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  copy_perm "$targetdir/$f" "$workdir/$f"
+
+  targetfiles="$targetfiles \"$f\""
+done
+
+notice ""
+notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
+cat $updatemanifestv1 >> $updatemanifestv2
+
 # Append remove instructions for any dead files.
-append_remove_instructions "$targetdir" >> $manifest
+notice ""
+notice "Adding file and directory remove instructions from file 'removed-files'"
+append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2"
 
-$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
+$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
+$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
+
+notice ""
+notice "Finished"
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -21,16 +21,21 @@ print_usage() {
 }
 
 check_for_forced_update() {
   force_list="$1"
   forced_file_chk="$2"
 
   local f
 
+  if [ "$forced_file_chk" = "precomplete" ]; then
+    ## "true" *giggle*
+    return 0;
+  fi
+
   if [ "${forced_file_chk##*.}" = "chk" ]
   then
     ## "true" *giggle*
     return 0;
   fi
 
   for f in $force_list; do
     #echo comparing $forced_file_chk to $f
@@ -65,122 +70,200 @@ done
 # -----------------------------------------------------------------------------
 
 let arg_start=$OPTIND-1
 shift $arg_start
 
 archive="$1"
 olddir="$2"
 newdir="$3"
+# Prevent the workdir from being inside the targetdir so it isn't included in
+# the update mar.
+if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
+  # Remove the /
+  newdir=$(echo "$newdir" | sed -e 's:\/$::')
+fi
 workdir="$newdir.work"
-manifest="$workdir/update.manifest"
-archivefiles="update.manifest"
+updatemanifestv1="$workdir/update.manifest"
+updatemanifestv2="$workdir/updatev2.manifest"
+archivefiles="update.manifest updatev2.manifest"
 
 mkdir -p "$workdir"
 
+# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
+# Info.plist so it launches the right architecture, bug 600098
+
 # Generate a list of all files in the target directory.
 pushd "$olddir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
 list_files oldfiles
+list_dirs olddirs
 
 popd
 
 pushd "$newdir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
+if [ ! -f "precomplete" ]; then
+  notice "precomplete file is missing!"
+  exit 1
+fi
+
+list_dirs newdirs
 list_files newfiles
 
+# Files that should be added on channel change
+ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
+
 popd
 
-> $manifest
+notice ""
+notice "Adding file patch and add instructions to file 'update.manifest'"
+> $updatemanifestv1
 
 num_oldfiles=${#oldfiles[*]}
+remove_array=
+num_removes=0
 
 for ((i=0; $i<$num_oldfiles; i=$i+1)); do
   f="${oldfiles[$i]}"
 
   # This file is created by Talkback, so we can ignore it
   if [ "$f" = "readme.txt" ]; then
     continue 1
   fi
 
+  # removed-files is excluded by make_incremental_updates.py so it is excluded
+  # here for consistency.
+  if [ `basename $f` = "removed-files" ]; then
+    continue 1
+  fi
+
   # If this file exists in the new directory as well, then check if it differs.
   if [ -f "$newdir/$f" ]; then
 
     if check_for_forced_update "$requested_forced_updates" "$f"; then
-      echo 1>&2 "  FORCING UPDATE for file '$f'..."
-      # The full workdir may not exist yet, so create it if necessary. 
+      # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
       $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
-      make_add_instruction "$f" >> $manifest
+      make_add_instruction "$f" 1 >> $updatemanifestv1
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
       # Compute both the compressed binary diff and the compressed file, and
       # compare the sizes.  Then choose the smaller of the two to package.
-      echo "  diffing $f"
       dir=$(dirname "$workdir/$f")
       mkdir -p "$dir"
       $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
       $BZIP2 -z9 "$workdir/$f.patch"
       $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
       patchfile="$workdir/$f.patch.bz2"
       patchsize=$(get_file_size "$patchfile")
       fullsize=$(get_file_size "$workdir/$f")
 
-      if [ $patchsize -lt $fullsize -a "$f" != "removed-files" ]; then
-        make_patch_instruction "$f" >> $manifest
+      if [ $patchsize -lt $fullsize ]; then
+        make_patch_instruction "$f" >> $updatemanifestv1
         mv -f "$patchfile" "$workdir/$f.patch"
         rm -f "$workdir/$f"
         archivefiles="$archivefiles \"$f.patch\""
       else
-        make_add_instruction "$f" >> $manifest
+        make_add_instruction "$f" >> $updatemanifestv1
         rm -f "$patchfile"
         archivefiles="$archivefiles \"$f\""
       fi
     fi
   else
-    echo "remove \"$f\"" >> $manifest
+    # remove instructions are added after add / patch instructions for
+    # consistency with make_incremental_updates.py
+    remove_array[$num_removes]=$f
+    (( num_removes++ ))
   fi
 done
 
-# Now, we just need to worry about newly added files
+# Newly added files
+notice ""
+notice "Adding file add instructions to file 'update.manifest'"
 num_newfiles=${#newfiles[*]}
 
 for ((i=0; $i<$num_newfiles; i=$i+1)); do
   f="${newfiles[$i]}"
 
+  # removed-files is excluded by make_incremental_updates.py so it is excluded
+  # here for consistency.
+  if [ `basename $f` = "removed-files" ]; then
+    continue 1
+  fi
+
   # If we've already tested this file, then skip it
   for ((j=0; $j<$num_oldfiles; j=$j+1)); do
     if [ "$f" = "${oldfiles[j]}" ]; then
       continue 2
     fi
   done
-  
+
   dir=$(dirname "$workdir/$f")
   mkdir -p "$dir"
 
   $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
   copy_perm "$newdir/$f" "$workdir/$f"
 
-  make_add_instruction "$f" >> "$manifest"
+  make_add_instruction "$f" >> "$updatemanifestv1"
   archivefiles="$archivefiles \"$f\""
 done
 
+notice ""
+notice "Adding file remove instructions to file 'update.manifest'"
+for ((i=0; $i<$num_removes; i=$i+1)); do
+  f="${remove_array[$i]}"
+  notice "     remove: $f"
+  echo "remove \"$f\"" >> $updatemanifestv1
+done
+
+# Add the type of update to the beginning of and cat the contents of the version
+# 1 update manifest to the version 2 update manifest.
+notice ""
+notice "Adding type instruction to file 'updatev2.manifest'"
+> $updatemanifestv2
+notice "       type: partial"
+echo "type \"partial\"" >> $updatemanifestv2
+
+notice ""
+notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
+cat $updatemanifestv1 >> $updatemanifestv2
+
 # Append remove instructions for any dead files.
-append_remove_instructions "$newdir" >> $manifest
+notice ""
+notice "Adding file and directory remove instructions from file 'removed-files'"
+append_remove_instructions "$newdir" "$updatemanifestv1" "$updatemanifestv2"
+
+notice ""
+notice "Adding directory remove instructions for directories that no longer exist"
+num_olddirs=${#olddirs[*]}
 
-$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
+for ((i=0; $i<$num_olddirs; i=$i+1)); do
+  f="${olddirs[$i]}"
+  # If this dir doesn't exist in the new directory remove it.
+  if [ ! -d "$newdir/$f" ]; then
+    notice "      rmdir: $f/"
+    echo "rmdir \"$f/\"" >> $updatemanifestv2
+  fi
+done
+
+$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
+$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
 
 eval "$MAR -C \"$workdir\" -c output.mar $archivefiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
+
+notice ""
+notice "Finished"
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -1,129 +1,166 @@
 import os
 import shutil
 import sha
-import sha
 from os.path import join, getsize
 from stat import *
 import re
 import sys
 import getopt
 import time
 import datetime
 import bz2
 import string
 import tempfile
 
 class PatchInfo:
     """ Represents the meta-data associated with a patch
         work_dir = working dir where files are stored for this patch
         archive_files = list of files to include in this patch
-        manifest = set of patch instructions
+        manifestv1 = set of manifest version 1 patch instructions
+        manifestv2 = set of manifest version 2 patch instructions
         file_exclusion_list = 
         files to exclude from this patch. names without slashes will be
                               excluded anywhere in the directory hiearchy.   names with slashes
                               will only be excluded at that exact path
         """
     def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
         self.work_dir=work_dir
         self.archive_files=[]
-        self.manifest=[]
+        self.manifestv1=[]
+        self.manifestv2=[]
         self.file_exclusion_list=file_exclusion_list
         self.path_exclusion_list=path_exclusion_list
         
     def append_add_instruction(self, filename):
         """ Appends an add instruction for this patch.   
             if the filename starts with extensions/ adds an add-if instruction
             to test the existence of the subdirectory.  This was ported from
             mozilla/tools/update-packaging/common.sh/make_add_instruction
         """
         if filename.startswith("extensions/"):
             # Directory immediately following extensions is used for the test
             testdir = "extensions/"+filename.split("/")[1]
-            self.manifest.append('add-if "'+testdir+'" "'+filename+'"')
+            self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"')
+            self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
         elif filename.startswith("Contents/MacOS/extensions/"):
             testdir = "Contents/MacOS/extensions/"+filename.split("/")[3]
-            self.manifest.append('add-if "'+testdir+'" "'+filename+'"')
+            self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"')
+            self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
         else:
-            self.manifest.append('add "'+filename+'"')
+            self.manifestv1.append('add "'+filename+'"')
+            self.manifestv2.append('add "'+filename+'"')
            
     def append_patch_instruction(self, filename, patchname):
         """ Appends an patch instruction for this patch.   
             
             filename = file to patch
             patchname = patchfile to apply to file
             
             if the filename starts with extensions/ adds a patch-if instruction
             to test the existence of the subdirectory.  
             if the filename starts with searchplugins/ add a add-if instruction for the filename
             This was ported from
             mozilla/tools/update-packaging/common.sh/make_patch_instruction
         """
         if filename.startswith("extensions/"):
             testdir = "extensions/"+filename.split("/")[1]
-            self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
         elif filename.startswith("Contents/MacOS/extensions/"):
             testdir = "Contents/MacOS/extensions/"+filename.split("/")[3]
-            self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
         elif (filename.startswith("searchplugins/") or
              filename.startswith("Contents/MacOS/searchplugins/")):
-            self.manifest.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv1.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
+            self.manifestv2.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
         else:
-            self.manifest.append('patch "'+patchname+'" "'+filename+'"')
+            self.manifestv1.append('patch "'+patchname+'" "'+filename+'"')
+            self.manifestv2.append('patch "'+patchname+'" "'+filename+'"')
                 
     def append_remove_instruction(self, filename):
         """ Appends an remove instruction for this patch.   
             This was ported from
             mozilla/tools/update-packaging/common.sh/make_remove_instruction
         """
-        self.manifest.append('remove "'+filename+'"')
+        if filename.endswith("/"):
+            self.manifestv2.append('rmdir "'+filename+'"')
+        elif filename.endswith("/*"):
+            filename = filename[:-1]
+            self.manifestv2.append('rmrfdir "'+filename+'"')
+        else:
+            self.manifestv1.append('remove "'+filename+'"')
+            self.manifestv2.append('remove "'+filename+'"')
 
-    def create_manifest_file(self):
-        """ Createst the manifest file into to the root of the work_dir """
+    def create_manifest_files(self):
+        """ Createst the v1 manifest file in the root of the work_dir """
         manifest_file_path = os.path.join(self.work_dir,"update.manifest")
-        manifest_file = open(manifest_file_path, "w")
-        manifest_file.writelines(string.join(self.manifest, '\n'))
+        manifest_file = open(manifest_file_path, "wb")
+        manifest_file.writelines(string.join(self.manifestv1, '\n'))
         manifest_file.writelines("\n")
         manifest_file.close()    
 
         bzip_file(manifest_file_path)
         self.archive_files.append('"update.manifest"')
 
+        """ Createst the v2 manifest file in the root of the work_dir """
+        manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
+        manifest_file = open(manifest_file_path, "wb")
+        manifest_file.writelines("type \"partial\"\n")
+        manifest_file.writelines(string.join(self.manifestv2, '\n'))
+        manifest_file.writelines("\n")
+        manifest_file.close()
+
+        bzip_file(manifest_file_path)
+        self.archive_files.append('"updatev2.manifest"')
 
     def build_marfile_entry_hash(self, root_path):
-         """ Iterates through the root_path, creating a MarFileEntry for each file
-             in that path.  Excluseds any filenames in the file_exclusion_list
-         """
-         mar_entry_hash = {}
-         filename_set = set()
-         for root, dirs, files in os.walk(root_path):
-             for name in files:
-                 # filename is relative path from root directory
-                 partial_path = root[len(root_path)+1:]
-                 if name not in self.file_exclusion_list:
-                     filename = os.path.join(partial_path, name)
-                     if "/"+filename not in self.path_exclusion_list:
-                         mar_entry_hash[filename]=MarFileEntry(root_path, filename)
-                         filename_set.add(filename)
-         return mar_entry_hash, filename_set
+        """ Iterates through the root_path, creating a MarFileEntry for each file
+            and directory in that path.  Excludes any filenames in the file_exclusion_list
+        """
+        mar_entry_hash = {}
+        filename_set = set()
+        dirname_set = set()
+        for root, dirs, files in os.walk(root_path):
+            for name in files:
+                # filename is the relative path from root directory
+                partial_path = root[len(root_path)+1:]
+                if name not in self.file_exclusion_list:
+                    filename = os.path.join(partial_path, name)
+                    if "/"+filename not in self.path_exclusion_list:
+                        mar_entry_hash[filename]=MarFileEntry(root_path, filename)
+                        filename_set.add(filename)
+
+            for name in dirs:
+                # dirname is the relative path from root directory
+                partial_path = root[len(root_path)+1:]
+                if name not in self.file_exclusion_list:
+                    dirname = os.path.join(partial_path, name)
+                    if "/"+dirname not in self.path_exclusion_list:
+                        dirname = dirname+"/"
+                        mar_entry_hash[dirname]=MarFileEntry(root_path, dirname)
+                        dirname_set.add(dirname)
+
+        return mar_entry_hash, filename_set, dirname_set
  
        
 class MarFileEntry:
     """Represents a file inside a Mozilla Archive Format (MAR)
         abs_path = abspath to the the file
         name =  relative path within the mar.  e.g.
           foo.mar/dir/bar.txt extracted into /tmp/foo:
             abs_path=/tmp/foo/dir/bar.txt
             name = dir/bar.txt
     """ 
     def __init__(self, root, name):
         """root = path the the top of the mar
            name = relative path within the mar"""
-        self.name=name
+        self.name=name.replace("\\", "/")
         self.abs_path=os.path.join(root,name)
         self.sha_cache=None
 
     def __str__(self):
         return 'Name: %s FullPath: %s' %(self.name,self.abs_path)
 
     def calc_file_sha_digest(self, filename):        
         """ Returns sha digest of given filename"""
@@ -214,20 +251,20 @@ def create_partial_patch_for_file(from_m
             os.remove(patch_file_abs_path)
             file_in_manifest_name = from_marfile_entry.name
             file_in_manifest_abspath = full_file_abs_path
             patch_info.append_add_instruction(file_in_manifest_name)
             
         shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath)
         patch_info.archive_files.append('"'+file_in_manifest_name+'"')        
     else:
-        print "skipping diff: " + from_marfile_entry.name
         filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()]
         # We've already calculated the patch for this pair of files.   
         if (filename.endswith(".patch")):
+            print "skipping diff: " + from_marfile_entry.name
             # Patch was smaller than file - add patch instruction to manifest
             file_in_manifest_name = to_marfile_entry.name+'.patch';
             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
         else:
             # File was smaller than file - add file to manifest
             file_in_manifest_name = to_marfile_entry.name
             patch_info.append_add_instruction(file_in_manifest_name)                
         # Copy the pre-calculated file into our new patch work aread
@@ -245,72 +282,88 @@ def process_explicit_remove_files(dir_pa
     """ Looks for a 'removed-files' file in the dir_path.  If the removed-files does not exist
     this will throw.  If found adds the removed-files
     found in that file to the patch_info"""
 
     # Windows and linux have this file at the root of the dir
     list_file_path = os.path.join(dir_path, "removed-files")
     prefix=""
     if not os.path.exists(list_file_path):
-        # Mac has is in Contents/MacOS/
+        # On Mac removed-files contains relative paths from Contents/MacOS/
         prefix= "Contents/MacOS"
         list_file_path = os.path.join(dir_path, prefix+"/removed-files")
 
     if (os.path.exists(list_file_path)):
         list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
 
+        lines = []
         for line in list_file:
-            line = line.strip()
-            # Exclude any blank lines or any lines ending with a slash, which indicate
-            # directories.  The updater doesn't know how to remove entire directories.
-            if line and not line.endswith("/"): 
-                patch_info.append_remove_instruction(os.path.join(prefix,line))
+            lines.append(line.strip())
+
+        lines.sort(reverse=True)
+        for line in lines:
+            # Exclude any blank and comment lines.
+            if line and not line.startswith("#"):
+                # Python on windows uses \ for path separators and the update
+                # manifests expects / for path separators on all platforms.
+                line=os.path.join(prefix,line).replace("\\", "/")
+                patch_info.append_remove_instruction(line)
 
 def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates):
     """ Builds a partial patch by comparing the files in from_dir_path to thoes of to_dir_path"""
     # Cannocolize the paths for safey
     from_dir_path = os.path.abspath(from_dir_path)
     to_dir_path = os.path.abspath(to_dir_path)
-    # First create a hashtable of the from  and to directories
-    from_dir_hash,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
-    to_dir_hash,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
+    # Create a hashtable of the from  and to directories
+    from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
+    to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
+    # Require that the precomplete file is included in the to complete update
+    if "precomplete" not in to_file_set:
+        raise Exception, "missing precomplete file in: "+to_dir_path
     # Create a list of the forced updates 
     forced_list = forced_updates.strip().split('|')
-    
+    forced_list.append("precomplete")
+
     # Files which exist in both sets need to be patched
-    patch_filenames = list(from_dir_set.intersection(to_dir_set))
-    patch_filenames.sort()
+    patch_filenames = list(from_file_set.intersection(to_file_set))
+    patch_filenames.sort(reverse=True)
     for filename in patch_filenames:
         from_marfile_entry = from_dir_hash[filename]
         to_marfile_entry = to_dir_hash[filename]
         if filename in forced_list:
             print "Forcing "+ filename
             # This filename is in the forced list, explicitly add
-       	    create_add_patch_for_file(to_dir_hash[filename], patch_info)
+            create_add_patch_for_file(to_dir_hash[filename], patch_info)
         else: 
           if from_marfile_entry.sha() != to_marfile_entry.sha():
               # Not the same - calculate a patch
               create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info)
 
+    # files in to_dir not in from_dir need to added
+    add_filenames = list(to_file_set - from_file_set)
+    add_filenames.sort(reverse=True)
+    for filename in add_filenames:
+        create_add_patch_for_file(to_dir_hash[filename], patch_info)
+
     # files in from_dir not in to_dir need to be removed
-    remove_filenames = list(from_dir_set - to_dir_set)
-    remove_filenames.sort()
+    remove_filenames = list(from_file_set - to_file_set)
+    remove_filenames.sort(reverse=True)
     for filename in remove_filenames:
         patch_info.append_remove_instruction(from_dir_hash[filename].name)
 
-    # files in to_dir not in from_dir need to added
-    add_filenames = list(to_dir_set - from_dir_set)
-    add_filenames.sort()
-    for filename in add_filenames:
-        create_add_patch_for_file(to_dir_hash[filename], patch_info)
-
     process_explicit_remove_files(to_dir_path, patch_info)
     
-    # Construct Manifest file
-    patch_info.create_manifest_file()
+    # directories in from_dir not in to_dir need to be removed
+    remove_dirnames = list(from_dir_set - to_dir_set)
+    remove_dirnames.sort(reverse=True)
+    for dirname in remove_dirnames:
+        patch_info.append_remove_instruction(from_dir_hash[dirname].name)
+
+    # Construct the Manifest files
+    patch_info.create_manifest_files()
     
     # And construct the mar
     mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')    
     exec_shell_cmd(mar_cmd)
 
     # Copy mar to final destination
     patch_file_dir = os.path.split(patch_filename)[0]
     if not os.path.exists(patch_file_dir):
@@ -397,17 +450,17 @@ def create_partial_patches(patches):
             extract_mar(to_filename, work_dir_to)
             to_decoded = decode_filename(from_filename)
             to_buildid = get_buildid(work_dir_to, to_decoded['platform'])
             to_shasum = sha.sha(open(to_filename).read()).hexdigest()
             to_size = str(os.path.getsize(to_filename))
 
             mar_extract_time = time.time()
 
-            partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt']),forced_updates)
+            partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt']),forced_updates)
             partial_buildid = to_buildid
             partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
             partial_size = str(os.path.getsize(partial_filename))
 
             metadata.append({
              'to_filename': os.path.basename(to_filename),
              'from_filename': os.path.basename(from_filename),
              'partial_filename': os.path.basename(partial_filename),
--- a/tools/update-packaging/test/buildrefmars.sh
+++ b/tools/update-packaging/test/buildrefmars.sh
@@ -1,16 +1,26 @@
 #!/bin/bash
 # Builds all the reference mars
 
-rm ref.mar
-rm ref-mac.mar
+if [ -f "ref.mar" ]; then
+  rm "ref.mar"
+fi
+if [ -f "ref-mac.mar" ]; then
+  rm "ref-mac.mar"
+fi
 
  ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to
  ../make_incremental_update.sh ref-mac.mar `pwd`/from `pwd`/to-mac
 
-rm product-1.0.lang.platform.complete.mar
-rm product-2.0.lang.platform.complete.mar
-rm product-2.0.lang.mac.complete.mar
+if [ -f "product-1.0.lang.platform.complete.mar" ]; then
+  rm "product-1.0.lang.platform.complete.mar"
+fi
+if [ -f "product-2.0.lang.platform.complete.mar" ]; then
+  rm "product-2.0.lang.platform.complete.mar"
+fi
+if [ -f "product-2.0.lang.mac.complete.mar" ]; then
+  rm "product-2.0.lang.mac.complete.mar"
+fi
 
-./make_full_update.sh product-1.0.lang.platform.complete.mar `pwd`/from
-./make_full_update.sh product-2.0.lang.platform.complete.mar `pwd`/to 
-./make_full_update.sh product-2.0.lang.mac.complete.mar `pwd`/to-mac 
+./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from"
+./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to"
+./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac"
--- a/tools/update-packaging/test/catmanifest.sh
+++ b/tools/update-packaging/test/catmanifest.sh
@@ -6,9 +6,9 @@ workdir="/tmp/catmanifest"
 
 rm -rf "$workdir"
 mkdir -p "$workdir"
 cp "$1" "$workdir"
 cd "$workdir"
 mar -x "$1"
 mv update.manifest update.manifest.bz2
 bzip2 -d update.manifest.bz2
-cat update.manifest
\ No newline at end of file
+cat update.manifest
--- a/tools/update-packaging/test/common.sh
+++ b/tools/update-packaging/test/common.sh
@@ -10,17 +10,17 @@
 MAR=${MAR:-mar}
 BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
-  echo $* 1>&2
+  echo "$*" 1>&2
 }
 
 get_file_size() {
   info=($(ls -ln "$1"))
   echo ${info[4]}
 }
 
 copy_perm() {
@@ -31,82 +31,137 @@ copy_perm() {
     chmod 0755 "$target"
   else
     chmod 0644 "$target"
   fi
 }
 
 make_add_instruction() {
   f="$1"
+
+  # Used to log to the console
+  if [ $2 ]; then
+    forced=" (forced)"
+  else
+    forced=
+  fi
+
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "     add-if: $f$forced"
     echo "add-if \"$testdir\" \"$f\""
   else
+    notice "        add: $f$forced"
     echo "add \"$f\""
   fi
 }
 
 make_patch_instruction() {
   f="$1"
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   is_search_plugin=$(echo "$f" | grep -c 'searchplugins/.*')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "   patch-if: $f"
     echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
   elif [ $is_search_plugin = "1" ]; then
+    notice "   patch-if: $f"
     echo "patch-if \"$f\" \"$f.patch\" \"$f\""
   else
+    notice "      patch: $f"
     echo "patch \"$f.patch\" \"$f\""
   fi
 }
 
 append_remove_instructions() {
   dir="$1"
+  filev1="$2"
+  filev2="$3"
   if [ -f "$dir/removed-files" ]; then
     prefix=
     listfile="$dir/removed-files"
   elif [ -f "$dir/Contents/MacOS/removed-files" ]; then
     prefix=Contents/MacOS/
     listfile="$dir/Contents/MacOS/removed-files"
   fi
   if [ -n "$listfile" ]; then
     # Map spaces to pipes so that we correctly handle filenames with spaces.
-    files=($(cat "$listfile" | tr " " "|"))  
+    files=($(cat "$listfile" | tr " " "|"  | sort -r))
     num_files=${#files[*]}
     for ((i=0; $i<$num_files; i=$i+1)); do
-      # Trim whitespace (including trailing carriage returns)
-      f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r')
-      # Exclude any blank lines or any lines ending with a slash, which indicate
-      # directories.  The updater doesn't know how to remove entire directories.
+      # Map pipes back to whitespace and remove carriage returns
+      f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
+      # Trim whitespace
+      f=$(echo $f)
+      # Exclude blank lines.
       if [ -n "$f" ]; then
-        if [ $(echo "$f" | grep -c '\/$') = 0 ]; then
-          echo "remove \"$prefix$f\""
-        else
-          notice "ignoring remove instruction for directory: $f"
+        # Exclude comments
+        if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
+          # Normalize the path to the root of the Mac OS X bundle if necessary
+          fixedprefix="$prefix"
+          if [ $prefix ]; then
+            if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then
+              if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then
+                f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::')
+                fixedprefix=""
+              else
+                f=$(echo $f | sed -e 's:^\.\.\/::')
+                fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::')
+              fi
+            fi
+          fi
+          if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
+            notice "      rmdir: $fixedprefix$f"
+            echo "rmdir \"$fixedprefix$f\"" >> $filev2
+          elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
+            # Remove the *
+            f=$(echo "$f" | sed -e 's:\*$::')
+            notice "    rmrfdir: $fixedprefix$f"
+            echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
+          else
+            notice "     remove: $fixedprefix$f"
+            echo "remove \"$fixedprefix$f\"" >> $filev1
+            echo "remove \"$fixedprefix$f\"" >> $filev2
+          fi
         fi
       fi
     done
   fi
 }
 
 # List all files in the current directory, stripping leading "./"
 # Skip the channel-prefs.js file as it should not be included in any
 # generated MAR files (see bug 306077). Pass a variable name and it will be
 # filled as an array.
 list_files() {
   count=0
 
-  # Schrep - removed the exclusion cases here to allow for generation
-  # of testing mars
+  # Removed the exclusion cases here to allow for generation of testing mars
   find . -type f \
     | sed 's/\.\/\(.*\)/\1/' \
-    | sort > "$workdir/temp-filelist"
+    | sort -r > "$workdir/temp-filelist"
   while read file; do
     eval "${1}[$count]=\"$file\""
     (( count++ ))
   done < "$workdir/temp-filelist"
   rm "$workdir/temp-filelist"
 }
+
+# List all directories in the current directory, stripping leading "./"
+list_dirs() {
+  count=0
+
+  find . -type d \
+    ! -name "." \
+    ! -name ".." \
+    | sed 's/\.\/\(.*\)/\1/' \
+    | sort -r > "$workdir/temp-dirlist"
+  while read dir; do
+    eval "${1}[$count]=\"$dir\""
+    (( count++ ))
+  done < "$workdir/temp-dirlist"
+  rm "$workdir/temp-dirlist"
+}
--- a/tools/update-packaging/test/diffmar.sh
+++ b/tools/update-packaging/test/diffmar.sh
@@ -2,34 +2,49 @@
 # Compares two mars
 
 marA="$1"
 marB="$2"
 workdir="/tmp/diffmar"
 fromdir="$workdir/0"
 todir="$workdir/1"
 
+# On Windows, creation time can be off by a second or more between the files in
+# the fromdir and todir due to them being extracted synchronously so use
+# time-style and exclude seconds from the creation time.
+lsargs="-algR"
+unamestr=`uname`
+if [ ! "$unamestr" = 'Darwin' ]; then
+  unamestr=`uname -o`
+  if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then
+     lsargs="-algR --time-style=+%Y-%m-%d-%H:%M"
+  fi
+fi
+
 rm -rf "$workdir"
 mkdir -p "$fromdir"
 mkdir -p "$todir"
 
 cp "$1" "$fromdir"
 cp "$2" "$todir"
 
 cd "$fromdir"
 mar -x "$1"
 rm "$1"
 mv update.manifest update.manifest.bz2
 bzip2 -d update.manifest.bz2
-ls -algR > files.txt
-# Sort the manifest so we don't get any diffs for ordering
-#cat update.manifest | sort > update.manifest
+mv updatev2.manifest updatev2.manifest.bz2
+bzip2 -d updatev2.manifest.bz2
+ls $lsargs > files.txt
 
 cd "$todir"
 mar -x "$2"
 rm "$2"
 mv update.manifest update.manifest.bz2
 bzip2 -d update.manifest.bz2
-# Sort the manifest so we don't get any diffs for ordering
-#cat update.manifest | sort > update.manifest
-ls -algR > files.txt
+mv updatev2.manifest updatev2.manifest.bz2
+bzip2 -d updatev2.manifest.bz2
+ls $lsargs > files.txt
 
-diff -r "$fromdir" "$todir"
\ No newline at end of file
+echo "diffing $fromdir and $todir"
+echo "on linux shell sort and python sort return different results"
+echo "which cause differences in the manifest files"
+diff -ru "$fromdir" "$todir"
new file mode 100644
--- /dev/null
+++ b/tools/update-packaging/test/from/precomplete
@@ -0,0 +1,21 @@
+remove {foodir/update.manifest
+remove {foodir/same.txt
+remove {foodir/same.bin
+remove {foodir/removed.txt
+remove {foodir/readme.txt
+remove {foodir/force.txt
+remove {foodir/diff-patch-larger-than-file.txt
+remove-cc {foodir/channel-prefs.js
+remove update.manifest
+remove searchplugins/diff/diff-patch-larger-than-file.txt
+remove same.txt
+remove same.bin
+remove removed.txt
+remove readme.txt
+remove precomplete
+remove force.txt
+remove diff-patch-larger-than-file.txt
+remove-cc channel-prefs.js
+rmdir {foodir/
+rmdir searchplugins/diff/
+rmdir searchplugins/
--- a/tools/update-packaging/test/make_full_update.sh
+++ b/tools/update-packaging/test/make_full_update.sh
@@ -28,53 +28,102 @@ if [ $1 = -h ]; then
   notice ""
   exit 1
 fi
 
 # -----------------------------------------------------------------------------
 
 archive="$1"
 targetdir="$2"
+# Prevent the workdir from being inside the targetdir so it isn't included in
+# the update mar.
+if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
+  # Remove the /
+  targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
+fi
 workdir="$targetdir.work"
-manifest="$workdir/update.manifest"
-targetfiles="update.manifest"
+updatemanifestv1="$workdir/update.manifest"
+updatemanifestv2="$workdir/updatev2.manifest"
+targetfiles="update.manifest updatev2.manifest"
 
 mkdir -p "$workdir"
 
+# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
+# Info.plist so it launches the right architecture, bug 600098
+
 # Generate a list of all files in the target directory.
 pushd "$targetdir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
+if [ ! -f "precomplete" ]; then
+  notice "precomplete file is missing!"
+  exit 1
+fi
+
 list_files files
 
+# Files that should be added on channel change
+ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
+
 popd
 
-> $manifest
+notice ""
+notice "Adding file add instructions to file 'update.manifest'"
+> $updatemanifestv1
 
 num_files=${#files[*]}
 
 for ((i=0; $i<$num_files; i=$i+1)); do
   f="${files[$i]}"
 
-  notice "processing $f"
-
-  make_add_instruction "$f" >> $manifest
+  make_add_instruction "$f" >> $updatemanifestv1
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
   $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
+# Add the type of update to the beginning of and cat the contents of the version
+# 1 update manifest to the version 2 update manifest.
+> $updatemanifestv2
+notice ""
+notice "Adding type instruction to file 'updatev2.manifest'"
+notice "       type: complete"
+echo "type \"complete\"" >> $updatemanifestv2
+
+notice ""
+notice "Adding file add on channel change instructions to file 'updatev2.manifest'"
+for f in $ccfiles; do
+  notice "     add-cc: $f"
+  echo "add-cc \"$f\"" >> $updatemanifestv2
+  dir=$(dirname "$f")
+  mkdir -p "$workdir/$dir"
+  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  copy_perm "$targetdir/$f" "$workdir/$f"
+
+  targetfiles="$targetfiles \"$f\""
+done
+
+notice ""
+notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
+cat $updatemanifestv1 >> $updatemanifestv2
+
 # Append remove instructions for any dead files.
-append_remove_instructions "$targetdir" >> $manifest
+notice ""
+notice "Adding file and directory remove instructions from file 'removed-files'"
+append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2"
 
-$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
+$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
+$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
+
+notice ""
+notice "Finished"
--- a/tools/update-packaging/test/runtests.sh
+++ b/tools/update-packaging/test/runtests.sh
@@ -2,18 +2,8 @@
 
 echo "testing make_incremental_updates.py"
 python ../make_incremental_updates.py -f testpatchfile.txt
 
 echo "diffing ref.mar and test.mar"
 ./diffmar.sh ref.mar test.mar
 echo "diffing ref-mac.mar and test-mac.mar"
 ./diffmar.sh ref-mac.mar test-mac.mar
-
-
-echo "testing make_incremental_updates_mar.py"
-python ../make_incremental_updates_mar.py -f testpatchfile.txt
-
-echo "diffing ref.mar and test.mar"
-./diffmar.sh ref.mar test.mar
-echo "diffing ref-mac.mar and test-mac.mar"
-./diffmar.sh ref-mac.mar test-mac.mar
-
new file mode 100644
--- /dev/null
+++ b/tools/update-packaging/test/to-mac/precomplete
@@ -0,0 +1,4 @@
+remove precomplete
+remove Contents/MacOS/removed-files
+rmdir Contents/MacOS/
+rmdir Contents/
new file mode 100644
--- /dev/null
+++ b/tools/update-packaging/test/to/precomplete
@@ -0,0 +1,21 @@
+remove {foodir/update.manifest
+remove {foodir/same.txt
+remove {foodir/same.bin
+remove {foodir/readme.txt
+remove {foodir/force.txt
+remove {foodir/diff-patch-larger-than-file.txt
+remove-cc {foodir/channel-prefs.js
+remove update.manifest
+remove searchplugins/diff/diff-patch-larger-than-file.txt
+remove same.txt
+remove same.bin
+remove removed-files
+remove readme.txt
+remove precomplete
+remove force.txt
+remove diff-patch-larger-than-file.txt
+remove-cc channel-prefs.js
+rmdir {foodir/
+rmdir searchplugins/diff/
+rmdir searchplugins/added/
+rmdir searchplugins/
--- a/tools/update-packaging/test_make_incremental_updates.py
+++ b/tools/update-packaging/test_make_incremental_updates.py
@@ -2,46 +2,47 @@
 
 import unittest
 import make_incremental_updates as mkup
 from make_incremental_updates import PatchInfo, MarFileEntry
 
 class TestPatchInfo(unittest.TestCase):
     def setUp(self):
         self.work_dir = 'work_dir'
-        self.file_exclusion_list = ['channel-prefs.js','update.manifest','removed-files']
+        self.file_exclusion_list = ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files']
         self.path_exclusion_list = ['/readme.txt']
         self.patch_info = PatchInfo(self.work_dir, self.file_exclusion_list, self.path_exclusion_list)
 
     def testPatchInfo(self):
         self.assertEquals(self.work_dir, self.patch_info.work_dir)
         self.assertEquals([], self.patch_info.archive_files)
-        self.assertEquals([], self.patch_info.manifest)
+        self.assertEquals([], self.patch_info.manifestv1)
+        self.assertEquals([], self.patch_info.manifestv2)
         self.assertEquals(self.file_exclusion_list, self.patch_info.file_exclusion_list)
         self.assertEquals(self.path_exclusion_list, self.patch_info.path_exclusion_list)
 
     def test_append_add_instruction(self):
         self.patch_info.append_add_instruction('file.test')
-        self.assertEquals(['add "file.test"'], self.patch_info.manifest)
+        self.assertEquals(['add "file.test"'], self.patch_info.manifestv1)
 
     def test_append_patch_instruction(self):
         self.patch_info.append_patch_instruction('file.test', 'patchname')
-        self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifest)
+        self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv1)
 
     """ FIXME touches the filesystem, need refactoring
     def test_append_remove_instruction(self):
         self.patch_info.append_remove_instruction('file.test')
-        self.assertEquals(['remove "file.test"'], self.patch_info.manifest)
+        self.assertEquals(['remove "file.test"'], self.patch_info.manifestv1)
 
     def test_create_manifest_file(self):
         self.patch_info.create_manifest_file()
     """
 
     def test_build_marfile_entry_hash(self):
-        self.assertEquals(({}, set([])), self.patch_info.build_marfile_entry_hash('root_path'))
+        self.assertEquals(({}, set([]), set([])), self.patch_info.build_marfile_entry_hash('root_path'))
 
 """ FIXME touches the filesystem, need refactoring
 class TestMarFileEntry(unittest.TestCase):
     def setUp(self):
         root_path = '.'
         self.filename = 'file.test'
         f = open(self.filename, 'w')
         f.write('Test data\n')
@@ -61,17 +62,17 @@ class TestMarFileEntry(unittest.TestCase
         f.close()
         sha = self.mar_file_entry.sha()
         self.assertEquals(goodSha, sha)
 """
 
 class TestMakeIncrementalUpdates(unittest.TestCase):
     def setUp(self):
         work_dir = '.'
-        self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt'])
+        self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt'])
         root_path = '/'
         filename = 'test.file'
         self.mar_file_entry = MarFileEntry(root_path, filename)
 
     """ FIXME makes direct shell calls, need refactoring
     def test_exec_shell_cmd(self):
         mkup.exec_shell_cmd('echo test')