author | Robert Strong <robert.bugzilla@gmail.com> |
Mon, 11 Apr 2011 21:23:18 -0700 | |
changeset 67955 | 4ceb3de77444dd4c97b9ec6f21a45785f7793ed5 |
parent 67954 | 3ac2e9681421f3348f482832579dbb66be9fc431 |
child 67956 | 860d857e899db6f7512a7a873c47391bdfafa16b |
push id | 1 |
push user | root |
push date | Tue, 26 Apr 2011 22:38:44 +0000 |
treeherder | mozilla-beta@bfdb6e623a36 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | nthomas, khuey |
bugs | 1, 386760 |
milestone | 2.2a1pre |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
new file mode 100644 --- /dev/null +++ b/config/createprecomplete.py @@ -0,0 +1,64 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +# Creates the precomplete file containing the remove, remove-cc, and rmdir +# application update instructions which is used to remove files and directories +# that are no longer present in a complete update. The current working directory +# is used for the location to enumerate and to create the precomplete file. + +import sys +import os + +def get_build_entries(root_path): + """ Iterates through the root_path, creating a list for each file and + directory. Excludes any path starting with extensions or distribution. + """ + rel_file_path_set = set() + rel_dir_path_set = set() + for root, dirs, files in os.walk(root_path): + for file_name in files: + parent_dir_rel_path = root[len(root_path)+1:] + rel_path_file = os.path.join(parent_dir_rel_path, file_name) + rel_path_file = rel_path_file.replace("\\", "/") + if not (rel_path_file.startswith("distribution/") or + rel_path_file.startswith("extensions/")): + rel_file_path_set.add(rel_path_file) + + for dir_name in dirs: + parent_dir_rel_path = root[len(root_path)+1:] + rel_path_dir = os.path.join(parent_dir_rel_path, dir_name) + rel_path_dir = rel_path_dir.replace("\\", "/")+"/" + if not (rel_path_dir.startswith("distribution/") or + rel_path_dir.startswith("extensions/")): + rel_dir_path_set.add(rel_path_dir) + + rel_file_path_list = list(rel_file_path_set) + rel_file_path_list.sort(reverse=True) + rel_dir_path_list = list(rel_dir_path_set) + rel_dir_path_list.sort(reverse=True) + + return rel_file_path_list, rel_dir_path_list + +def generate_precomplete(): + """ Creates the precomplete file containing the remove, remove-cc, and rmdir + application update instructions. The current working directory is used + for the location to enumerate and to create the precomplete file. + """ + root_path = os.getcwd() + rel_file_path_list, rel_dir_path_list = get_build_entries(root_path) + precomplete_file_path = os.path.join(root_path,"precomplete") + # open in binary mode to prevent OS specific line endings. + precomplete_file = open(precomplete_file_path, "wb") + for rel_file_path in rel_file_path_list: + if rel_file_path.endswith("channel-prefs.js"): + precomplete_file.writelines("remove-cc \""+rel_file_path+"\"\n") + else: + precomplete_file.writelines("remove \""+rel_file_path+"\"\n") + + for rel_dir_path in rel_dir_path_list: + precomplete_file.writelines("rmdir \""+rel_dir_path+"\"\n") + + precomplete_file.close() + +if __name__ == "__main__": + generate_precomplete()
--- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -18,16 +18,17 @@ # Netscape Communications Corporation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. # # Contributor(s): # Benjamin Smedberg <bsmedberg@covad.net> # Arthur Wiebe <artooro@gmail.com> # Mark Mentovai <mark@moxienet.com> +# Robert Strong <robert.bugzilla@gmail.com> # # Alternatively, the contents of this file may be used under the terms of # either of the GNU General Public License Version 2 or later (the "GPL"), # or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your @@ -373,16 +374,18 @@ INNER_UNMAKE_PACKAGE = \ SDK_SUFFIX = .tar.bz2 SDK = $(MOZ_PKG_APPNAME)-$(MOZ_PKG_VERSION).$(AB_CD).mac-$(TARGET_CPU).sdk$(SDK_SUFFIX) ifeq ($(MOZ_APP_NAME),xulrunner) SDK = $(SDK_PATH)$(MOZ_APP_NAME)-$(MOZ_PKG_VERSION).$(AB_CD).mac-$(TARGET_CPU).sdk$(SDK_SUFFIX) endif MAKE_SDK = $(CREATE_FINAL_TAR) - $(MOZ_APP_NAME)-sdk | bzip2 -vf > $(SDK) endif +CREATE_PRECOMPLETE = $(PYTHON) $(MOZILLA_DIR)/config/createprecomplete.py + ifdef MOZ_OMNIJAR GENERATE_CACHE ?= true OMNIJAR_FILES = \ chrome \ chrome.manifest \ components/*.js \ components/*.xpt \ @@ -414,20 +417,21 @@ PACK_OMNIJAR = \ printf "manifest components/binary.manifest\n" > chrome.manifest UNPACK_OMNIJAR = \ $(OPTIMIZE_JARS_CMD) --deoptimize $(_ABS_DIST)/jarlog/ ./ ./ && \ unzip -o omni.jar && \ rm -f components/binary.manifest && \ sed -e 's/^\#binary-component/binary-component/' components/components.manifest > components.manifest && \ mv components.manifest components -MAKE_PACKAGE = (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && $(INNER_MAKE_PACKAGE) +MAKE_PACKAGE = (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && \ + (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE) UNMAKE_PACKAGE = $(INNER_UNMAKE_PACKAGE) && (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(UNPACK_OMNIJAR)) else -MAKE_PACKAGE = $(INNER_MAKE_PACKAGE) +MAKE_PACKAGE = (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE) UNMAKE_PACKAGE = $(INNER_UNMAKE_PACKAGE) endif # dummy macro if we don't have PSM built SIGN_NSS = ifneq (1_,$(if $(CROSS_COMPILE),1,0)_$(UNIVERSAL_BINARY)) ifdef MOZ_PSM SIGN_NSS = @echo signing nss libraries; @@ -560,16 +564,17 @@ ifndef MOZ_PKG_MANIFEST endif @rm -rf $(DEPTH)/installer-stage $(DIST)/xpt @echo "Staging installer files..." @$(NSINSTALL) -D $(DEPTH)/installer-stage/core ifdef MOZ_OMNIJAR @(cd $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) endif @cp -av $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH)/. $(DEPTH)/installer-stage/core + @(cd $(DEPTH)/installer-stage/core && $(CREATE_PRECOMPLETE)) ifdef MOZ_OPTIONAL_PKG_LIST @$(NSINSTALL) -D $(DEPTH)/installer-stage/optional $(call PACKAGER_COPY, "$(call core_abspath,$(DIST))",\ "$(call core_abspath,$(DEPTH)/installer-stage/optional)", \ "$(MOZ_PKG_MANIFEST)", "$(PKGCP_OS)", 1, 0, 1 \ $(foreach pkg,$(MOZ_OPTIONAL_PKG_LIST),$(PKG_ARG)) ) @cd $(DEPTH)/installer-stage/optional/extensions; find -maxdepth 1 -mindepth 1 -exec rm -r ../../core/extensions/{} \; endif
--- a/tools/update-packaging/common.sh +++ b/tools/update-packaging/common.sh @@ -9,17 +9,17 @@ MAR=${MAR:-mar} BZIP2=${BZIP2:-bzip2} MBSDIFF=${MBSDIFF:-mbsdiff} # ----------------------------------------------------------------------------- # Helper routines notice() { - echo $* 1>&2 + echo "$*" 1>&2 } get_file_size() { info=($(ls -ln "$1")) echo ${info[4]} } copy_perm() { @@ -30,83 +30,141 @@ copy_perm() { chmod 0755 "$target" else chmod 0644 "$target" fi } make_add_instruction() { f="$1" + + # Used to log to the console + if [ $2 ]; then + forced=" (forced)" + else + forced= + fi + is_extension=$(echo "$f" | grep -c 'extensions/.*/') if [ $is_extension = "1" ]; then # Use the subdirectory of the extensions folder as the file to test # before performing this add instruction. testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/') + notice " add-if: $f$forced" echo "add-if \"$testdir\" \"$f\"" else + notice " add: $f$forced" echo "add \"$f\"" fi } make_patch_instruction() { f="$1" is_extension=$(echo "$f" | grep -c 'extensions/.*/') is_search_plugin=$(echo "$f" | grep -c 'searchplugins/.*') if [ $is_extension = "1" ]; then # Use the subdirectory of the extensions folder as the file to test # before performing this add instruction. testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/') + notice " patch-if: $f" echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" elif [ $is_search_plugin = "1" ]; then + notice " patch-if: $f" echo "patch-if \"$f\" \"$f.patch\" \"$f\"" else + notice " patch: $f" echo "patch \"$f.patch\" \"$f\"" fi } append_remove_instructions() { dir="$1" + filev1="$2" + filev2="$3" if [ -f "$dir/removed-files" ]; then prefix= listfile="$dir/removed-files" elif [ -f "$dir/Contents/MacOS/removed-files" ]; then prefix=Contents/MacOS/ listfile="$dir/Contents/MacOS/removed-files" fi if [ -n "$listfile" ]; then # Map spaces to pipes so that we correctly handle filenames with spaces. - files=($(cat "$listfile" | tr " " "|")) + files=($(cat "$listfile" | tr " " "|" | sort -r)) num_files=${#files[*]} for ((i=0; $i<$num_files; i=$i+1)); do - # Trim whitespace (including trailing carriage returns) - f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r') - # Exclude any blank lines or any lines ending with a slash, which indicate - # directories. The updater doesn't know how to remove entire directories. + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. if [ -n "$f" ]; then - if [ $(echo "$f" | grep -c '\/$') = 0 ]; then - echo "remove \"$prefix$f\"" - else - notice "ignoring remove instruction for directory: $f" + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + # Normalize the path to the root of the Mac OS X bundle if necessary + fixedprefix="$prefix" + if [ $prefix ]; then + if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then + if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then + f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::') + fixedprefix="" + else + f=$(echo $f | sed -e 's:^\.\.\/::') + fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::') + fi + fi + fi + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + notice " rmdir: $fixedprefix$f" + echo "rmdir \"$fixedprefix$f\"" >> $filev2 + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + notice " rmrfdir: $fixedprefix$f" + echo "rmrfdir \"$fixedprefix$f\"" >> $filev2 + else + notice " remove: $fixedprefix$f" + echo "remove \"$fixedprefix$f\"" >> $filev1 + echo "remove \"$fixedprefix$f\"" >> $filev2 + fi fi fi done fi } # List all files in the current directory, stripping leading "./" # Skip the channel-prefs.js file as it should not be included in any # generated MAR files (see bug 306077). Pass a variable name and it will be # filled as an array. list_files() { count=0 find . -type f \ ! -name "channel-prefs.js" \ ! -name "update.manifest" \ + ! -name "updatev2.manifest" \ + ! -name "temp-dirlist" \ ! -name "temp-filelist" \ | sed 's/\.\/\(.*\)/\1/' \ - | sort > "temp-filelist" + | sort -r > "temp-filelist" while read file; do eval "${1}[$count]=\"$file\"" (( count++ )) done < "temp-filelist" rm "temp-filelist" } + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "temp-dirlist" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "temp-dirlist" + rm "temp-dirlist" +}
--- a/tools/update-packaging/make_full_update.sh +++ b/tools/update-packaging/make_full_update.sh @@ -27,59 +27,102 @@ if [ $1 = -h ]; then notice "" exit 1 fi # ----------------------------------------------------------------------------- archive="$1" targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi workdir="$targetdir.work" -manifest="$workdir/update.manifest" -targetfiles="update.manifest" +updatemanifestv1="$workdir/update.manifest" +updatemanifestv2="$workdir/updatev2.manifest" +targetfiles="update.manifest updatev2.manifest" mkdir -p "$workdir" +# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the +# Info.plist so it launches the right architecture, bug 600098 + # Generate a list of all files in the target directory. pushd "$targetdir" if test $? -ne 0 ; then exit 1 fi -# On Mac, force a top-level file so that OS X reloads the Info.plist -# and launches the right architecture for the OS version, bug 600098 -if [[ -d Contents ]]; then - touch force_plist_reload +if [ ! -f "precomplete" ]; then + notice "precomplete file is missing!" + exit 1 fi list_files files +# Files that should be added on channel change +ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/') + popd -> $manifest +notice "" +notice "Adding file add instructions to file 'update.manifest'" +> $updatemanifestv1 num_files=${#files[*]} for ((i=0; $i<$num_files; i=$i+1)); do f="${files[$i]}" - notice "processing $f" - - make_add_instruction "$f" >> $manifest + make_add_instruction "$f" >> $updatemanifestv1 dir=$(dirname "$f") mkdir -p "$workdir/$dir" $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" copy_perm "$targetdir/$f" "$workdir/$f" targetfiles="$targetfiles \"$f\"" done +# Add the type of update to the beginning of and cat the contents of the version +# 1 update manifest to the version 2 update manifest. +> $updatemanifestv2 +notice "" +notice "Adding type instruction to file 'updatev2.manifest'" +notice " type: complete" +echo "type \"complete\"" >> $updatemanifestv2 + +notice "" +notice "Adding file ADD instructions for channel change to file 'updatev2.manifest'" +for f in $ccfiles; do + notice " add-cc: $f" + echo "add-cc \"$f\"" >> $updatemanifestv2 + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" +done + +notice "" +notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'" +cat $updatemanifestv1 >> $updatemanifestv2 + # Append remove instructions for any dead files. -append_remove_instructions "$targetdir" >> $manifest +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2" -$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest" +$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1" +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" eval "$MAR -C \"$workdir\" -c output.mar $targetfiles" mv -f "$workdir/output.mar" "$archive" # cleanup rm -fr "$workdir" + +notice "" +notice "Finished"
--- a/tools/update-packaging/make_incremental_update.sh +++ b/tools/update-packaging/make_incremental_update.sh @@ -21,16 +21,21 @@ print_usage() { } check_for_forced_update() { force_list="$1" forced_file_chk="$2" local f + if [ "$forced_file_chk" = "precomplete" ]; then + ## "true" *giggle* + return 0; + fi + if [ "${forced_file_chk##*.}" = "chk" ] then ## "true" *giggle* return 0; fi for f in $force_list; do #echo comparing $forced_file_chk to $f @@ -65,122 +70,200 @@ done # ----------------------------------------------------------------------------- let arg_start=$OPTIND-1 shift $arg_start archive="$1" olddir="$2" newdir="$3" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then + # Remove the / + newdir=$(echo "$newdir" | sed -e 's:\/$::') +fi workdir="$newdir.work" -manifest="$workdir/update.manifest" -archivefiles="update.manifest" +updatemanifestv1="$workdir/update.manifest" +updatemanifestv2="$workdir/updatev2.manifest" +archivefiles="update.manifest updatev2.manifest" mkdir -p "$workdir" +# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the +# Info.plist so it launches the right architecture, bug 600098 + # Generate a list of all files in the target directory. pushd "$olddir" if test $? -ne 0 ; then exit 1 fi list_files oldfiles +list_dirs olddirs popd pushd "$newdir" if test $? -ne 0 ; then exit 1 fi +if [ ! -f "precomplete" ]; then + notice "precomplete file is missing!" + exit 1 +fi + +list_dirs newdirs list_files newfiles +# Files that should be added on channel change +ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/') + popd -> $manifest +notice "" +notice "Adding file patch and add instructions to file 'update.manifest'" +> $updatemanifestv1 num_oldfiles=${#oldfiles[*]} +remove_array= +num_removes=0 for ((i=0; $i<$num_oldfiles; i=$i+1)); do f="${oldfiles[$i]}" # This file is created by Talkback, so we can ignore it if [ "$f" = "readme.txt" ]; then continue 1 fi + # removed-files is excluded by make_incremental_updates.py so it is excluded + # here for consistency. + if [ `basename $f` = "removed-files" ]; then + continue 1 + fi + # If this file exists in the new directory as well, then check if it differs. if [ -f "$newdir/$f" ]; then if check_for_forced_update "$requested_forced_updates" "$f"; then - echo 1>&2 " FORCING UPDATE for file '$f'..." - # The full workdir may not exist yet, so create it if necessary. + # The full workdir may not exist yet, so create it if necessary. mkdir -p `dirname "$workdir/$f"` $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" copy_perm "$newdir/$f" "$workdir/$f" - make_add_instruction "$f" >> $manifest + make_add_instruction "$f" 1 >> $updatemanifestv1 archivefiles="$archivefiles \"$f\"" continue 1 fi if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then # Compute both the compressed binary diff and the compressed file, and # compare the sizes. Then choose the smaller of the two to package. - echo " diffing $f" dir=$(dirname "$workdir/$f") mkdir -p "$dir" $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" $BZIP2 -z9 "$workdir/$f.patch" $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" copy_perm "$newdir/$f" "$workdir/$f" patchfile="$workdir/$f.patch.bz2" patchsize=$(get_file_size "$patchfile") fullsize=$(get_file_size "$workdir/$f") - if [ $patchsize -lt $fullsize -a "$f" != "removed-files" ]; then - make_patch_instruction "$f" >> $manifest + if [ $patchsize -lt $fullsize ]; then + make_patch_instruction "$f" >> $updatemanifestv1 mv -f "$patchfile" "$workdir/$f.patch" rm -f "$workdir/$f" archivefiles="$archivefiles \"$f.patch\"" else - make_add_instruction "$f" >> $manifest + make_add_instruction "$f" >> $updatemanifestv1 rm -f "$patchfile" archivefiles="$archivefiles \"$f\"" fi fi else - echo "remove \"$f\"" >> $manifest + # remove instructions are added after add / patch instructions for + # consistency with make_incremental_updates.py + remove_array[$num_removes]=$f + (( num_removes++ )) fi done -# Now, we just need to worry about newly added files +# Newly added files +notice "" +notice "Adding file add instructions to file 'update.manifest'" num_newfiles=${#newfiles[*]} for ((i=0; $i<$num_newfiles; i=$i+1)); do f="${newfiles[$i]}" + # removed-files is excluded by make_incremental_updates.py so it is excluded + # here for consistency. + if [ `basename $f` = "removed-files" ]; then + continue 1 + fi + # If we've already tested this file, then skip it for ((j=0; $j<$num_oldfiles; j=$j+1)); do if [ "$f" = "${oldfiles[j]}" ]; then continue 2 fi done - + dir=$(dirname "$workdir/$f") mkdir -p "$dir" $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" copy_perm "$newdir/$f" "$workdir/$f" - make_add_instruction "$f" >> "$manifest" + make_add_instruction "$f" >> "$updatemanifestv1" archivefiles="$archivefiles \"$f\"" done +notice "" +notice "Adding file remove instructions to file 'update.manifest'" +for ((i=0; $i<$num_removes; i=$i+1)); do + f="${remove_array[$i]}" + notice " remove: $f" + echo "remove \"$f\"" >> $updatemanifestv1 +done + +# Add the type of update to the beginning of and cat the contents of the version +# 1 update manifest to the version 2 update manifest. +notice "" +notice "Adding type instruction to file 'updatev2.manifest'" +> $updatemanifestv2 +notice " type: partial" +echo "type \"partial\"" >> $updatemanifestv2 + +notice "" +notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'" +cat $updatemanifestv1 >> $updatemanifestv2 + # Append remove instructions for any dead files. -append_remove_instructions "$newdir" >> $manifest +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$newdir" "$updatemanifestv1" "$updatemanifestv2" + +notice "" +notice "Adding directory remove instructions for directories that no longer exist" +num_olddirs=${#olddirs[*]} -$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest" +for ((i=0; $i<$num_olddirs; i=$i+1)); do + f="${olddirs[$i]}" + # If this dir doesn't exist in the new directory remove it. + if [ ! -d "$newdir/$f" ]; then + notice " rmdir: $f/" + echo "rmdir \"$f/\"" >> $updatemanifestv2 + fi +done + +$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1" +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" eval "$MAR -C \"$workdir\" -c output.mar $archivefiles" mv -f "$workdir/output.mar" "$archive" # cleanup rm -fr "$workdir" + +notice "" +notice "Finished"
--- a/tools/update-packaging/make_incremental_updates.py +++ b/tools/update-packaging/make_incremental_updates.py @@ -1,129 +1,166 @@ import os import shutil import sha -import sha from os.path import join, getsize from stat import * import re import sys import getopt import time import datetime import bz2 import string import tempfile class PatchInfo: """ Represents the meta-data associated with a patch work_dir = working dir where files are stored for this patch archive_files = list of files to include in this patch - manifest = set of patch instructions + manifestv1 = set of manifest version 1 patch instructions + manifestv2 = set of manifest version 2 patch instructions file_exclusion_list = files to exclude from this patch. names without slashes will be excluded anywhere in the directory hiearchy. names with slashes will only be excluded at that exact path """ def __init__(self, work_dir, file_exclusion_list, path_exclusion_list): self.work_dir=work_dir self.archive_files=[] - self.manifest=[] + self.manifestv1=[] + self.manifestv2=[] self.file_exclusion_list=file_exclusion_list self.path_exclusion_list=path_exclusion_list def append_add_instruction(self, filename): """ Appends an add instruction for this patch. if the filename starts with extensions/ adds an add-if instruction to test the existence of the subdirectory. This was ported from mozilla/tools/update-packaging/common.sh/make_add_instruction """ if filename.startswith("extensions/"): # Directory immediately following extensions is used for the test testdir = "extensions/"+filename.split("/")[1] - self.manifest.append('add-if "'+testdir+'" "'+filename+'"') + self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"') + self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"') elif filename.startswith("Contents/MacOS/extensions/"): testdir = "Contents/MacOS/extensions/"+filename.split("/")[3] - self.manifest.append('add-if "'+testdir+'" "'+filename+'"') + self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"') + self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"') else: - self.manifest.append('add "'+filename+'"') + self.manifestv1.append('add "'+filename+'"') + self.manifestv2.append('add "'+filename+'"') def append_patch_instruction(self, filename, patchname): """ Appends an patch instruction for this patch. filename = file to patch patchname = patchfile to apply to file if the filename starts with extensions/ adds a patch-if instruction to test the existence of the subdirectory. if the filename starts with searchplugins/ add a add-if instruction for the filename This was ported from mozilla/tools/update-packaging/common.sh/make_patch_instruction """ if filename.startswith("extensions/"): testdir = "extensions/"+filename.split("/")[1] - self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"') + self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"') + self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"') elif filename.startswith("Contents/MacOS/extensions/"): testdir = "Contents/MacOS/extensions/"+filename.split("/")[3] - self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"') + self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"') + self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"') elif (filename.startswith("searchplugins/") or filename.startswith("Contents/MacOS/searchplugins/")): - self.manifest.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"') + self.manifestv1.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"') + self.manifestv2.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"') else: - self.manifest.append('patch "'+patchname+'" "'+filename+'"') + self.manifestv1.append('patch "'+patchname+'" "'+filename+'"') + self.manifestv2.append('patch "'+patchname+'" "'+filename+'"') def append_remove_instruction(self, filename): """ Appends an remove instruction for this patch. This was ported from mozilla/tools/update-packaging/common.sh/make_remove_instruction """ - self.manifest.append('remove "'+filename+'"') + if filename.endswith("/"): + self.manifestv2.append('rmdir "'+filename+'"') + elif filename.endswith("/*"): + filename = filename[:-1] + self.manifestv2.append('rmrfdir "'+filename+'"') + else: + self.manifestv1.append('remove "'+filename+'"') + self.manifestv2.append('remove "'+filename+'"') - def create_manifest_file(self): - """ Createst the manifest file into to the root of the work_dir """ + def create_manifest_files(self): + """ Createst the v1 manifest file in the root of the work_dir """ manifest_file_path = os.path.join(self.work_dir,"update.manifest") - manifest_file = open(manifest_file_path, "w") - manifest_file.writelines(string.join(self.manifest, '\n')) + manifest_file = open(manifest_file_path, "wb") + manifest_file.writelines(string.join(self.manifestv1, '\n')) manifest_file.writelines("\n") manifest_file.close() bzip_file(manifest_file_path) self.archive_files.append('"update.manifest"') + """ Createst the v2 manifest file in the root of the work_dir """ + manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest") + manifest_file = open(manifest_file_path, "wb") + manifest_file.writelines("type \"partial\"\n") + manifest_file.writelines(string.join(self.manifestv2, '\n')) + manifest_file.writelines("\n") + manifest_file.close() + + bzip_file(manifest_file_path) + self.archive_files.append('"updatev2.manifest"') def build_marfile_entry_hash(self, root_path): - """ Iterates through the root_path, creating a MarFileEntry for each file - in that path. Excluseds any filenames in the file_exclusion_list - """ - mar_entry_hash = {} - filename_set = set() - for root, dirs, files in os.walk(root_path): - for name in files: - # filename is relative path from root directory - partial_path = root[len(root_path)+1:] - if name not in self.file_exclusion_list: - filename = os.path.join(partial_path, name) - if "/"+filename not in self.path_exclusion_list: - mar_entry_hash[filename]=MarFileEntry(root_path, filename) - filename_set.add(filename) - return mar_entry_hash, filename_set + """ Iterates through the root_path, creating a MarFileEntry for each file + and directory in that path. Excludes any filenames in the file_exclusion_list + """ + mar_entry_hash = {} + filename_set = set() + dirname_set = set() + for root, dirs, files in os.walk(root_path): + for name in files: + # filename is the relative path from root directory + partial_path = root[len(root_path)+1:] + if name not in self.file_exclusion_list: + filename = os.path.join(partial_path, name) + if "/"+filename not in self.path_exclusion_list: + mar_entry_hash[filename]=MarFileEntry(root_path, filename) + filename_set.add(filename) + + for name in dirs: + # dirname is the relative path from root directory + partial_path = root[len(root_path)+1:] + if name not in self.file_exclusion_list: + dirname = os.path.join(partial_path, name) + if "/"+dirname not in self.path_exclusion_list: + dirname = dirname+"/" + mar_entry_hash[dirname]=MarFileEntry(root_path, dirname) + dirname_set.add(dirname) + + return mar_entry_hash, filename_set, dirname_set class MarFileEntry: """Represents a file inside a Mozilla Archive Format (MAR) abs_path = abspath to the the file name = relative path within the mar. e.g. foo.mar/dir/bar.txt extracted into /tmp/foo: abs_path=/tmp/foo/dir/bar.txt name = dir/bar.txt """ def __init__(self, root, name): """root = path the the top of the mar name = relative path within the mar""" - self.name=name + self.name=name.replace("\\", "/") self.abs_path=os.path.join(root,name) self.sha_cache=None def __str__(self): return 'Name: %s FullPath: %s' %(self.name,self.abs_path) def calc_file_sha_digest(self, filename): """ Returns sha digest of given filename""" @@ -214,20 +251,20 @@ def create_partial_patch_for_file(from_m os.remove(patch_file_abs_path) file_in_manifest_name = from_marfile_entry.name file_in_manifest_abspath = full_file_abs_path patch_info.append_add_instruction(file_in_manifest_name) shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath) patch_info.archive_files.append('"'+file_in_manifest_name+'"') else: - print "skipping diff: " + from_marfile_entry.name filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()] # We've already calculated the patch for this pair of files. if (filename.endswith(".patch")): + print "skipping diff: " + from_marfile_entry.name # Patch was smaller than file - add patch instruction to manifest file_in_manifest_name = to_marfile_entry.name+'.patch'; patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name) else: # File was smaller than file - add file to manifest file_in_manifest_name = to_marfile_entry.name patch_info.append_add_instruction(file_in_manifest_name) # Copy the pre-calculated file into our new patch work aread @@ -245,72 +282,88 @@ def process_explicit_remove_files(dir_pa """ Looks for a 'removed-files' file in the dir_path. If the removed-files does not exist this will throw. If found adds the removed-files found in that file to the patch_info""" # Windows and linux have this file at the root of the dir list_file_path = os.path.join(dir_path, "removed-files") prefix="" if not os.path.exists(list_file_path): - # Mac has is in Contents/MacOS/ + # On Mac removed-files contains relative paths from Contents/MacOS/ prefix= "Contents/MacOS" list_file_path = os.path.join(dir_path, prefix+"/removed-files") if (os.path.exists(list_file_path)): list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist + lines = [] for line in list_file: - line = line.strip() - # Exclude any blank lines or any lines ending with a slash, which indicate - # directories. The updater doesn't know how to remove entire directories. - if line and not line.endswith("/"): - patch_info.append_remove_instruction(os.path.join(prefix,line)) + lines.append(line.strip()) + + lines.sort(reverse=True) + for line in lines: + # Exclude any blank and comment lines. + if line and not line.startswith("#"): + # Python on windows uses \ for path separators and the update + # manifests expects / for path separators on all platforms. + line=os.path.join(prefix,line).replace("\\", "/") + patch_info.append_remove_instruction(line) def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates): """ Builds a partial patch by comparing the files in from_dir_path to thoes of to_dir_path""" # Cannocolize the paths for safey from_dir_path = os.path.abspath(from_dir_path) to_dir_path = os.path.abspath(to_dir_path) - # First create a hashtable of the from and to directories - from_dir_hash,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path) - to_dir_hash,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path) + # Create a hashtable of the from and to directories + from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path) + to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path) + # Require that the precomplete file is included in the to complete update + if "precomplete" not in to_file_set: + raise Exception, "missing precomplete file in: "+to_dir_path # Create a list of the forced updates forced_list = forced_updates.strip().split('|') - + forced_list.append("precomplete") + # Files which exist in both sets need to be patched - patch_filenames = list(from_dir_set.intersection(to_dir_set)) - patch_filenames.sort() + patch_filenames = list(from_file_set.intersection(to_file_set)) + patch_filenames.sort(reverse=True) for filename in patch_filenames: from_marfile_entry = from_dir_hash[filename] to_marfile_entry = to_dir_hash[filename] if filename in forced_list: print "Forcing "+ filename # This filename is in the forced list, explicitly add - create_add_patch_for_file(to_dir_hash[filename], patch_info) + create_add_patch_for_file(to_dir_hash[filename], patch_info) else: if from_marfile_entry.sha() != to_marfile_entry.sha(): # Not the same - calculate a patch create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info) + # files in to_dir not in from_dir need to added + add_filenames = list(to_file_set - from_file_set) + add_filenames.sort(reverse=True) + for filename in add_filenames: + create_add_patch_for_file(to_dir_hash[filename], patch_info) + # files in from_dir not in to_dir need to be removed - remove_filenames = list(from_dir_set - to_dir_set) - remove_filenames.sort() + remove_filenames = list(from_file_set - to_file_set) + remove_filenames.sort(reverse=True) for filename in remove_filenames: patch_info.append_remove_instruction(from_dir_hash[filename].name) - # files in to_dir not in from_dir need to added - add_filenames = list(to_dir_set - from_dir_set) - add_filenames.sort() - for filename in add_filenames: - create_add_patch_for_file(to_dir_hash[filename], patch_info) - process_explicit_remove_files(to_dir_path, patch_info) - # Construct Manifest file - patch_info.create_manifest_file() + # directories in from_dir not in to_dir need to be removed + remove_dirnames = list(from_dir_set - to_dir_set) + remove_dirnames.sort(reverse=True) + for dirname in remove_dirnames: + patch_info.append_remove_instruction(from_dir_hash[dirname].name) + + # Construct the Manifest files + patch_info.create_manifest_files() # And construct the mar mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ') exec_shell_cmd(mar_cmd) # Copy mar to final destination patch_file_dir = os.path.split(patch_filename)[0] if not os.path.exists(patch_file_dir): @@ -397,17 +450,17 @@ def create_partial_patches(patches): extract_mar(to_filename, work_dir_to) to_decoded = decode_filename(from_filename) to_buildid = get_buildid(work_dir_to, to_decoded['platform']) to_shasum = sha.sha(open(to_filename).read()).hexdigest() to_size = str(os.path.getsize(to_filename)) mar_extract_time = time.time() - partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt']),forced_updates) + partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt']),forced_updates) partial_buildid = to_buildid partial_shasum = sha.sha(open(partial_filename).read()).hexdigest() partial_size = str(os.path.getsize(partial_filename)) metadata.append({ 'to_filename': os.path.basename(to_filename), 'from_filename': os.path.basename(from_filename), 'partial_filename': os.path.basename(partial_filename),
--- a/tools/update-packaging/test/buildrefmars.sh +++ b/tools/update-packaging/test/buildrefmars.sh @@ -1,16 +1,26 @@ #!/bin/bash # Builds all the reference mars -rm ref.mar -rm ref-mac.mar +if [ -f "ref.mar" ]; then + rm "ref.mar" +fi +if [ -f "ref-mac.mar" ]; then + rm "ref-mac.mar" +fi ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to ../make_incremental_update.sh ref-mac.mar `pwd`/from `pwd`/to-mac -rm product-1.0.lang.platform.complete.mar -rm product-2.0.lang.platform.complete.mar -rm product-2.0.lang.mac.complete.mar +if [ -f "product-1.0.lang.platform.complete.mar" ]; then + rm "product-1.0.lang.platform.complete.mar" +fi +if [ -f "product-2.0.lang.platform.complete.mar" ]; then + rm "product-2.0.lang.platform.complete.mar" +fi +if [ -f "product-2.0.lang.mac.complete.mar" ]; then + rm "product-2.0.lang.mac.complete.mar" +fi -./make_full_update.sh product-1.0.lang.platform.complete.mar `pwd`/from -./make_full_update.sh product-2.0.lang.platform.complete.mar `pwd`/to -./make_full_update.sh product-2.0.lang.mac.complete.mar `pwd`/to-mac +./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from" +./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to" +./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac"
--- a/tools/update-packaging/test/catmanifest.sh +++ b/tools/update-packaging/test/catmanifest.sh @@ -6,9 +6,9 @@ workdir="/tmp/catmanifest" rm -rf "$workdir" mkdir -p "$workdir" cp "$1" "$workdir" cd "$workdir" mar -x "$1" mv update.manifest update.manifest.bz2 bzip2 -d update.manifest.bz2 -cat update.manifest \ No newline at end of file +cat update.manifest
--- a/tools/update-packaging/test/common.sh +++ b/tools/update-packaging/test/common.sh @@ -10,17 +10,17 @@ MAR=${MAR:-mar} BZIP2=${BZIP2:-bzip2} MBSDIFF=${MBSDIFF:-mbsdiff} # ----------------------------------------------------------------------------- # Helper routines notice() { - echo $* 1>&2 + echo "$*" 1>&2 } get_file_size() { info=($(ls -ln "$1")) echo ${info[4]} } copy_perm() { @@ -31,82 +31,137 @@ copy_perm() { chmod 0755 "$target" else chmod 0644 "$target" fi } make_add_instruction() { f="$1" + + # Used to log to the console + if [ $2 ]; then + forced=" (forced)" + else + forced= + fi + is_extension=$(echo "$f" | grep -c 'extensions/.*/') if [ $is_extension = "1" ]; then # Use the subdirectory of the extensions folder as the file to test # before performing this add instruction. testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/') + notice " add-if: $f$forced" echo "add-if \"$testdir\" \"$f\"" else + notice " add: $f$forced" echo "add \"$f\"" fi } make_patch_instruction() { f="$1" is_extension=$(echo "$f" | grep -c 'extensions/.*/') is_search_plugin=$(echo "$f" | grep -c 'searchplugins/.*') if [ $is_extension = "1" ]; then # Use the subdirectory of the extensions folder as the file to test # before performing this add instruction. testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/') + notice " patch-if: $f" echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" elif [ $is_search_plugin = "1" ]; then + notice " patch-if: $f" echo "patch-if \"$f\" \"$f.patch\" \"$f\"" else + notice " patch: $f" echo "patch \"$f.patch\" \"$f\"" fi } append_remove_instructions() { dir="$1" + filev1="$2" + filev2="$3" if [ -f "$dir/removed-files" ]; then prefix= listfile="$dir/removed-files" elif [ -f "$dir/Contents/MacOS/removed-files" ]; then prefix=Contents/MacOS/ listfile="$dir/Contents/MacOS/removed-files" fi if [ -n "$listfile" ]; then # Map spaces to pipes so that we correctly handle filenames with spaces. - files=($(cat "$listfile" | tr " " "|")) + files=($(cat "$listfile" | tr " " "|" | sort -r)) num_files=${#files[*]} for ((i=0; $i<$num_files; i=$i+1)); do - # Trim whitespace (including trailing carriage returns) - f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r') - # Exclude any blank lines or any lines ending with a slash, which indicate - # directories. The updater doesn't know how to remove entire directories. + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. if [ -n "$f" ]; then - if [ $(echo "$f" | grep -c '\/$') = 0 ]; then - echo "remove \"$prefix$f\"" - else - notice "ignoring remove instruction for directory: $f" + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + # Normalize the path to the root of the Mac OS X bundle if necessary + fixedprefix="$prefix" + if [ $prefix ]; then + if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then + if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then + f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::') + fixedprefix="" + else + f=$(echo $f | sed -e 's:^\.\.\/::') + fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::') + fi + fi + fi + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + notice " rmdir: $fixedprefix$f" + echo "rmdir \"$fixedprefix$f\"" >> $filev2 + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + notice " rmrfdir: $fixedprefix$f" + echo "rmrfdir \"$fixedprefix$f\"" >> $filev2 + else + notice " remove: $fixedprefix$f" + echo "remove \"$fixedprefix$f\"" >> $filev1 + echo "remove \"$fixedprefix$f\"" >> $filev2 + fi fi fi done fi } # List all files in the current directory, stripping leading "./" # Skip the channel-prefs.js file as it should not be included in any # generated MAR files (see bug 306077). Pass a variable name and it will be # filled as an array. list_files() { count=0 - # Schrep - removed the exclusion cases here to allow for generation - # of testing mars + # Removed the exclusion cases here to allow for generation of testing mars find . -type f \ | sed 's/\.\/\(.*\)/\1/' \ - | sort > "$workdir/temp-filelist" + | sort -r > "$workdir/temp-filelist" while read file; do eval "${1}[$count]=\"$file\"" (( count++ )) done < "$workdir/temp-filelist" rm "$workdir/temp-filelist" } + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "$workdir/temp-dirlist" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "$workdir/temp-dirlist" + rm "$workdir/temp-dirlist" +}
--- a/tools/update-packaging/test/diffmar.sh +++ b/tools/update-packaging/test/diffmar.sh @@ -2,34 +2,49 @@ # Compares two mars marA="$1" marB="$2" workdir="/tmp/diffmar" fromdir="$workdir/0" todir="$workdir/1" +# On Windows, creation time can be off by a second or more between the files in +# the fromdir and todir due to them being extracted synchronously so use +# time-style and exclude seconds from the creation time. +lsargs="-algR" +unamestr=`uname` +if [ ! "$unamestr" = 'Darwin' ]; then + unamestr=`uname -o` + if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then + lsargs="-algR --time-style=+%Y-%m-%d-%H:%M" + fi +fi + rm -rf "$workdir" mkdir -p "$fromdir" mkdir -p "$todir" cp "$1" "$fromdir" cp "$2" "$todir" cd "$fromdir" mar -x "$1" rm "$1" mv update.manifest update.manifest.bz2 bzip2 -d update.manifest.bz2 -ls -algR > files.txt -# Sort the manifest so we don't get any diffs for ordering -#cat update.manifest | sort > update.manifest +mv updatev2.manifest updatev2.manifest.bz2 +bzip2 -d updatev2.manifest.bz2 +ls $lsargs > files.txt cd "$todir" mar -x "$2" rm "$2" mv update.manifest update.manifest.bz2 bzip2 -d update.manifest.bz2 -# Sort the manifest so we don't get any diffs for ordering -#cat update.manifest | sort > update.manifest -ls -algR > files.txt +mv updatev2.manifest updatev2.manifest.bz2 +bzip2 -d updatev2.manifest.bz2 +ls $lsargs > files.txt -diff -r "$fromdir" "$todir" \ No newline at end of file +echo "diffing $fromdir and $todir" +echo "on linux shell sort and python sort return different results" +echo "which cause differences in the manifest files" +diff -ru "$fromdir" "$todir"
new file mode 100644 --- /dev/null +++ b/tools/update-packaging/test/from/precomplete @@ -0,0 +1,21 @@ +remove {foodir/update.manifest +remove {foodir/same.txt +remove {foodir/same.bin +remove {foodir/removed.txt +remove {foodir/readme.txt +remove {foodir/force.txt +remove {foodir/diff-patch-larger-than-file.txt +remove-cc {foodir/channel-prefs.js +remove update.manifest +remove searchplugins/diff/diff-patch-larger-than-file.txt +remove same.txt +remove same.bin +remove removed.txt +remove readme.txt +remove precomplete +remove force.txt +remove diff-patch-larger-than-file.txt +remove-cc channel-prefs.js +rmdir {foodir/ +rmdir searchplugins/diff/ +rmdir searchplugins/
--- a/tools/update-packaging/test/make_full_update.sh +++ b/tools/update-packaging/test/make_full_update.sh @@ -28,53 +28,102 @@ if [ $1 = -h ]; then notice "" exit 1 fi # ----------------------------------------------------------------------------- archive="$1" targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi workdir="$targetdir.work" -manifest="$workdir/update.manifest" -targetfiles="update.manifest" +updatemanifestv1="$workdir/update.manifest" +updatemanifestv2="$workdir/updatev2.manifest" +targetfiles="update.manifest updatev2.manifest" mkdir -p "$workdir" +# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the +# Info.plist so it launches the right architecture, bug 600098 + # Generate a list of all files in the target directory. pushd "$targetdir" if test $? -ne 0 ; then exit 1 fi +if [ ! -f "precomplete" ]; then + notice "precomplete file is missing!" + exit 1 +fi + list_files files +# Files that should be added on channel change +ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/') + popd -> $manifest +notice "" +notice "Adding file add instructions to file 'update.manifest'" +> $updatemanifestv1 num_files=${#files[*]} for ((i=0; $i<$num_files; i=$i+1)); do f="${files[$i]}" - notice "processing $f" - - make_add_instruction "$f" >> $manifest + make_add_instruction "$f" >> $updatemanifestv1 dir=$(dirname "$f") mkdir -p "$workdir/$dir" $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" copy_perm "$targetdir/$f" "$workdir/$f" targetfiles="$targetfiles \"$f\"" done +# Add the type of update to the beginning of and cat the contents of the version +# 1 update manifest to the version 2 update manifest. +> $updatemanifestv2 +notice "" +notice "Adding type instruction to file 'updatev2.manifest'" +notice " type: complete" +echo "type \"complete\"" >> $updatemanifestv2 + +notice "" +notice "Adding file add on channel change instructions to file 'updatev2.manifest'" +for f in $ccfiles; do + notice " add-cc: $f" + echo "add-cc \"$f\"" >> $updatemanifestv2 + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" +done + +notice "" +notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'" +cat $updatemanifestv1 >> $updatemanifestv2 + # Append remove instructions for any dead files. -append_remove_instructions "$targetdir" >> $manifest +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2" -$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest" +$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1" +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" eval "$MAR -C \"$workdir\" -c output.mar $targetfiles" mv -f "$workdir/output.mar" "$archive" # cleanup rm -fr "$workdir" + +notice "" +notice "Finished"
--- a/tools/update-packaging/test/runtests.sh +++ b/tools/update-packaging/test/runtests.sh @@ -2,18 +2,8 @@ echo "testing make_incremental_updates.py" python ../make_incremental_updates.py -f testpatchfile.txt echo "diffing ref.mar and test.mar" ./diffmar.sh ref.mar test.mar echo "diffing ref-mac.mar and test-mac.mar" ./diffmar.sh ref-mac.mar test-mac.mar - - -echo "testing make_incremental_updates_mar.py" -python ../make_incremental_updates_mar.py -f testpatchfile.txt - -echo "diffing ref.mar and test.mar" -./diffmar.sh ref.mar test.mar -echo "diffing ref-mac.mar and test-mac.mar" -./diffmar.sh ref-mac.mar test-mac.mar -
new file mode 100644 --- /dev/null +++ b/tools/update-packaging/test/to-mac/precomplete @@ -0,0 +1,4 @@ +remove precomplete +remove Contents/MacOS/removed-files +rmdir Contents/MacOS/ +rmdir Contents/
new file mode 100644 --- /dev/null +++ b/tools/update-packaging/test/to/precomplete @@ -0,0 +1,21 @@ +remove {foodir/update.manifest +remove {foodir/same.txt +remove {foodir/same.bin +remove {foodir/readme.txt +remove {foodir/force.txt +remove {foodir/diff-patch-larger-than-file.txt +remove-cc {foodir/channel-prefs.js +remove update.manifest +remove searchplugins/diff/diff-patch-larger-than-file.txt +remove same.txt +remove same.bin +remove removed-files +remove readme.txt +remove precomplete +remove force.txt +remove diff-patch-larger-than-file.txt +remove-cc channel-prefs.js +rmdir {foodir/ +rmdir searchplugins/diff/ +rmdir searchplugins/added/ +rmdir searchplugins/
--- a/tools/update-packaging/test_make_incremental_updates.py +++ b/tools/update-packaging/test_make_incremental_updates.py @@ -2,46 +2,47 @@ import unittest import make_incremental_updates as mkup from make_incremental_updates import PatchInfo, MarFileEntry class TestPatchInfo(unittest.TestCase): def setUp(self): self.work_dir = 'work_dir' - self.file_exclusion_list = ['channel-prefs.js','update.manifest','removed-files'] + self.file_exclusion_list = ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'] self.path_exclusion_list = ['/readme.txt'] self.patch_info = PatchInfo(self.work_dir, self.file_exclusion_list, self.path_exclusion_list) def testPatchInfo(self): self.assertEquals(self.work_dir, self.patch_info.work_dir) self.assertEquals([], self.patch_info.archive_files) - self.assertEquals([], self.patch_info.manifest) + self.assertEquals([], self.patch_info.manifestv1) + self.assertEquals([], self.patch_info.manifestv2) self.assertEquals(self.file_exclusion_list, self.patch_info.file_exclusion_list) self.assertEquals(self.path_exclusion_list, self.patch_info.path_exclusion_list) def test_append_add_instruction(self): self.patch_info.append_add_instruction('file.test') - self.assertEquals(['add "file.test"'], self.patch_info.manifest) + self.assertEquals(['add "file.test"'], self.patch_info.manifestv1) def test_append_patch_instruction(self): self.patch_info.append_patch_instruction('file.test', 'patchname') - self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifest) + self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv1) """ FIXME touches the filesystem, need refactoring def test_append_remove_instruction(self): self.patch_info.append_remove_instruction('file.test') - self.assertEquals(['remove "file.test"'], self.patch_info.manifest) + self.assertEquals(['remove "file.test"'], self.patch_info.manifestv1) def test_create_manifest_file(self): self.patch_info.create_manifest_file() """ def test_build_marfile_entry_hash(self): - self.assertEquals(({}, set([])), self.patch_info.build_marfile_entry_hash('root_path')) + self.assertEquals(({}, set([]), set([])), self.patch_info.build_marfile_entry_hash('root_path')) """ FIXME touches the filesystem, need refactoring class TestMarFileEntry(unittest.TestCase): def setUp(self): root_path = '.' self.filename = 'file.test' f = open(self.filename, 'w') f.write('Test data\n') @@ -61,17 +62,17 @@ class TestMarFileEntry(unittest.TestCase f.close() sha = self.mar_file_entry.sha() self.assertEquals(goodSha, sha) """ class TestMakeIncrementalUpdates(unittest.TestCase): def setUp(self): work_dir = '.' - self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt']) + self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt']) root_path = '/' filename = 'test.file' self.mar_file_entry = MarFileEntry(root_path, filename) """ FIXME makes direct shell calls, need refactoring def test_exec_shell_cmd(self): mkup.exec_shell_cmd('echo test')