Bug 1392914 Remove updatev2.manifest r=mhowell
authorSimon Fraser <sfraser@mozilla.com>
Thu, 18 Jun 2020 07:42:21 +0000
changeset 537588 3275aa91dcf96d3ee791bc8948119eee0a713566
parent 537587 634e89e8e71729fb4d144a6ab835769fa81d95df
child 537589 785599d7c456a29f7b14eada9b6b668c0b1f4727
push id37545
push usernerli@mozilla.com
push dateSat, 27 Jun 2020 09:38:32 +0000
treeherdermozilla-central@0a4b3f99d2d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmhowell
bugs1392914
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1392914 Remove updatev2.manifest r=mhowell I've left the mention of updatev2.manifest in list_files and list_dirs for now, similar to how v1 has been left. Differential Revision: https://phabricator.services.mozilla.com/D80011
tools/update-packaging/common.sh
tools/update-packaging/make_full_update.sh
tools/update-packaging/make_incremental_update.sh
tools/update-packaging/test/catmanifest.sh
tools/update-packaging/test/diffmar.sh
tools/update-packaging/test/make_full_update.sh
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -62,105 +62,78 @@ copy_perm() {
     chmod 0755 "$target"
   else
     chmod 0644 "$target"
   fi
 }
 
 make_add_instruction() {
   f="$1"
-  filev2="$2"
-  # The third param will be an empty string when a file add instruction is only
-  # needed in the version 2 manifest. This only happens when the file has an
-  # add-if-not instruction in the version 3 manifest. This is due to the
-  # precomplete file prior to the version 3 manifest having a remove instruction
-  # for this file so the file is removed before applying a complete update.
-  filev3="$3"
+  filev3="$2"
 
   # Used to log to the console
   if [ $4 ]; then
     forced=" (forced)"
   else
     forced=
   fi
 
   is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
     verbose_notice "     add-if \"$testdir\" \"$f\""
-    echo "add-if \"$testdir\" \"$f\"" >> "$filev2"
-    if [ ! $filev3 = "" ]; then
-      echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
-    fi
+    echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
   else
     verbose_notice "        add \"$f\"$forced"
-    echo "add \"$f\"" >> "$filev2"
-    if [ ! "$filev3" = "" ]; then
-      echo "add \"$f\"" >> "$filev3"
-    fi
+    echo "add \"$f\"" >> "$filev3"
   fi
 }
 
 check_for_add_if_not_update() {
   add_if_not_file_chk="$1"
 
   if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
        `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
     ## "true" *giggle*
     return 0;
   fi
   ## 'false'... because this is bash. Oh yay!
   return 1;
 }
 
-check_for_add_to_manifestv2() {
-  add_if_not_file_chk="$1"
-
-  if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
-    ## "true" *giggle*
-    return 0;
-  fi
-  ## 'false'... because this is bash. Oh yay!
-  return 1;
-}
-
 make_add_if_not_instruction() {
   f="$1"
   filev3="$2"
 
   verbose_notice " add-if-not \"$f\" \"$f\""
   echo "add-if-not \"$f\" \"$f\"" >> "$filev3"
 }
 
 make_patch_instruction() {
   f="$1"
-  filev2="$2"
-  filev3="$3"
+  filev3="$2"
 
   is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
     verbose_notice "   patch-if \"$testdir\" \"$f.patch\" \"$f\""
-    echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev2"
     echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3"
   else
     verbose_notice "      patch \"$f.patch\" \"$f\""
-    echo "patch \"$f.patch\" \"$f\"" >> "$filev2"
     echo "patch \"$f.patch\" \"$f\"" >> "$filev3"
   fi
 }
 
 append_remove_instructions() {
   dir="$1"
-  filev2="$2"
-  filev3="$3"
+  filev3="$2"
 
   if [ -f "$dir/removed-files" ]; then
     listfile="$dir/removed-files"
   elif [ -f "$dir/Contents/Resources/removed-files" ]; then
     listfile="$dir/Contents/Resources/removed-files"
   fi
   if [ -n "$listfile" ]; then
     # Map spaces to pipes so that we correctly handle filenames with spaces.
@@ -172,27 +145,24 @@ append_remove_instructions() {
       # Trim whitespace
       f=$(echo $f)
       # Exclude blank lines.
       if [ -n "$f" ]; then
         # Exclude comments
         if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
           if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
             verbose_notice "      rmdir \"$f\""
-            echo "rmdir \"$f\"" >> "$filev2"
             echo "rmdir \"$f\"" >> "$filev3"
           elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
             # Remove the *
             f=$(echo "$f" | sed -e 's:\*$::')
             verbose_notice "    rmrfdir \"$f\""
-            echo "rmrfdir \"$f\"" >> "$filev2"
             echo "rmrfdir \"$f\"" >> "$filev3"
           else
             verbose_notice "     remove \"$f\""
-            echo "remove \"$f\"" >> "$filev2"
             echo "remove \"$f\"" >> "$filev3"
           fi
         fi
       fi
     done
   fi
 }
 
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -30,35 +30,35 @@ if [ $1 = -h ]; then
   notice "  -h  show this help text"
   notice "  -q  be less verbose"
   notice ""
   exit 1
 fi
 
 if [ $1 = -q ]; then
   QUIET=1
+  export QUIET
   shift
 fi
 
 # -----------------------------------------------------------------------------
 
 mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}"
 
 archive="$1"
 targetdir="$2"
 # Prevent the workdir from being inside the targetdir so it isn't included in
 # the update mar.
 if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
   # Remove the /
   targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
 fi
 workdir="$targetdir.work"
-updatemanifestv2="$workdir/updatev2.manifest"
 updatemanifestv3="$workdir/updatev3.manifest"
-targetfiles="updatev2.manifest updatev3.manifest"
+targetfiles="updatev3.manifest"
 
 mkdir -p "$workdir"
 
 # Generate a list of all files in the target directory.
 pushd "$targetdir"
 if test $? -ne 0 ; then
   exit 1
 fi
@@ -70,54 +70,48 @@ if [ ! -f "precomplete" ]; then
   fi
 fi
 
 list_files files
 
 popd
 
 # Add the type of update to the beginning of the update manifests.
-> "$updatemanifestv2"
 > "$updatemanifestv3"
 notice ""
 notice "Adding type instruction to update manifests"
 notice "       type complete"
-echo "type \"complete\"" >> "$updatemanifestv2"
 echo "type \"complete\"" >> "$updatemanifestv3"
 
 notice ""
 notice "Adding file add instructions to update manifests"
 num_files=${#files[*]}
 
 for ((i=0; $i<$num_files; i=$i+1)); do
   f="${files[$i]}"
 
   if check_for_add_if_not_update "$f"; then
     make_add_if_not_instruction "$f" "$updatemanifestv3"
-    if check_for_add_to_manifestv2 "$f"; then
-      make_add_instruction "$f" "$updatemanifestv2" "" 1
-    fi
   else
-    make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
+    make_add_instruction "$f" "$updatemanifestv3"
   fi
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
   $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
 # Append remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
-append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
+append_remove_instructions "$targetdir" "$updatemanifestv3"
 
-$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
 $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
 
 mar_command="$mar_command -C \"$workdir\" -c output.mar"
 eval "$mar_command $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -106,19 +106,18 @@ olddir="$2"
 newdir="$3"
 # Prevent the workdir from being inside the targetdir so it isn't included in
 # the update mar.
 if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
   # Remove the /
   newdir=$(echo "$newdir" | sed -e 's:\/$::')
 fi
 workdir="$(mktemp -d)"
-updatemanifestv2="$workdir/updatev2.manifest"
 updatemanifestv3="$workdir/updatev3.manifest"
-archivefiles="updatev2.manifest updatev3.manifest"
+archivefiles="updatev3.manifest"
 
 mkdir -p "$workdir"
 
 # Generate a list of all files in the target directory.
 pushd "$olddir"
 if test $? -ne 0 ; then
   exit 1
 fi
@@ -143,20 +142,18 @@ fi
 list_dirs newdirs
 list_files newfiles
 
 popd
 
 # Add the type of update to the beginning of the update manifests.
 notice ""
 notice "Adding type instruction to update manifests"
-> $updatemanifestv2
 > $updatemanifestv3
 notice "       type partial"
-echo "type \"partial\"" >> $updatemanifestv2
 echo "type \"partial\"" >> $updatemanifestv3
 
 notice ""
 notice "Adding file patch and add instructions to update manifests"
 
 num_oldfiles=${#oldfiles[*]}
 remove_array=
 num_removes=0
@@ -177,17 +174,17 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
       continue 1
     fi
 
     if check_for_forced_update "$requested_forced_updates" "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
       $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
-      make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
+      make_add_instruction "$f" "$updatemanifestv3" 1
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
       # Compute both the compressed binary diff and the compressed file, and
       # compare the sizes.  Then choose the smaller of the two to package.
       dir=$(dirname "$workdir/$f")
@@ -222,22 +219,22 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
       fi
       $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
       patchfile="$workdir/$f.patch.xz"
       patchsize=$(get_file_size "$patchfile")
       fullsize=$(get_file_size "$workdir/$f")
 
       if [ $patchsize -lt $fullsize ]; then
-        make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
+        make_patch_instruction "$f" "$updatemanifestv3"
         mv -f "$patchfile" "$workdir/$f.patch"
         rm -f "$workdir/$f"
         archivefiles="$archivefiles \"$f.patch\""
       else
-        make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
+        make_add_instruction "$f" "$updatemanifestv3"
         rm -f "$patchfile"
         archivefiles="$archivefiles \"$f\""
       fi
     fi
   else
     # remove instructions are added after add / patch instructions for
     # consistency with make_incremental_updates.py
     remove_array[$num_removes]=$f
@@ -264,52 +261,49 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); d
   mkdir -p "$dir"
 
   $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
   copy_perm "$newdir/$f" "$workdir/$f"
 
   if check_for_add_if_not_update "$f"; then
     make_add_if_not_instruction "$f" "$updatemanifestv3"
   else
-    make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
+    make_add_instruction "$f" "$updatemanifestv3"
   fi
 
 
   archivefiles="$archivefiles \"$f\""
 done
 
 notice ""
 notice "Adding file remove instructions to update manifests"
 for ((i=0; $i<$num_removes; i=$i+1)); do
   f="${remove_array[$i]}"
   verbose_notice "     remove \"$f\""
-  echo "remove \"$f\"" >> $updatemanifestv2
   echo "remove \"$f\"" >> $updatemanifestv3
 done
 
 # Add remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
-append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
+append_remove_instructions "$newdir" "$updatemanifestv3"
 
 notice ""
 notice "Adding directory remove instructions for directories that no longer exist"
 num_olddirs=${#olddirs[*]}
 
 for ((i=0; $i<$num_olddirs; i=$i+1)); do
   f="${olddirs[$i]}"
   # If this dir doesn't exist in the new directory remove it.
   if [ ! -d "$newdir/$f" ]; then
     verbose_notice "      rmdir $f/"
-    echo "rmdir \"$f/\"" >> $updatemanifestv2
     echo "rmdir \"$f/\"" >> $updatemanifestv3
   fi
 done
 
-$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
 $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
 
 mar_command="$mar_command -C \"$workdir\" -c output.mar"
 eval "$mar_command $archivefiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
deleted file mode 100755
--- a/tools/update-packaging/test/catmanifest.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-# helper tool for testing.  Cats the manifest out of a mar file
-
-mar="$1"
-workdir="/tmp/catmanifest"
-
-rm -rf "$workdir"
-mkdir -p "$workdir"
-cp "$1" "$workdir"
-cd "$workdir"
-mar -x "$1"
-mv updatev2.manifest updatev2.manifest.xz
-xz -d updatev2.manifest.xz
-cat updatev2.manifest
--- a/tools/update-packaging/test/diffmar.sh
+++ b/tools/update-packaging/test/diffmar.sh
@@ -25,27 +25,24 @@ mkdir -p "$fromdir"
 mkdir -p "$todir"
 
 cp "$1" "$fromdir"
 cp "$2" "$todir"
 
 cd "$fromdir"
 mar -x "$1"
 rm "$1"
-mv updatev2.manifest updatev2.manifest.xz
-xz -d updatev2.manifest.xz
+rm -f updatev2.manifest  # Older files may contain this
 mv updatev3.manifest updatev3.manifest.xz
 xz -d updatev3.manifest.xz
 ls $lsargs > files.txt
 
 cd "$todir"
 mar -x "$2"
 rm "$2"
-mv updatev2.manifest updatev2.manifest.xz
-xz -d updatev2.manifest.xz
 mv updatev3.manifest updatev3.manifest.xz
 xz -d updatev3.manifest.xz
 ls $lsargs > files.txt
 
 echo "diffing $fromdir and $todir"
 echo "on linux shell sort and python sort return different results"
 echo "which can cause differences in the manifest files"
 diff -ru "$fromdir" "$todir"
--- a/tools/update-packaging/test/make_full_update.sh
+++ b/tools/update-packaging/test/make_full_update.sh
@@ -39,19 +39,18 @@ archive="$1"
 targetdir="$2"
 # Prevent the workdir from being inside the targetdir so it isn't included in
 # the update mar.
 if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
   # Remove the /
   targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
 fi
 workdir="$targetdir.work"
-updatemanifestv2="$workdir/updatev2.manifest"
 updatemanifestv3="$workdir/updatev3.manifest"
-targetfiles="updatev2.manifest updatev3.manifest"
+targetfiles="updatev3.manifest"
 
 mkdir -p "$workdir"
 
 # Generate a list of all files in the target directory.
 pushd "$targetdir"
 if test $? -ne 0 ; then
   exit 1
 fi
@@ -63,54 +62,48 @@ if [ ! -f "precomplete" ]; then
   fi
 fi
 
 list_files files
 
 popd
 
 # Add the type of update to the beginning of the update manifests.
-> $updatemanifestv2
 > $updatemanifestv3
 notice ""
 notice "Adding type instruction to update manifests"
 notice "       type complete"
-echo "type \"complete\"" >> $updatemanifestv2
 echo "type \"complete\"" >> $updatemanifestv3
 
 notice ""
 notice "Adding file add instructions to update manifests"
 num_files=${#files[*]}
 
 for ((i=0; $i<$num_files; i=$i+1)); do
   f="${files[$i]}"
 
   if check_for_add_if_not_update "$f"; then
     make_add_if_not_instruction "$f" "$updatemanifestv3"
-    if check_for_add_to_manifestv2 "$f"; then
-      make_add_instruction "$f" "$updatemanifestv2" "" 1
-    fi
   else
-    make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
+    make_add_instruction "$f" "$updatemanifestv3"
   fi
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
   $XZ $XZ_OPT --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
 # Append remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
-append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
+append_remove_instructions "$targetdir" "$updatemanifestv3"
 
-$XZ $XZ_OPT --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
 $XZ $XZ_OPT --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"