bug 1490119: support mar cert replacement in update verify. r=nthomas
authorBen Hearsum <bhearsum@mozilla.com>
Thu, 13 Sep 2018 08:11:36 -0400
changeset 8454 b04a81e9c98c
parent 8453 992b368f8c05
child 8455 d8cd2c0aba09
push id6177
push userbhearsum@mozilla.com
push dateThu, 13 Sep 2018 12:11:44 +0000
reviewersnthomas
bugs1490119
bug 1490119: support mar cert replacement in update verify. r=nthomas
lib/python/release/updates/verify.py
release/mar_certs/README
release/mar_certs/dep1.der
release/mar_certs/dep2.der
release/mar_certs/nightly_aurora_level3_primary.der
release/mar_certs/nightly_aurora_level3_secondary.der
release/mar_certs/release_primary.der
release/mar_certs/release_secondary.der
release/replace-updater-certs.py
release/updates/verify.sh
--- a/lib/python/release/updates/verify.py
+++ b/lib/python/release/updates/verify.py
@@ -7,37 +7,39 @@ class UpdateVerifyError(Exception):
     pass
 
 
 class UpdateVerifyConfig(object):
     comment_regex = re.compile("^#")
     key_write_order = ("release", "product", "platform", "build_id", "locales",
                        "channel", "patch_types", "from", "aus_server",
                        "ftp_server_from", "ftp_server_to", "to",
-                       "mar_channel_IDs", "to_build_id", "to_display_version",
-                       "to_app_version", "updater_package")
+                       "mar_channel_IDs", "override_certs", "to_build_id",
+                       "to_display_version", "to_app_version", "updater_package")
     global_keys = ("product", "channel", "aus_server", "to", "to_build_id",
-                   "to_display_version", "to_app_version")
+                   "to_display_version", "to_app_version", "override_certs")
     release_keys = ("release", "build_id", "locales", "patch_types", "from",
                     "ftp_server_from", "ftp_server_to", "mar_channel_IDs",
                     "platform", "updater_package")
     first_only_keys = ("from", "aus_server", "to", "to_build_id",
-                       "to_display_version", "to_app_version")
+                       "to_display_version", "to_app_version", "override_certs")
     compare_attrs = global_keys + ("releases",)
 
     def __init__(self, product=None, channel=None,
                  aus_server=None, to=None, to_build_id=None,
-                 to_display_version=None, to_app_version=None):
+                 to_display_version=None, to_app_version=None,
+                 override_certs=None):
         self.product = product
         self.channel = channel
         self.aus_server = aus_server
         self.to = to
         self.to_build_id = to_build_id
         self.to_display_version = to_display_version
         self.to_app_version = to_app_version
+        self.override_certs = override_certs
         self.releases = []
 
     def __eq__(self, other):
         self_list = [getattr(self, attr) for attr in self.compare_attrs]
         other_list = [getattr(other, attr) for attr in self.compare_attrs]
         return self_list == other_list
 
     def __ne__(self, other):
@@ -162,17 +164,18 @@ class UpdateVerifyConfig(object):
                 quickTests.append([test["build_id"], locale, test["from"]])
         allTests = getChunk(fullTests, chunks, thisChunk)
         allTests.extend(getChunk(quickTests, chunks, thisChunk))
 
         newConfig = UpdateVerifyConfig(self.product, self.channel,
                                        self.aus_server, self.to,
                                        self.to_build_id,
                                        self.to_display_version,
-                                       self.to_app_version)
+                                       self.to_app_version,
+                                       self.override_certs)
         for t in allTests:
             build_id, locale, from_path = t
             if from_path == "None":
                 from_path = None
             r = self.getRelease(build_id, from_path)
             try:
                 newConfig.addRelease(r["release"], build_id, locales=[],
                                      ftp_server_from=r["ftp_server_from"],
new file mode 100644
--- /dev/null
+++ b/release/mar_certs/README
@@ -0,0 +1,24 @@
+These certificates are imported from mozilla-central (https://hg.mozilla.org/mozilla-central/file/tip/toolkit/mozapps/update/updater)
+and used to support staging update verify jobs. These jobs end up replacing the certificates within the binaries
+(through a binary search and replace), and must all be the same length for this to work correctly. If we recreate
+these certificates, and the resulting public certificates are not the same length anymore, the commonName may be
+changed to line them up again. https://github.com/google/der-ascii is a useful tool for doing this. For example:
+
+To convert the certificate to ascii:
+der2ascii -i dep1.der -o dep1.ascii
+
+Then use your favourite editor to change the commonName field. That block will look something like:
+    SEQUENCE {
+      SET {
+        SEQUENCE {
+          # commonName
+          OBJECT_IDENTIFIER { 2.5.4.3 }
+          PrintableString { "CI MAR signing key 1" }
+        }
+      }
+    }
+
+You can pad the PrintableString with spaces to increase the length of the cert (1 space = 1 byte).
+
+Then, convert back to der:
+ascii2der -i dep1.ascii -o newdep1.der
new file mode 100644
new file mode 100644
new file mode 100644
new file mode 100644
new file mode 100644
new file mode 100644
new file mode 100755
--- /dev/null
+++ b/release/replace-updater-certs.py
@@ -0,0 +1,33 @@
+import os.path
+import sys
+
+cert_dir = sys.argv[1]
+# Read twice, because strings cannot be copied
+updater_data = open(sys.argv[2], "rb").read()
+new_updater = open(sys.argv[2], "rb").read()
+outfile = sys.argv[3]
+
+cert_pairs = sys.argv[4:]
+
+if (len(cert_pairs) % 2) != 0:
+    print("Certs must be provided in pairs")
+    sys.exit(1)
+
+for find_cert, replace_cert in zip(*[iter(cert_pairs)]*2):
+    find = open(os.path.join(cert_dir, find_cert), "rb").read()
+    replace = open(os.path.join(cert_dir, replace_cert), "rb").read()
+    print("Looking for {}...".format(find_cert))
+    if find in new_updater:
+        print("Replacing {} with {}".format(find_cert, replace_cert))
+        new_updater = new_updater.replace(find, replace)
+    else:
+        print("Didn't find {}...".format(find_cert))
+
+if len(updater_data) != len(new_updater):
+    print("WARNING: new updater is not the same length as the old one (old: {}, new: {})".format(len(updater_data), len(new_updater)))
+
+if updater_data == new_updater:
+    print("WARNING: updater is unchanged")
+
+with open(outfile, 'wb+') as f:
+    f.write(new_updater)
--- a/release/updates/verify.sh
+++ b/release/updates/verify.sh
@@ -12,26 +12,32 @@ create_cache
 
 ftp_server_to="http://stage.mozilla.org/pub/mozilla.org"
 ftp_server_from="http://stage.mozilla.org/pub/mozilla.org"
 aus_server="https://aus4.mozilla.org"
 to=""
 to_build_id=""
 to_app_version=""
 to_display_version=""
+override_certs=""
 diff_summary_log="$PWD/diff-summary.log"
 if [ -e ${diff_summary_log} ]; then
   rm ${diff_summary_log}
 fi
 touch ${diff_summary_log}
 
 pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
 retry="$MY_DIR/../../buildfarm/utils/retry.py -s 1 -r 3"
+cert_replacer="$MY_DIR/../replace-updater-certs.py"
+
+dep_overrides="nightly_aurora_level3_primary.der dep1.der nightly_aurora_level3_secondary.der dep2.der release_primary.der dep1.der release_secondary.der dep2.der"
+nightly_overrides="dep1.der nightly_aurora_level3_primary.der dep2.der nightly_aurora_level3_secondary.der release_primary.der nightly_aurora_level3_primary.der release_secondary.der nightly_aurora_level3_secondary.der"
+release_overrides="dep1.der release_primary.der dep2.der release_secondary.der nightly_aurora_level3_primary.der release_primary.der nightly_aurora_level3_secondary.der release_secondary.der"
 
 runmode=0
 config_file="updates.cfg"
 UPDATE_ONLY=1
 TEST_ONLY=2
 MARS_ONLY=3
 COMPLETE=4
 
@@ -201,16 +207,38 @@ do
         for updater_bin in $updater_bins; do
             if [ -e "$cwd/$updater_bin" ]; then
                 echo "Found updater at $updater_bin"
                 updater="$cwd/$updater_bin"
                 break
             fi
         done
 
+        if [ ! -z "$override_certs" ]; then
+            echo "Replacing certs in updater binary"
+            cp "${updater}" "${updater}.orig"
+            case ${override_certs} in
+              dep)
+                overrides=${dep_overrides}
+                ;;
+              nightly)
+                overrides=${nightly_overrides}
+                ;;
+              release)
+                overrides=${release_overrides}
+                ;;
+              *)
+                echo "Unknown override cert - skipping"
+                ;;
+            esac
+            python "${cert_replacer}" "${MY_DIR}/../mar_certs" "${updater}.orig" "${updater}" ${overrides}
+        else
+            echo "override_certs is '${override_certs}', not replacing any certificates"
+        fi
+
         if [ "$updater" == "null" ]; then
             echo "Couldn't find updater binary"
             continue
         fi
 
         from_path=`echo $from | sed "s/%locale%/${locale}/"`
         to_path=`echo $to | sed "s/%locale%/${locale}/"`
         download_builds "${ftp_server_from}${from_path}" "${ftp_server_to}${to_path}"