Bug 1543662 Introduce channel verification to partials r=catlee
authorSimon Fraser <sfraser@mozilla.com>
Fri, 12 Apr 2019 11:59:35 +0000
changeset 469268 7a209bab1bfe826dc163bedcf78493f0a8098ac2
parent 469267 6b039311ee97a426700db0d7dd96e5d6b0c36ed0
child 469269 5d85f0ed28cc8bf0c636c4675210b5f4c6788f97
push id112776
push usershindli@mozilla.com
push dateFri, 12 Apr 2019 16:20:17 +0000
treeherdermozilla-inbound@b4501ced5619 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscatlee
bugs1543662
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1543662 Introduce channel verification to partials r=catlee Differential Revision: https://phabricator.services.mozilla.com/D27115
taskcluster/docker/funsize-update-generator/scripts/funsize.py
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -88,16 +88,34 @@ def is_lzma_compressed_mar(mar):
     result = MarReader(open(mar, 'rb')).compression_type == 'xz'
     if result:
         log.info("%s is lzma compressed", mar)
     else:
         log.info("%s is not lzma compressed", mar)
     return result
 
 
+def validate_mar_channel_id(mar, channel_ids):
+    log.info("Checking %s for MAR_CHANNEL_ID %s", mar, channel_ids)
+    # We may get a string with a list representation, or a single entry string.
+    channel_ids = set(channel_ids.split(','))
+
+    product_info = MarReader(open(mar, 'rb')).productinfo
+    if not isinstance(product_info, tuple):
+        raise ValueError("Malformed product information in mar: {}".format(product_info))
+
+    found_channel_ids = set(product_info[1].split(','))
+
+    if not found_channel_ids.issubset(channel_ids):
+        raise ValueError("MAR_CHANNEL_ID mismatch, {} not in {}".format(
+            product_info[1], channel_ids))
+
+    log.info("%s channel %s in %s", mar, product_info[1], channel_ids)
+
+
 @redo.retriable()
 def get_secret(secret_name):
     secrets_url = "http://taskcluster/secrets/v1/secret/{}"
     log.debug("Fetching {}".format(secret_name))
     r = requests.get(secrets_url.format(secret_name))
     # 403: If unauthorized, just give up.
     if r.status_code == 403:
         log.info("Unable to get secret key")
@@ -225,16 +243,17 @@ async def generate_partial(work_env, fro
         env['MAR_OLD_FORMAT'] = '1'
     elif 'MAR_OLD_FORMAT' in env:
         del env['MAR_OLD_FORMAT']
     make_incremental_update = os.path.join(work_env.workdir,
                                            "make_incremental_update.sh")
     cmd = " ".join([make_incremental_update, dest_mar, from_dir, to_dir])
 
     await run_command(cmd, cwd=work_env.workdir, env=env, label=dest_mar.split('/')[-1])
+    validate_mar_channel_id(dest_mar, mar_data["ACCEPTED_MAR_CHANNEL_IDS"])
 
 
 def get_hash(path, hash_type="sha512"):
     h = hashlib.new(hash_type)
     with open(path, "rb") as f:
         h.update(f.read())
     return h.hexdigest()
 
@@ -305,17 +324,17 @@ async def manage_partial(partial_def, fi
     )
     await work_env.setup()
 
     for mar in (partial_def["from_mar"], partial_def["to_mar"]):
         verify_allowed_url(mar, allowed_url_prefixes)
 
     complete_mars = {}
     use_old_format = False
-
+    check_channels_in_files = list()
     for mar_type, f in (("from", partial_def["from_mar"]), ("to", partial_def["to_mar"])):
         dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
         unpack_dir = os.path.join(work_env.workdir, mar_type)
 
         metric_tags = [
             "platform:{}".format(partial_def['platform']),
         ]
         with ddstats.timer('mar.download.time', tags=metric_tags):
@@ -325,16 +344,18 @@ async def manage_partial(partial_def, fi
             verify_signature(dest, signing_certs)
 
         complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
         complete_mars["%s_hash" % mar_type] = get_hash(dest)
 
         with ddstats.timer('mar.unpack.time'):
             await unpack(work_env, dest, unpack_dir)
 
+        check_channels_in_files.append(dest)
+
         if mar_type == 'from':
             version = get_option(unpack_dir, filename="application.ini",
                                  section="App", option="Version")
             major = int(version.split(".")[0])
             # The updater for versions less than 56.0 requires BZ2
             # compressed MAR files
             if major < 56:
                 use_old_format = True
@@ -365,16 +386,20 @@ async def manage_partial(partial_def, fi
         "from_mar": partial_def["from_mar"],
         "to_mar": partial_def["to_mar"],
         "platform": partial_def["platform"],
         "locale": partial_def["locale"],
     }
     # Override ACCEPTED_MAR_CHANNEL_IDS if needed
     if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ:
         mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ["ACCEPTED_MAR_CHANNEL_IDS"]
+
+    for filename in check_channels_in_files:
+        validate_mar_channel_id(filename, mar_data["ACCEPTED_MAR_CHANNEL_IDS"])
+
     for field in ("update_number", "previousVersion", "previousBuildNumber",
                   "toVersion", "toBuildNumber"):
         if field in partial_def:
             mar_data[field] = partial_def[field]
     mar_data.update(complete_mars)
 
     # if branch not set explicitly use repo-name
     mar_data['branch'] = partial_def.get('branch', mar_data['repo'].rstrip('/').split('/')[-1])