Bug 1473416: [release] Allow staging releases to build partials from http://ftp.stage.mozaws.net/; r=sfraser
authorTom Prince <mozilla@hocat.ca>
Fri, 12 Oct 2018 17:53:20 +0000
changeset 499378 74ee6e2258c119cb2795b74f836fac35d3679448
parent 499377 32976a484a5ee5060962cbe84475f2bfddbece08
child 499379 d0b593b6d065227a080fc483eb73919dc020a2de
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfraser
bugs1473416
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1473416: [release] Allow staging releases to build partials from http://ftp.stage.mozaws.net/; r=sfraser The partial generation code checks the URLs of the source versions. To allow building partials from staging releases, allow the staging CDN when generating partials. Differential Revision: https://phabricator.services.mozilla.com/D5710
taskcluster/docker/funsize-update-generator/scripts/funsize.py
taskcluster/taskgraph/transforms/partials.py
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -13,43 +13,47 @@ import hashlib
 import json
 import logging
 import os
 import shutil
 import tempfile
 import time
 import requests
 import sh
+from distutils.util import strtobool
 
 import redo
 from scriptworker.utils import retry_async
 from mardor.reader import MarReader
 from mardor.signing import get_keysize
 
 from datadog import initialize, ThreadStats
 
 
 log = logging.getLogger(__name__)
 
 # Create this even when not sending metrics, so the context manager
 # statements work.
 ddstats = ThreadStats(namespace='releng.releases.partials')
 
 
-ALLOWED_URL_PREFIXES = [
+ALLOWED_URL_PREFIXES = (
     "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/",
     "http://download.cdn.mozilla.net/pub/firefox/nightly/",
     "https://mozilla-nightly-updates.s3.amazonaws.com",
     "https://queue.taskcluster.net/",
     "http://ftp.mozilla.org/",
     "http://download.mozilla.org/",
     "https://archive.mozilla.org/",
     "http://archive.mozilla.org/",
     "https://queue.taskcluster.net/v1/task/",
-]
+)
+STAGING_URL_PREFIXES = (
+    "http://ftp.stage.mozaws.net/",
+)
 
 DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
                             "{locale}-{from_buildid}-{to_buildid}.partial.mar"
 
 
 def write_dogrc(api_key):
     """Datadog .dogrc file for command line interface."""
     dogrc_path = os.path.join(os.path.expanduser('~'), '.dogrc')
@@ -224,31 +228,32 @@ def get_hash(path, hash_type="sha512"):
     h = hashlib.new(hash_type)
     with open(path, "rb") as f:
         h.update(f.read())
     return h.hexdigest()
 
 
 class WorkEnv(object):
 
-    def __init__(self, mar=None, mbsdiff=None):
+    def __init__(self, allowed_url_prefixes, mar=None, mbsdiff=None):
         self.workdir = tempfile.mkdtemp()
         self.paths = {
             'unwrap_full_update.pl': os.path.join(self.workdir, 'unwrap_full_update.pl'),
             'mar': os.path.join(self.workdir, 'mar'),
             'mbsdiff': os.path.join(self.workdir, 'mbsdiff')
         }
         self.urls = {
             'unwrap_full_update.pl': 'https://hg.mozilla.org/mozilla-central/raw-file/default/'
             'tools/update-packaging/unwrap_full_update.pl',
             'mar': 'https://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/'
             'latest-mozilla-central/mar-tools/linux64/mar',
             'mbsdiff': 'https://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/'
             'latest-mozilla-central/mar-tools/linux64/mbsdiff'
         }
+        self.allowed_url_prefixes = allowed_url_prefixes
         if mar:
             self.urls['mar'] = mar
         if mbsdiff:
             self.urls['mbsdiff'] = mbsdiff
 
     async def setup(self, mar=None, mbsdiff=None):
         for filename, url in self.urls.items():
             if filename not in self.paths:
@@ -270,27 +275,27 @@ class WorkEnv(object):
     def env(self):
         my_env = os.environ.copy()
         my_env['LC_ALL'] = 'C'
         my_env['MAR'] = self.paths['mar']
         my_env['MBSDIFF'] = self.paths['mbsdiff']
         return my_env
 
 
-def verify_allowed_url(mar):
-    if not any(mar.startswith(prefix) for prefix in ALLOWED_URL_PREFIXES):
+def verify_allowed_url(mar, allowed_url_prefixes):
+    if not any(mar.startswith(prefix) for prefix in allowed_url_prefixes):
         raise ValueError("{mar} is not in allowed URL prefixes: {p}".format(
-            mar=mar, p=ALLOWED_URL_PREFIXES
+            mar=mar, p=allowed_url_prefixes
         ))
 
 
 async def manage_partial(partial_def, work_env, filename_template, artifacts_dir, signing_certs):
     """Manage the creation of partial mars based on payload."""
     for mar in (partial_def["from_mar"], partial_def["to_mar"]):
-        verify_allowed_url(mar)
+        verify_allowed_url(mar, work_env.allowed_url_prefixes)
 
     complete_mars = {}
     use_old_format = False
 
     for mar_type, f in (("from", partial_def["from_mar"]), ("to", partial_def["to_mar"])):
         dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
         unpack_dir = os.path.join(work_env.workdir, mar_type)
 
@@ -402,20 +407,25 @@ async def manage_partial(partial_def, wo
     work_env.cleanup()
 
     return mar_data
 
 
 async def async_main(args, signing_certs):
     tasks = []
 
+    allowed_url_prefixes = list(ALLOWED_URL_PREFIXES)
+    if args.allow_staging_prefixes:
+        allowed_url_prefixes += STAGING_URL_PREFIXES
+
     task = json.load(args.task_definition)
     # TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
     for definition in task["extra"]["funsize"]["partials"]:
         workenv = WorkEnv(
+            allowed_url_prefixes=allowed_url_prefixes,
             mar=definition.get('mar_binary'),
             mbsdiff=definition.get('mbsdiff_binary')
         )
         await workenv.setup()
         tasks.append(asyncio.ensure_future(retry_async(
                                            manage_partial,
                                            retry_exceptions=(
                                                aiohttp.ClientError,
@@ -437,16 +447,21 @@ def main():
     start = time.time()
 
     parser = argparse.ArgumentParser()
     parser.add_argument("--artifacts-dir", required=True)
     parser.add_argument("--sha1-signing-cert", required=True)
     parser.add_argument("--sha384-signing-cert", required=True)
     parser.add_argument("--task-definition", required=True,
                         type=argparse.FileType('r'))
+    parser.add_argument("--allow-staging-prefixes",
+                        action="store_true",
+                        default=strtobool(
+                            os.environ.get('FUNSIZE_ALLOW_STAGING_PREFIXES', "false")),
+                        help="Allow files from staging buckets.")
     parser.add_argument("--filename-template",
                         default=DEFAULT_FILENAME_TEMPLATE)
     parser.add_argument("--no-freshclam", action="store_true", default=False,
                         help="Do not refresh ClamAV DB")
     parser.add_argument("-q", "--quiet", dest="log_level",
                         action="store_const", const=logging.WARNING,
                         default=logging.DEBUG)
     args = parser.parse_args()
--- a/taskcluster/taskgraph/transforms/partials.py
+++ b/taskcluster/taskgraph/transforms/partials.py
@@ -139,16 +139,18 @@ def make_task_description(config, jobs):
                 'SHA1_SIGNING_CERT': 'nightly_sha1',
                 'SHA384_SIGNING_CERT': 'nightly_sha384',
                 'DATADOG_API_SECRET':
                     'project/releng/gecko/build/level-{}/datadog-api-key'.format(level),
             }
         }
         if mar_channel_id:
             worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
+        if config.params.release_level() == 'staging':
+            worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true'
 
         task = {
             'label': label,
             'description': "{} Partials".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
             'dependencies': dependencies,
             'scopes': [