Bug 1442545: [partner-repack] Add parameters needed for doing partner repacks; r=Callek, a=release
authorTom Prince <mozilla@hocat.ca>
Mon, 16 Apr 2018 20:48:40 -0600
changeset 460951 5ee837cbe55c00871123042552a36dd2b6138186
parent 460950 746f2e444725ed36cd0a30eefd3c41a5d53d86df
child 460952 bf9420bfbd0a7b7043652f833e87fd18399e279f
push id9098
push userbhearsum@mozilla.com
push dateThu, 19 Apr 2018 20:02:16 +0000
treeherdermozilla-beta@98d536130349 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek, release
bugs1442545
milestone60.0
Bug 1442545: [partner-repack] Add parameters needed for doing partner repacks; r=Callek, a=release Differential Revision: https://phabricator.services.mozilla.com/D977
taskcluster/docs/parameters.rst
taskcluster/taskgraph/actions/release_promotion.py
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/parameters.py
taskcluster/taskgraph/util/partners.py
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -154,16 +154,31 @@ Release Promotion
    Specify the next version for version bump tasks.
 
 ``release_type``
    The type of release being promoted. One of "beta", "devedition", "esr", "rc", or "release".
 
 ``release_eta``
    The time and date when a release is scheduled to live. This value is passed to Balrog.
 
+``release_enable_partners``
+   Boolean which controls repacking vanilla Firefox builds for partners.
+
+``release_partners``
+   List of partners to repack. A null value defaults to all.
+
+``release_partner_config``
+   Configuration for partner repacks.
+
+``release_partner_build_number``
+   The build number for partner repacks. We sometimes have multiple partner build numbers per release build number; this parameter lets us bump them independently. Defaults to 1.
+
+``release_enable_emefree``
+   Boolean which controls repacking vanilla Firefox builds into EME-free builds.
+
 Comm Push Information
 ---------------------
 
 These parameters correspond to the repository and revision of the comm-central
 repository to checkout. Their meaning is the same as the corresponding
 parameters for the gecko repository above. They are optional, but if any of
 them are specified, they must all be specified.
 
--- a/taskcluster/taskgraph/actions/release_promotion.py
+++ b/taskcluster/taskgraph/actions/release_promotion.py
@@ -10,16 +10,17 @@ import json
 import os
 
 from .registry import register_callback_action
 
 from .util import (find_decision_task, find_existing_tasks_from_previous_kinds,
                    find_hg_revision_pushlog_id)
 from taskgraph.util.taskcluster import get_artifact
 from taskgraph.util.partials import populate_release_history
+from taskgraph.util.partners import fix_partner_config
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.decision import taskgraph_decision
 from taskgraph.parameters import Parameters
 from taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS
 
 RELEASE_PROMOTION_CONFIG = {
     'promote_fennec': {
         'target_tasks_method': 'promote_fennec',
@@ -51,16 +52,28 @@ RELEASE_PROMOTION_CONFIG = {
         'product': 'firefox',
         'release_type': 'rc',
     },
     'ship_firefox_rc': {
         'target_tasks_method': 'ship_firefox',
         'product': 'firefox',
         'release_type': 'rc',
     },
+    'promote_firefox_partners': {
+        'target_tasks_method': 'promote_firefox',
+        'product': 'firefox',
+        'rebuild_kinds': [
+            'release-partner-repack',
+            'release-partner-beetmover',
+            'release-partner-repack-chunking-dummy',
+            'release-partner-repackage-signing',
+            'release-partner-repackage',
+            'release-partner-signing',
+        ],
+    },
     'promote_devedition': {
         'target_tasks_method': 'promote_devedition',
         'product': 'devedition',
     },
     'push_devedition': {
         'target_tasks_method': 'push_devedition',
         'product': 'devedition',
     },
@@ -83,16 +96,19 @@ PARTIAL_UPDATES_FLAVORS = (
     'push_firefox',
     'push_firefox_rc',
     'push_devedition',
     'ship_firefox',
     'ship_firefox_rc',
     'ship_devedition',
 )
 
+PARTNER_BRANCHES = ('mozilla-beta', 'mozilla-release', 'maple', 'birch', 'jamun')
+EMEFREE_BRANCHES = ('mozilla-beta', 'mozilla-release', 'maple', 'birch', 'jamun')
+
 
 def is_release_promotion_available(parameters):
     return parameters['project'] in RELEASE_PROMOTION_PROJECTS
 
 
 @register_callback_action(
     name='release-promotion',
     title='Release Promotion',
@@ -198,21 +214,52 @@ def is_release_promotion_available(param
                     },
                     'required': [
                         'buildNumber',
                         'locales',
                     ],
                     'additionalProperties': False,
                 }
             },
-
             'release_eta': {
                 'type': 'string',
                 'default': '',
             },
+            'release_enable_partners': {
+                'type': 'boolean',
+                'default': False,
+                'description': ('Toggle for creating partner repacks'),
+            },
+            'release_partner_build_number': {
+                'type': 'integer',
+                'default': 1,
+                'minimum': 1,
+                'description': ('The partner build number. This translates to, e.g. '
+                                '`v1` in the path. We generally only have to '
+                                'bump this on off-cycle partner rebuilds.'),
+            },
+            'release_partners': {
+                'type': 'array',
+                'description': ('A list of partners to repack, or if null or empty then use '
+                                'the current full set'),
+                'items': {
+                    'type': 'string',
+                }
+            },
+            'release_partner_config': {
+                'type': 'object',
+                'description': ('Partner configuration to use for partner repacks.'),
+                'properties': {},
+                'additionalProperties': True,
+            },
+            'release_enable_emefree': {
+                'type': 'boolean',
+                'default': False,
+                'description': ('Toggle for creating EME-free repacks'),
+            },
         },
         "required": ['release_promotion_flavor', 'build_number'],
     }
 )
 def release_promotion_action(parameters, input, task_group_id, task_id, task):
     release_promotion_flavor = input['release_promotion_flavor']
     promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
     release_history = {}
@@ -248,16 +295,24 @@ def release_promotion_action(parameters,
         project=parameters['project']
     )
     rebuild_kinds = input.get(
         'rebuild_kinds', promotion_config.get('rebuild_kinds', [])
     )
     do_not_optimize = input.get(
         'do_not_optimize', promotion_config.get('do_not_optimize', [])
     )
+    release_enable_partners = input.get(
+        'release_enable_partners',
+        parameters['project'] in PARTNER_BRANCHES and product in ('firefox',)
+    )
+    release_enable_emefree = input.get(
+        'release_enable_emefree',
+        parameters['project'] in EMEFREE_BRANCHES and product in ('firefox',)
+    )
 
     # make parameters read-write
     parameters = dict(parameters)
     # Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
     # or ``revision``.
     previous_graph_ids = input.get('previous_graph_ids')
     if not previous_graph_ids:
         revision = input.get('revision')
@@ -282,15 +337,21 @@ def release_promotion_action(parameters,
     )
     parameters['do_not_optimize'] = do_not_optimize
     parameters['target_tasks_method'] = target_tasks_method
     parameters['build_number'] = int(input['build_number'])
     parameters['next_version'] = next_version
     parameters['release_history'] = release_history
     parameters['release_type'] = promotion_config.get('release_type', '')
     parameters['release_eta'] = input.get('release_eta', '')
+    parameters['release_enable_partners'] = release_enable_partners
+    parameters['release_partners'] = input.get('release_partners')
+    if input.get('release_partner_config'):
+        parameters['release_partner_config'] = fix_partner_config(input['release_partner_config'])
+    parameters['release_enable_emefree'] = release_enable_emefree
+
     if input['version']:
         parameters['version'] = input['version']
 
     # make parameters read-only
     parameters = Parameters(**parameters)
 
     taskgraph_decision({}, parameters=parameters)
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -193,16 +193,21 @@ def get_decision_parameters(options):
     parameters['existing_tasks'] = {}
     parameters['do_not_optimize'] = []
     parameters['build_number'] = 1
     parameters['version'] = get_version()
     parameters['app_version'] = get_app_version()
     parameters['next_version'] = None
     parameters['release_type'] = ''
     parameters['release_eta'] = ''
+    parameters['release_enable_partners'] = False
+    parameters['release_partners'] = []
+    parameters['release_partner_config'] = {}
+    parameters['release_partner_build_number'] = 1
+    parameters['release_enable_emefree'] = False
 
     # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
     # case, fake it
     if '@' not in parameters['owner']:
         parameters['owner'] += '@noreply.mozilla.org'
 
     # use the pushdate as build_date if given, else use current time
     parameters['build_date'] = parameters['pushdate'] or int(time.time())
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -55,18 +55,23 @@ PARAMETERS = {
     'message': '',
     'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"),
     'next_version': None,
     'optimize_target_tasks': True,
     'owner': 'nobody@mozilla.com',
     'project': 'mozilla-central',
     'pushdate': lambda: int(time.time()),
     'pushlog_id': '0',
+    'release_enable_emefree': False,
+    'release_enable_partners': False,
     'release_eta': '',
     'release_history': {},
+    'release_partners': None,
+    'release_partner_config': None,
+    'release_partner_build_number': 1,
     'release_type': '',
     'target_tasks_method': 'default',
     'try_mode': None,
     'try_options': None,
     'try_task_config': None,
     'version': get_version(),
 }
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/partners.py
@@ -0,0 +1,103 @@
+from __future__ import absolute_import, print_function, unicode_literals
+
+from copy import deepcopy
+import json
+import os
+
+LOCALES_FILE = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
+    'browser', 'locales', 'l10n-changesets.json'
+)
+
+
+def check_if_partners_enabled(config, tasks):
+    if (
+        config.params['release_enable_partners'] and
+        config.kind.startswith('release-partner-repack')
+    ) or (
+        config.params['release_enable_emefree'] and
+        config.kind.startswith('release-eme-free-repack')
+    ):
+        for task in tasks:
+            yield task
+
+
+def get_partner_config_by_kind(config, kind):
+    """ Retrieve partner data starting from the manifest url, which points to a repository
+    containing a default.xml that is intended to be drive the Google tool 'repo'. It
+    descends into each partner repo to lookup and parse the repack.cfg file(s).
+
+    Supports caching data by kind to avoid repeated requests, relying on the related kinds for
+    partner repacking, signing, repackage, repackage signing all having the same kind prefix.
+    """
+    partner_subset = config.params['release_partners']
+    partner_configs = config.params['release_partner_config'] or {}
+
+    # TODO eme-free should be a partner; we shouldn't care about per-kind
+    for k in partner_configs:
+        if kind.startswith(k):
+            kind_config = partner_configs[k]
+            break
+    else:
+        return {}
+    # if we're only interested in a subset of partners we remove the rest
+    if isinstance(partner_subset, (list, tuple)):
+        # TODO - should be fatal to have an unknown partner in partner_subset
+        for partner in kind_config.keys():
+            if partner not in partner_subset:
+                del(kind_config[partner])
+
+    return kind_config
+
+
+def _fix_subpartner_locales(orig_config, all_locales):
+    subpartner_config = deepcopy(orig_config)
+    # Get an ordered list of subpartner locales that is a subset of all_locales
+    subpartner_config['locales'] = sorted(list(
+        set(orig_config['locales']) & set(all_locales)
+    ))
+    return subpartner_config
+
+
+def fix_partner_config(orig_config):
+    pc = {}
+    with open(LOCALES_FILE, 'r') as fh:
+        all_locales = json.load(fh).keys()
+    # l10n-changesets.json doesn't include en-US, but the repack list does
+    if 'en-US' not in all_locales:
+        all_locales.append('en-US')
+    for kind, kind_config in orig_config.iteritems():
+        for partner, partner_config in kind_config.iteritems():
+            for subpartner, subpartner_config in partner_config.iteritems():
+                # get rid of empty subpartner configs
+                if not subpartner_config:
+                    continue
+                # Make sure our locale list is a subset of all_locales
+                pc.setdefault(kind, {}).setdefault(partner, {})[subpartner] = \
+                    _fix_subpartner_locales(subpartner_config, all_locales)
+    return pc
+
+
+# seems likely this exists elsewhere already
+def get_ftp_platform(platform):
+    if platform.startswith('win32'):
+        return 'win32'
+    elif platform.startswith('win64'):
+        return 'win64'
+    elif platform.startswith('macosx'):
+        return 'mac'
+    elif platform.startswith('linux-'):
+        return 'linux-i686'
+    elif platform.startswith('linux64'):
+        return 'linux-x86_64'
+    else:
+        raise ValueError('Unimplemented platform %s'.format(platform))
+
+
+# Ugh
+def locales_per_build_platform(build_platform, locales):
+    if build_platform.startswith('mac'):
+        exclude = ['ja']
+    else:
+        exclude = ['ja-JP-mac']
+    return [locale for locale in locales if locale not in exclude]