bug 1457034 - generate `release_partner_config` in `release_promotion` r=nthomas
authorAki Sasaki <asasaki@mozilla.com>
Wed, 25 Apr 2018 15:16:49 -0700
changeset 463617 878f15390bf7
parent 463616 a66cd30297c4
child 463618 00b58dfcf206
push id1711
push userasasaki@mozilla.com
push date2018-05-09 16:09 +0000
treeherdermozilla-release@00b58dfcf206 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnthomas
bugs1457034
milestone60.0.1
bug 1457034 - generate `release_partner_config` in `release_promotion` r=nthomas If we're not passed `release_partner_config` in the input, let's poll github for it in the action. MozReview-Commit-ID: 2swx76vhTE5
taskcluster/ci/config.yml
taskcluster/ci/release-eme-free-repack/kind.yml
taskcluster/ci/release-partner-repack/kind.yml
taskcluster/taskgraph/actions/release_promotion.py
taskcluster/taskgraph/config.py
taskcluster/taskgraph/transforms/partner_repack.py
taskcluster/taskgraph/util/partners.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -119,8 +119,17 @@ scriptworker:
         'scriptworker-prov-v1/balrogworker-v1':
             - 'project:releng:balrog:server:nightly'
             - 'project:releng:balrog:server:aurora'
             - 'project:releng:balrog:server:beta'
             - 'project:releng:balrog:server:release'
             - 'project:releng:balrog:server:esr'
         'scriptworker-prov-v1/balrog-dev':
             - 'project:releng:balrog:server:dep'
+
+
+partner:
+    release:
+        release-partner-repack: git@github.com:mozilla-partners/repack-manifests.git
+        release-eme-free-repack: git@github.com:mozilla-partners/mozilla-EME-free-manifest
+    staging:
+        release-partner-repack: git@github.com:mozilla-releng/staging-repack-manifests.git
+        release-eme-free-repack: git@github.com:mozilla-releng/staging-repack-manifests.git
--- a/taskcluster/ci/release-eme-free-repack/kind.yml
+++ b/taskcluster/ci/release-eme-free-repack/kind.yml
@@ -21,23 +21,16 @@ job-defaults:
    shipping-product: firefox
    shipping-phase: promote
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
       docker-image:
          in-tree: "partner-repack"
       chain-of-trust: true
       max-run-time: 7200
-      env:
-         REPACK_MANIFESTS_URL:
-            by-project:
-               mozilla-beta: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-               mozilla-release: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-               maple: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-               default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
    run:
       using: mozharness
       config:
          - partner_repacks/release_mozilla-release_desktop.py
       script: mozharness/scripts/desktop_partner_repacks.py
       job-script: taskcluster/scripts/builder/repackage.sh
       need-xvfb: false
       tooltool-downloads: false
--- a/taskcluster/ci/release-partner-repack/kind.yml
+++ b/taskcluster/ci/release-partner-repack/kind.yml
@@ -21,23 +21,16 @@ job-defaults:
    shipping-product: firefox
    shipping-phase: promote
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
       docker-image:
          in-tree: "partner-repack"
       chain-of-trust: true
       max-run-time: 7200
-      env:
-         REPACK_MANIFESTS_URL:
-            by-project:
-               mozilla-beta: "git@github.com:mozilla-partners/repack-manifests.git"
-               mozilla-release: "git@github.com:mozilla-partners/repack-manifests.git"
-               maple: "git@github.com:mozilla-partners/repack-manifests.git"
-               default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
    run:
       using: mozharness
       config:
          - partner_repacks/release_mozilla-release_desktop.py
       script: mozharness/scripts/desktop_partner_repacks.py
       job-script: taskcluster/scripts/builder/repackage.sh
       need-xvfb: false
       tooltool-downloads: false
--- a/taskcluster/taskgraph/actions/release_promotion.py
+++ b/taskcluster/taskgraph/actions/release_promotion.py
@@ -10,17 +10,24 @@ import json
 import os
 
 from .registry import register_callback_action
 
 from .util import (find_decision_task, find_existing_tasks_from_previous_kinds,
                    find_hg_revision_pushlog_id)
 from taskgraph.util.taskcluster import get_artifact
 from taskgraph.util.partials import populate_release_history
-from taskgraph.util.partners import fix_partner_config
+from taskgraph.util.partners import (
+    EMEFREE_BRANCHES,
+    PARTNER_BRANCHES,
+    fix_partner_config,
+    get_partner_config_by_url,
+    get_partner_url_config,
+    get_token
+)
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.decision import taskgraph_decision
 from taskgraph.parameters import Parameters
 from taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS
 
 RELEASE_PROMOTION_CONFIG = {
     'promote_fennec': {
         'target_tasks_method': 'promote_fennec',
@@ -96,24 +103,28 @@ PARTIAL_UPDATES_FLAVORS = (
     'push_firefox',
     'push_firefox_rc',
     'push_devedition',
     'ship_firefox',
     'ship_firefox_rc',
     'ship_devedition',
 )
 
-PARTNER_BRANCHES = ('mozilla-beta', 'mozilla-release', 'maple', 'birch', 'jamun')
-EMEFREE_BRANCHES = ('mozilla-beta', 'mozilla-release', 'maple', 'birch', 'jamun')
-
 
 def is_release_promotion_available(parameters):
     return parameters['project'] in RELEASE_PROMOTION_PROJECTS
 
 
+def get_partner_config(partner_url_config, github_token):
+    partner_config = {}
+    for kind, url in partner_url_config.items():
+        partner_config[kind] = get_partner_config_by_url(url, kind, github_token)
+    return partner_config
+
+
 @register_callback_action(
     name='release-promotion',
     title='Release Promotion',
     symbol='${input.release_promotion_flavor}',
     description="Promote a release.",
     order=10000,
     context=[],
     available=is_release_promotion_available,
@@ -339,19 +350,29 @@ def release_promotion_action(parameters,
     parameters['target_tasks_method'] = target_tasks_method
     parameters['build_number'] = int(input['build_number'])
     parameters['next_version'] = next_version
     parameters['release_history'] = release_history
     parameters['release_type'] = promotion_config.get('release_type', '')
     parameters['release_eta'] = input.get('release_eta', '')
     parameters['release_enable_partners'] = release_enable_partners
     parameters['release_partners'] = input.get('release_partners')
-    if input.get('release_partner_config'):
-        parameters['release_partner_config'] = fix_partner_config(input['release_partner_config'])
     parameters['release_enable_emefree'] = release_enable_emefree
 
+    partner_config = input.get('release_partner_config')
+    if not partner_config and (release_enable_emefree or release_enable_partners):
+        partner_url_config = get_partner_url_config(
+            parameters, graph_config, enable_emefree=release_enable_emefree,
+            enable_partners=release_enable_partners
+        )
+        github_token = get_token(parameters)
+        partner_config = get_partner_config(partner_url_config, github_token)
+
+    if partner_config:
+        parameters['release_partner_config'] = fix_partner_config(partner_config)
+
     if input['version']:
         parameters['version'] = input['version']
 
     # make parameters read-only
     parameters = Parameters(**parameters)
 
     taskgraph_decision({}, parameters=parameters)
--- a/taskcluster/taskgraph/config.py
+++ b/taskcluster/taskgraph/config.py
@@ -27,13 +27,19 @@ graph_config_schema = Schema({
         Required('ridealong-builds', default={}): {basestring: [basestring]},
     },
     Required('scriptworker'): {
         # Prefix to add to scopes controlling scriptworkers
         Required('scope-prefix'): basestring,
         # Mapping of scriptworker types to scopes they accept
         Required('worker-types'): {basestring: [basestring]}
     },
+    Required('partner'): {
+        # Release config for partner repacks
+        Required('release'): {basestring: basestring},
+        # Staging config for partner repacks
+        Required('staging'): {basestring: basestring},
+    },
 })
 
 
 def validate_graph_config(config):
     return validate_schema(graph_config_schema, config, "Invalid graph configuration:")
--- a/taskcluster/taskgraph/transforms/partner_repack.py
+++ b/taskcluster/taskgraph/transforms/partner_repack.py
@@ -3,32 +3,40 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the partner repack task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import resolve_keyed_by
 from taskgraph.util.scriptworker import get_release_config
-from taskgraph.util.partners import check_if_partners_enabled
+from taskgraph.util.partners import (
+    check_if_partners_enabled,
+    get_partner_url_config,
+)
 
 
 transforms = TransformSequence()
 
 transforms.add(check_if_partners_enabled)
 
 
 @transforms.add
-def resolve_properties(config, tasks):
+def populate_repack_manifests_url(config, tasks):
     for task in tasks:
-        for property in ("REPACK_MANIFESTS_URL", ):
-            property = "worker.env.{}".format(property)
-            resolve_keyed_by(task, property, property, **config.params)
+        partner_url_config = get_partner_url_config(config.params, config.graph_config)
+
+        for k in partner_url_config:
+            if config.kind.startswith(k):
+                task['worker'].setdefault('env', {})['REPACK_MANIFESTS_URL'] = \
+                    partner_url_config[k]
+                break
+        else:
+            raise Exception("Can't find partner REPACK_MANIFESTS_URL")
 
         if task['worker']['env']['REPACK_MANIFESTS_URL'].startswith('git@'):
             task.setdefault('scopes', []).append(
                 'secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh'.format(
                     **config.params
                 )
             )
 
@@ -47,16 +55,19 @@ def add_command_arguments(config, tasks)
     release_config = get_release_config(config)
     for task in tasks:
         # add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64
         task['run']['options'] = [
             'version={}'.format(release_config['version']),
             'build-number={}'.format(release_config['build_number']),
             'platform={}'.format(task['attributes']['build_platform'].split('-')[0]),
         ]
+        if 'partner' in config.kind and config.params['release_partners']:
+            for partner in config.params['release_partners']:
+                task['run']['options'].append('p={}'.format(partner))
 
         # The upstream taskIds are stored a special environment variable, because we want to use
         # task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS
         # blocks that. It's space-separated string of ids in the end.
         task['worker']['env']['UPSTREAM_TASKIDS'] = {
             'task-reference': ' '.join(['<{}>'.format(dep) for dep in task['dependencies']])
         }
 
--- a/taskcluster/taskgraph/util/partners.py
+++ b/taskcluster/taskgraph/util/partners.py
@@ -1,24 +1,39 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 from copy import deepcopy
 import json
 import logging
 import os
+from redo import retry
 import requests
 import xml.etree.ElementTree as ET
 
 
 # Suppress chatty requests logging
 logging.getLogger("requests").setLevel(logging.WARNING)
 
 log = logging.getLogger(__name__)
 
 GITHUB_API_ENDPOINT = "https://api.github.com/graphql"
+PARTNER_BRANCHES = {
+    'mozilla-beta': 'release',
+    'mozilla-release': 'release',
+    'maple': 'release',
+    'birch': 'release',
+    'jamun': 'release',
+}
+EMEFREE_BRANCHES = {
+    'mozilla-beta': 'release',
+    'mozilla-release': 'release',
+    'maple': 'release',
+    'birch': 'release',
+    'jamun': 'release',
+}
 
 """
 LOGIN_QUERY, MANIFEST_QUERY, and REPACK_CFG_QUERY are all written to the Github v4 API,
 which users GraphQL. See https://developer.github.com/v4/
 """
 
 LOGIN_QUERY = """query {
   viewer {
@@ -125,27 +140,51 @@ Example response:
 TC_PLATFORM_PER_FTP = {
     'linux-i686': 'linux-nightly',
     'linux-x86_64': 'linux64-nightly',
     'mac': 'macosx64-nightly',
     'win32': 'win32-nightly',
     'win64': 'win64-nightly',
 }
 
-TASKCLUSTER_PROXY_SECRET_ROOT = 'http://taskcluster/secrets/v1/secret/'
+TASKCLUSTER_PROXY_SECRET_ROOT = 'http://taskcluster/secrets/v1/secret'
 
 LOCALES_FILE = os.path.join(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
     'browser', 'locales', 'l10n-changesets.json'
 )
 
 # cache data at the module level
 partner_configs = {}
 
 
+# TODO - grant private repo access to P.A.T.
+# TODO - add level-3 as well, cleanup as of level
+def get_token(params):
+    """ We use a Personal Access Token from Github to lookup partner config. No extra scopes are
+    needed on the token to read public repositories, but need the 'repo' scope to see private
+    repositories. This is not fine grained and also grants r/w access, but is revoked at the repo
+    level.
+    """
+
+    # The 'usual' method - via taskClusterProxy for decision tasks
+    # TODO use {level}? Or allow the token to level 1 and remove level from the path?
+    url = "{secret_root}/project/releng/gecko/build/level-2/partner-github-api".format(
+        secret_root=TASKCLUSTER_PROXY_SECRET_ROOT, **params
+    )
+    try:
+        resp = retry(requests.get, attempts=2, sleeptime=10,
+                     args=(url, ),
+                     kwargs={'timeout': 60, 'headers': ''})
+        j = resp.json()
+        return j['secret']['key']
+    except (requests.ConnectionError, ValueError, KeyError):
+        raise RuntimeError('Could not get Github API token to lookup partner data')
+
+
 def query_api(query, token):
     """ Make a query with a Github auth header, returning the json """
     headers = {'Authorization': 'bearer %s' % token}
     r = requests.post(GITHUB_API_ENDPOINT, json={'query': query}, headers=headers)
     r.raise_for_status()
 
     j = r.json()
     if 'errors' in j:
@@ -261,19 +300,17 @@ def get_partner_config_by_url(manifest_u
         log.info('Looking up data for %s from %s', kind, manifest_url)
         check_login(token)
         partners = get_partners(manifest_url, token)
 
         partner_configs[kind] = {}
         for partner, partner_url in partners.items():
             if partner_subset and partner not in partner_subset:
                 continue
-            partner_configs[kind][partner] = get_repack_configs(
-                partner_url, token, partner_subset
-            )
+            partner_configs[kind][partner] = get_repack_configs(partner_url, token)
     return partner_configs[kind]
 
 
 def check_if_partners_enabled(config, tasks):
     if (
         config.params['release_enable_partners'] and
         config.kind.startswith('release-partner-repack')
     ) or (
@@ -358,8 +395,22 @@ def get_ftp_platform(platform):
 
 # Ugh
 def locales_per_build_platform(build_platform, locales):
     if build_platform.startswith('mac'):
         exclude = ['ja']
     else:
         exclude = ['ja-JP-mac']
     return [locale for locale in locales if locale not in exclude]
+
+
+def get_partner_url_config(parameters, graph_config, enable_emefree=True, enable_partners=True):
+    partner_url_config = {}
+    project = parameters['project']
+    if enable_emefree:
+        alias = EMEFREE_BRANCHES[project]
+        partner_url_config['release-eme-free-repack'] = \
+            graph_config['partner'][alias]['release-eme-free-repack']
+    if enable_partners:
+        alias = PARTNER_BRANCHES[project]
+        partner_url_config['release-partner-repack'] = \
+            graph_config['partner'][alias]['release-partner-repack']
+    return partner_url_config