bug 1457034 - re-add partner github code. r=nthomas
authorAki Sasaki <asasaki@mozilla.com>
Wed, 25 Apr 2018 13:38:51 -0700
changeset 463616 a66cd30297c4
parent 463615 3202d5534730
child 463617 878f15390bf7
push id1711
push userasasaki@mozilla.com
push date2018-05-09 16:09 +0000
treeherdermozilla-release@00b58dfcf206 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnthomas
bugs1457034
milestone60.0.1
bug 1457034 - re-add partner github code. r=nthomas We originally had this logic here, and called it from the `partner_repack` transform. This kept the config more hidden, but had the downsides of a) being difficult to test, and b) hitting the network during transforms, which we're trying to avoid. We moved this code to release-runner3, and passed in the `release_partner_config` as input, and saved it as a parameter. Parameterizing the partner config means that we can refer to it easily throughout taskgraph generation and in local testing, and we don't have to hit the network during transforms. The downsides include potentially having to generate this config in multiple places (rr3, ship-it-v2, the partner hook), and risking hitting the 20k gpg cleartext character limit in the `ACTION_INPUT` environment variable. Now I'm moving this code back into util.partners, but I'm calling it from the action, not from the transform. The action populates the `release_partner_config` parameter, so we can still access the config from anywhere in the taskgraph generation code, more easily test, and avoid hitting the network during transforms. It also means that release-runner3, ship-it-v2, and the partner hook can all use the partner config generation code from a single location, rather than having to duplicate it. Hoping this is the last major change we need to make here for a while. MozReview-Commit-ID: 8UmvmcAoZgD
taskcluster/taskgraph/util/partners.py
--- a/taskcluster/taskgraph/util/partners.py
+++ b/taskcluster/taskgraph/util/partners.py
@@ -1,19 +1,281 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 from copy import deepcopy
 import json
+import logging
 import os
+import requests
+import xml.etree.ElementTree as ET
+
+
+# Suppress chatty requests logging
+logging.getLogger("requests").setLevel(logging.WARNING)
+
+log = logging.getLogger(__name__)
+
+GITHUB_API_ENDPOINT = "https://api.github.com/graphql"
+
+"""
+LOGIN_QUERY, MANIFEST_QUERY, and REPACK_CFG_QUERY are all written to the Github v4 API,
+which users GraphQL. See https://developer.github.com/v4/
+"""
+
+LOGIN_QUERY = """query {
+  viewer {
+    login
+    name
+  }
+}
+"""
+
+# Returns the contents of default.xml from a manifest repository
+MANIFEST_QUERY = """query {
+  repository(owner:"%(owner)s", name:"%(repo)s") {
+    object(expression: "master:default.xml") {
+      ... on Blob {
+        text
+      }
+    }
+  }
+}
+"""
+
+r"""
+Example response:
+{
+  "data": {
+    "repository": {
+      "object": {
+        "text": "<?xml version=\"1.0\" ?>\n<manifest>\n  " +
+        "<remote fetch=\"git@github.com:mozilla-partners/\" name=\"mozilla-partners\"/>\n  " +
+        "<remote fetch=\"git@github.com:mozilla/\" name=\"mozilla\"/>\n\n  " +
+        "<project name=\"repack-scripts\" path=\"scripts\" remote=\"mozilla-partners\" " +
+        "revision=\"master\"/>\n  <project name=\"build-tools\" path=\"scripts/tools\" " +
+        "remote=\"mozilla\" revision=\"master\"/>\n  <project name=\"mozilla-EME-free\" " +
+        "path=\"partners/mozilla-EME-free\" remote=\"mozilla-partners\" " +
+        "revision=\"master\"/>\n</manifest>\n"
+      }
+    }
+  }
+}
+"""
+
+# Returns the contents of desktop/*/repack.cfg for a partner repository
+REPACK_CFG_QUERY = """query{
+  repository(owner:"%(owner)s", name:"%(repo)s") {
+    object(expression: "master:desktop/"){
+      ... on Tree {
+        entries {
+          name
+          object {
+            ... on Tree {
+              entries {
+                name
+                object {
+                  ... on Blob {
+                    text
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+"""
+r"""
+Example response:
+{
+  "data": {
+    "repository": {
+      "object": {
+        "entries": [
+          {
+            "name": "mozilla-EME-free",
+            "object": {
+              "entries": [
+                {
+                  "name": "distribution",
+                  "object": {}
+                },
+                {
+                  "name": "repack.cfg",
+                  "object": {
+                    "text": "aus=\"mozilla-EMEfree\"\ndist_id=\"mozilla-EMEfree\"\n" +
+                            "dist_version=\"1.0\"\nlinux-i686=true\nlinux-x86_64=true\n" +
+                            " locales=\"ach af de en-US\"\nmac=true\nwin32=true\nwin64=true\n" +
+                            "output_dir=\"%(platform)s-EME-free/%(locale)s\"\n\n" +
+                            "# Upload params\nbucket=\"net-mozaws-prod-delivery-firefox\"\n" +
+                            "upload_to_candidates=true\n"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  }
+}
+"""
+
+# Map platforms in repack.cfg into their equivalents in taskcluster
+TC_PLATFORM_PER_FTP = {
+    'linux-i686': 'linux-nightly',
+    'linux-x86_64': 'linux64-nightly',
+    'mac': 'macosx64-nightly',
+    'win32': 'win32-nightly',
+    'win64': 'win64-nightly',
+}
+
+TASKCLUSTER_PROXY_SECRET_ROOT = 'http://taskcluster/secrets/v1/secret/'
 
 LOCALES_FILE = os.path.join(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
     'browser', 'locales', 'l10n-changesets.json'
 )
 
+# cache data at the module level
+partner_configs = {}
+
+
+def query_api(query, token):
+    """ Make a query with a Github auth header, returning the json """
+    headers = {'Authorization': 'bearer %s' % token}
+    r = requests.post(GITHUB_API_ENDPOINT, json={'query': query}, headers=headers)
+    r.raise_for_status()
+
+    j = r.json()
+    if 'errors' in j:
+        raise RuntimeError("Github query error - %s", j['errors'])
+    return j
+
+
+def check_login(token):
+    log.debug("Checking we have a valid login")
+    query_api(LOGIN_QUERY, token)
+
+
+def get_repo_params(repo):
+    """ Parse the organisation and repo name from an https or git url for a repo """
+    if repo.startswith('https'):
+        # eg https://github.com/mozilla-partners/mozilla-EME-free
+        return repo.rsplit('/', 2)[-2:]
+    elif repo.startswith('git@'):
+        # eg git@github.com:mozilla-partners/mailru.git
+        repo = repo.replace('.git', '')
+        return repo.split(':')[-1].split('/')
+
+
+def get_partners(manifestRepo, token):
+    """ Given the url to a manifest repository, retieve the default.xml and parse it into a
+    list of parter repos.
+    """
+    log.debug("Querying for manifest in %s", manifestRepo)
+    owner, repo = get_repo_params(manifestRepo)
+    query = MANIFEST_QUERY % {'owner': owner, 'repo': repo}
+    raw_manifest = query_api(query, token)
+    log.debug("Raw manifest: %s", raw_manifest)
+    if not raw_manifest['data']['repository']:
+        raise RuntimeError(
+            "Couldn't load partner manifest at %s, insufficient permissions ?" %
+            manifestRepo
+        )
+    e = ET.fromstring(raw_manifest['data']['repository']['object']['text'])
+
+    remotes = {}
+    partners = {}
+    for child in e:
+        if child.tag == 'remote':
+            name = child.attrib['name']
+            url = child.attrib['fetch']
+            remotes[name] = url
+            log.debug('Added remote %s from %s', name, url)
+        elif child.tag == 'project':
+            # we don't need to check any code repos
+            if 'scripts' in child.attrib['path']:
+                continue
+            partner_url = "%s%s" % (remotes[child.attrib['remote']],
+                                    child.attrib['name'])
+            partners[child.attrib['name']] = partner_url
+            log.debug("Added partner %s" % partner_url)
+    return partners
+
+
+def parse_config(data):
+    """ Parse a single repack.cfg file into a python dictionary.
+    data is contents of the file, in "foo=bar\nbaz=buzz" style. We do some translation on
+    locales and platforms data, otherewise passthrough
+    """
+    ALLOWED_KEYS = ('locales', 'upload_to_candidates', 'platforms')
+    config = {'platforms': []}
+    for l in data.splitlines():
+        if '=' in l:
+            l = str(l)
+            key, value = l.split('=', 2)
+            value = value.strip('\'"').rstrip('\'"')
+            if key in ('linux-i686', 'linux-x86_64', 'mac', 'win32', 'win64'):
+                if value.lower() == 'true':
+                    config['platforms'].append(TC_PLATFORM_PER_FTP[key])
+                continue
+            if key not in ALLOWED_KEYS:
+                continue
+            if key == 'locales':
+                # a list please
+                value = value.split(" ")
+            config[key] = value
+    return config
+
+
+def get_repack_configs(repackRepo, token):
+    """ For a partner repository, retrieve all the repack.cfg files and parse them into a dict """
+    log.debug("Querying for configs in %s", repackRepo)
+    owner, repo = get_repo_params(repackRepo)
+    query = REPACK_CFG_QUERY % {'owner': owner, 'repo': repo}
+    raw_configs = query_api(query, token)
+    raw_configs = raw_configs['data']['repository']['object']['entries']
+
+    configs = {}
+    for sub_config in raw_configs:
+        name = sub_config['name']
+        for file in sub_config['object'].get('entries', []):
+            if file['name'] != 'repack.cfg':
+                continue
+            configs[name] = parse_config(file['object']['text'])
+    return configs
+
+
+def get_partner_config_by_url(manifest_url, kind, token, partner_subset=None):
+    """ Retrieve partner data starting from the manifest url, which points to a repository
+    containing a default.xml that is intended to be drive the Google tool 'repo'. It
+    descends into each partner repo to lookup and parse the repack.cfg file(s).
+
+    If partner_subset is a list of sub_config names only return data for those.
+
+    Supports caching data by kind to avoid repeated requests, relying on the related kinds for
+    partner repacking, signing, repackage, repackage signing all having the same kind prefix.
+    """
+    if kind not in partner_configs:
+        log.info('Looking up data for %s from %s', kind, manifest_url)
+        check_login(token)
+        partners = get_partners(manifest_url, token)
+
+        partner_configs[kind] = {}
+        for partner, partner_url in partners.items():
+            if partner_subset and partner not in partner_subset:
+                continue
+            partner_configs[kind][partner] = get_repack_configs(
+                partner_url, token, partner_subset
+            )
+    return partner_configs[kind]
+
 
 def check_if_partners_enabled(config, tasks):
     if (
         config.params['release_enable_partners'] and
         config.kind.startswith('release-partner-repack')
     ) or (
         config.params['release_enable_emefree'] and
         config.kind.startswith('release-eme-free-repack')