bug 1451481 - add partner config support to rr3. r=rail
authorAki Sasaki <asasaki@mozilla.com>
Thu, 05 Apr 2018 18:31:18 -0700
changeset 8369 8a19f66d18bc
parent 8368 88c77a1fb26a
child 8370 06a71a5fe369
push id6108
push userasasaki@mozilla.com
push dateMon, 09 Apr 2018 17:06:01 +0000
reviewersrail
bugs1451481
bug 1451481 - add partner config support to rr3. r=rail
buildfarm/release/release-runner.yml.example
buildfarm/release/release-runner3.py
lib/python/kickoff/partners.py
--- a/buildfarm/release/release-runner.yml.example
+++ b/buildfarm/release/release-runner.yml.example
@@ -21,16 +21,24 @@ release-runner:
   sendchange_master: localhost:9001
   staging: false
   verbose: true
   sleeptime: 30
   ssh_username: cltbld
   ssh_key: /home/cltbld/.ssh/release-runner
   docker_worker_key: /home/cltbld/docker-worker-public.pem
 
+git:
+  github_token: asdfasdfas
+
+partners:
+  partner_urls:
+    release-eme-free-repack: https://url
+  partner_min_version: 61
+
 symlinks:
   production_config.py: "{buildbot_configs}/mozilla/localconfig.py"
   thunderbird_production_config.py: "{buildbot_configs}/mozilla/thunderbird_localconfig.py"
 
 releases:
   - product: firefox
     pattern: Firefox-.*
     checks:
--- a/buildfarm/release/release-runner3.py
+++ b/buildfarm/release/release-runner3.py
@@ -1,14 +1,15 @@
 #!/usr/bin/env python
 """
 Release runner to schedule action tasks (in-tree scheduling)
 """
 
 import argparse
+from copy import deepcopy
 import logging
 import os
 import re
 import site
 import sys
 import taskcluster
 import time
 import yaml
@@ -20,16 +21,17 @@ site.addsitedir(path.join(path.dirname(_
 
 from release.info import isFinalRelease
 
 from kickoff import (ReleaseRunner, long_revision, email_release_drivers,
                      bump_version, get_partials)
 from kickoff.sanity.partials import PartialsSanitizer
 from kickoff.sanity.revisions import RevisionsSanitizer
 from kickoff.actions import generate_action_task, submit_action_task, find_decision_task_id
+from kickoff.partners import get_partner_config_by_url
 
 
 log = logging.getLogger(__name__)
 
 
 def check_and_assign_long_revision(release_runner, release, releases_config):
     # Revisions must be checked before trying to get the long one.
     RevisionsSanitizer(**release).run()
@@ -115,16 +117,27 @@ def is_rc(release):
 
 
 def get_beta_num(version):
     if is_beta(version):
         parts = version.split('b')
         return int(parts[-1])
 
 
+def is_partner_enabled(release, min_version):
+    major_version = release['version'].split('.')[0]
+    if release["product"] == "firefox" and major_version >= min_version:
+        if is_beta(release['version']):
+            if get_beta_num(release['version']) >= 8:
+                return True
+        elif not is_esr(release['version']):
+            return True
+    return False
+
+
 def main(options):
     log.info('Loading config from %s' % options.config)
 
     with open(options.config, 'r') as config_file:
         config = yaml.load(config_file)
 
     if config['release-runner'].get('verbose', False):
         log_level = logging.DEBUG
@@ -134,16 +147,18 @@ def main(options):
                         level=log_level)
     # Suppress logging of retry(), see bug 925321 for the details
     logging.getLogger("util.retry").setLevel(logging.WARN)
 
     api_root = config['api']['api_root']
     username = config['api']['username']
     password = config['api']['password']
 
+    github_token = config['git']['github_token']
+    partner_config = deepcopy(config['partners'])
     rr_config = config['release-runner']
     sleeptime = rr_config['sleeptime']
     smtp_server = rr_config.get('smtp_server', 'localhost')
     notify_from = rr_config.get('notify_from')
     notify_to = rr_config.get('notify_to_announce')
     if isinstance(notify_to, basestring):
         notify_to = [x.strip() for x in notify_to.split(',')]
 
@@ -207,16 +222,22 @@ def main(options):
                     # XXX The current plan is to run promote_firefox_rc, then
                     # ship_firefox_rc, then push_firefox, then ship_firefox.
                     # We need to support this workflow. However, rr3 doesn't
                     # support anything more than the first action task yet,
                     # so this isn't a missing feature for RCs specifically.
                     action_task_input["release_promotion_flavor"] = "{}_rc".format(
                         action_task_input["release_promotion_flavor"]
                     )
+            if is_partner_enabled(release, partner_config['partner_min_version']):
+                action_task_input['release_partner_config'] = {}
+                for kind, url in partner_config['partner_urls'].items():
+                    action_task_input['release_partner_config'][kind] = get_partner_config_by_url(
+                        url, kind, github_token
+                    )
             action_task_id, action_task = generate_action_task(
                 project=release["branchShortName"],
                 revision=release["mozillaRevision"],
                 action_task_input=action_task_input,
             )
             submit_action_task(queue=queue, action_task_id=action_task_id,
                                action_task=action_task)
             rr.mark_as_completed(release)
new file mode 100644
--- /dev/null
+++ b/lib/python/kickoff/partners.py
@@ -0,0 +1,273 @@
+# TODO - redo errors to keep in style ??
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import os
+import xml.etree.ElementTree as ET
+
+import requests
+
+# Suppress chatty requests logging
+logging.getLogger("requests").setLevel(logging.WARNING)
+
+log = logging.getLogger(__name__)
+
+GITHUB_API_ENDPOINT = "https://api.github.com/graphql"
+
+"""
+LOGIN_QUERY, MANIFEST_QUERY, and REPACK_CFG_QUERY are all written to the Github v4 API,
+which users GraphQL. See https://developer.github.com/v4/
+"""
+
+LOGIN_QUERY = """query {
+  viewer {
+    login
+    name
+  }
+}
+"""
+
+# Returns the contents of default.xml from a manifest repository
+MANIFEST_QUERY = """query {
+  repository(owner:"%(owner)s", name:"%(repo)s") {
+    object(expression: "master:default.xml") {
+      ... on Blob {
+        text
+      }
+    }
+  }
+}
+"""
+
+r"""
+Example response:
+{
+  "data": {
+    "repository": {
+      "object": {
+        "text": "<?xml version=\"1.0\" ?>\n<manifest>\n  " +
+        "<remote fetch=\"git@github.com:mozilla-partners/\" name=\"mozilla-partners\"/>\n  " +
+        "<remote fetch=\"git@github.com:mozilla/\" name=\"mozilla\"/>\n\n  " +
+        "<project name=\"repack-scripts\" path=\"scripts\" remote=\"mozilla-partners\" " +
+        "revision=\"master\"/>\n  <project name=\"build-tools\" path=\"scripts/tools\" " +
+        "remote=\"mozilla\" revision=\"master\"/>\n  <project name=\"mozilla-EME-free\" " +
+        "path=\"partners/mozilla-EME-free\" remote=\"mozilla-partners\" " +
+        "revision=\"master\"/>\n</manifest>\n"
+      }
+    }
+  }
+}
+"""
+
+# Returns the contents of desktop/*/repack.cfg for a partner repository
+REPACK_CFG_QUERY = """query{
+  repository(owner:"%(owner)s", name:"%(repo)s") {
+    object(expression: "master:desktop/"){
+      ... on Tree {
+        entries {
+          name
+          object {
+            ... on Tree {
+              entries {
+                name
+                object {
+                  ... on Blob {
+                    text
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+"""
+r"""
+Example response:
+{
+  "data": {
+    "repository": {
+      "object": {
+        "entries": [
+          {
+            "name": "mozilla-EME-free",
+            "object": {
+              "entries": [
+                {
+                  "name": "distribution",
+                  "object": {}
+                },
+                {
+                  "name": "repack.cfg",
+                  "object": {
+                    "text": "aus=\"mozilla-EMEfree\"\ndist_id=\"mozilla-EMEfree\"\n" +
+                            "dist_version=\"1.0\"\nlinux-i686=true\nlinux-x86_64=true\n" +
+                            " locales=\"ach af de en-US\"\nmac=true\nwin32=true\nwin64=true\n" +
+                            "output_dir=\"%(platform)s-EME-free/%(locale)s\"\n\n" +
+                            "# Upload params\nbucket=\"net-mozaws-prod-delivery-firefox\"\n" +
+                            "upload_to_candidates=true\n"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  }
+}
+"""
+
+# Map platforms in repack.cfg into their equivalents in taskcluster
+TC_PLATFORM_PER_FTP = {
+    'linux-i686': 'linux-nightly',
+    'linux-x86_64': 'linux64-nightly',
+    'mac': 'macosx64-nightly',
+    'win32': 'win32-nightly',
+    'win64': 'win64-nightly',
+}
+
+TASKCLUSTER_PROXY_SECRET_ROOT = 'http://taskcluster/secrets/v1/secret/'
+
+# cache data at the module level
+partner_configs = {}
+
+
+def query_api(query, token):
+    """ Make a query with a Github auth header, returning the json """
+    headers = {'Authorization': 'bearer %s' % token}
+    r = requests.post(GITHUB_API_ENDPOINT, json={'query': query}, headers=headers)
+    r.raise_for_status()
+
+    j = r.json()
+    if 'errors' in j:
+        raise RuntimeError("Github query error - %s", j['errors'])
+    return j
+
+
+def check_login(token):
+    log.debug("Checking we have a valid login")
+    query_api(LOGIN_QUERY, token)
+
+
+def get_repo_params(repo):
+    """ Parse the organisation and repo name from an https or git url for a repo """
+    if repo.startswith('https'):
+        # eg https://github.com/mozilla-partners/mozilla-EME-free
+        return repo.rsplit('/', 2)[-2:]
+    elif repo.startswith('git@'):
+        # eg git@github.com:mozilla-partners/mailru.git
+        repo = repo.replace('.git', '')
+        return repo.split(':')[-1].split('/')
+
+
+def get_partners(manifestRepo, token):
+    """ Given the url to a manifest repository, retieve the default.xml and parse it into a
+    list of parter repos.
+    """
+    log.debug("Querying for manifest in %s", manifestRepo)
+    owner, repo = get_repo_params(manifestRepo)
+    query = MANIFEST_QUERY % {'owner': owner, 'repo': repo}
+    raw_manifest = query_api(query, token)
+    log.debug("Raw manifest: %s", raw_manifest)
+    if not raw_manifest['data']['repository']:
+        raise RuntimeError(
+            "Couldn't load partner manifest at %s, insufficient permissions ?" %
+            manifestRepo
+        )
+    e = ET.fromstring(raw_manifest['data']['repository']['object']['text'])
+
+    remotes = {}
+    partners = {}
+    for child in e:
+        if child.tag == 'remote':
+            name = child.attrib['name']
+            url = child.attrib['fetch']
+            remotes[name] = url
+            log.debug('Added remote %s from %s', name, url)
+        elif child.tag == 'project':
+            # we don't need to check any code repos
+            if 'scripts' in child.attrib['path']:
+                continue
+            partner_url = "%s%s" % (remotes[child.attrib['remote']],
+                                    child.attrib['name'])
+            partners[child.attrib['name']] = partner_url
+            log.debug("Added partner %s" % partner_url)
+    return partners
+
+
+def parse_config(data):
+    """ Parse a single repack.cfg file into a python dictionary.
+    data is contents of the file, in "foo=bar\nbaz=buzz" style. We do some translation on
+    locales and platforms data, otherewise passthrough
+    """
+    config = {'platforms': []}
+    for l in data.splitlines():
+        if '=' in l:
+            l = str(l)
+            key, value = l.split('=', 2)
+            value = value.strip('\'"').rstrip('\'"')
+            if key in ('linux-i686', 'linux-x86_64', 'mac', 'win32', 'win64'):
+                if value.lower() == 'true':
+                    config['platforms'].append(TC_PLATFORM_PER_FTP[key])
+                continue
+            if key == 'locales':
+                # a list please
+                value = value.split(" ")
+            config[key] = value
+    return config
+
+
+def get_repack_configs(repackRepo, token):
+    """ For a partner repository, retrieve all the repack.cfg files and parse them into a dict """
+    log.debug("Querying for configs in %s", repackRepo)
+    owner, repo = get_repo_params(repackRepo)
+    query = REPACK_CFG_QUERY % {'owner': owner, 'repo': repo}
+    raw_configs = query_api(query, token)
+    raw_configs = raw_configs['data']['repository']['object']['entries']
+
+    configs = {repo: {}}
+    for sub_config in raw_configs:
+        name = sub_config['name']
+        for file in sub_config['object'].get('entries', []):
+            if file['name'] != 'repack.cfg':
+                continue
+            configs[name] = parse_config(file['object']['text'])
+    return configs
+
+
+def get_partner_config_by_url(manifest_url, kind, token, partner_subset=None):
+    """ Retrieve partner data starting from the manifest url, which points to a repository
+    containing a default.xml that is intended to be drive the Google tool 'repo'. It
+    descends into each partner repo to lookup and parse the repack.cfg file(s).
+
+    If partner_subset is a list of sub_config names only return data for those.
+
+    Supports caching data by kind to avoid repeated requests, relying on the related kinds for
+    partner repacking, signing, repackage, repackage signing all having the same kind prefix.
+    """
+    if kind not in partner_configs:
+        log.info('Looking up data for %s from %s', kind, manifest_url)
+        check_login(token)
+        partners = get_partners(manifest_url, token)
+
+        partner_configs[kind] = {}
+        for partner, partner_url in partners.items():
+            partner_configs[kind][partner] = get_repack_configs(partner_url, token)
+
+        # if we're only interested in a subset of partners we remove the rest
+        if partner_subset:
+            new_config = {}
+            for partner in partner_subset:
+                if partner not in partner_configs[kind]:
+                    # TODO - should be fatal ?
+                    log.warning('Partner config for %s not found, skipping', partner)
+                    partner_subset.remove(partner)
+                else:
+                    new_config[partner] = partner_configs[kind][partner]
+            partner_configs[kind] = new_config
+
+    return partner_configs[kind]