Bug 1630809 - read attribution configuration from github r=aki
authorNick Thomas <nthomas@mozilla.com>
Thu, 27 Aug 2020 22:19:39 +0000
changeset 546701 6eae524a8eec354c282b3341aa7ae83d6a37e972
parent 546700 2683d8570ef86d494c80cfd8932aa453c3525c3b
child 546702 e90a86c44d52b7470b8fa8df6022e1ad90bc907d
push id37736
push userapavel@mozilla.com
push dateFri, 28 Aug 2020 15:31:26 +0000
treeherdermozilla-central@56166cae2e26 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1630809
milestone82.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1630809 - read attribution configuration from github r=aki The partner attribution config is stored in the same repository as the repo manifest for partner repacks, but all in attribution_config.yml instead of default.xml. This extends the existing support for using the Github API to read files to retrieve and process the attribution config. Differential Revision: https://phabricator.services.mozilla.com/D87728
taskcluster/ci/config.yml
taskcluster/taskgraph/config.py
taskcluster/taskgraph/util/partners.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -408,16 +408,26 @@ partner-urls:
                     beta|release.*:
                         by-release-level:
                             production: 'git@github.com:mozilla-partners/repack-manifests.git'
                             staging: 'git@github.com:moz-releng-automation-stage/repack-manifests.git'
                     esr.*:
                         by-release-level:
                             production: 'git@github.com:mozilla-partners/esr-repack-manifests.git'
                             staging: 'git@github.com:moz-releng-automation-stage/esr-repack-manifests.git'
+    release-partner-attribution:
+        by-release-product:
+            default: null
+            firefox:
+                by-release-type:
+                    default: null
+                    beta|release.*:
+                        by-release-level:
+                            production: 'git@github.com:mozilla-partners/repack-manifests.git'
+                            staging: 'git@github.com:moz-releng-automation-stage/repack-manifests.git'
     release-eme-free-repack:
         by-release-product:
             default: null
             firefox:
                 by-release-type:
                     default: null
                     beta|release.*:
                         by-release-level:
--- a/taskcluster/taskgraph/config.py
+++ b/taskcluster/taskgraph/config.py
@@ -87,16 +87,19 @@ graph_config_schema = Schema({
         'low',
         'very-low',
         'lowest',
     )),
     Required('partner-urls'): {
         Required('release-partner-repack'):
             optionally_keyed_by('release-product', 'release-level', 'release-type',
                                 Any(text_type, None)),
+        Optional('release-partner-attribution'):
+            optionally_keyed_by('release-product', 'release-level', 'release-type',
+                                Any(text_type, None)),
         Required('release-eme-free-repack'):
             optionally_keyed_by('release-product', 'release-level', 'release-type',
                                 Any(text_type, None)),
     },
     Required('workers'): {
         Required('aliases'): {
             text_type: {
                 Required('provisioner'): optionally_keyed_by('level', text_type),
--- a/taskcluster/taskgraph/util/partners.py
+++ b/taskcluster/taskgraph/util/partners.py
@@ -10,16 +10,17 @@ import logging
 import os
 from redo import retry
 import requests
 import xml.etree.ElementTree as ET
 
 from taskgraph.util.attributes import release_level
 from taskgraph.util.schema import resolve_keyed_by
 import six
+import yaml
 
 # Suppress chatty requests logging
 logging.getLogger("requests").setLevel(logging.WARNING)
 
 log = logging.getLogger(__name__)
 
 GITHUB_API_ENDPOINT = "https://api.github.com/graphql"
 
@@ -34,17 +35,17 @@ LOGIN_QUERY = """query {
     name
   }
 }
 """
 
 # Returns the contents of default.xml from a manifest repository
 MANIFEST_QUERY = """query {
   repository(owner:"%(owner)s", name:"%(repo)s") {
-    object(expression: "master:default.xml") {
+    object(expression: "master:%(file)s") {
       ... on Blob {
         text
       }
     }
   }
 }
 """
 # Example response:
@@ -193,19 +194,19 @@ def get_repo_params(repo):
         repo = repo.replace('.git', '')
         return repo.split(':')[-1].split('/')
 
 
 def get_partners(manifestRepo, token):
     """ Given the url to a manifest repository, retrieve the default.xml and parse it into a
     list of partner repos.
     """
-    log.debug("Querying for manifest in %s", manifestRepo)
+    log.debug("Querying for manifest default.xml in %s", manifestRepo)
     owner, repo = get_repo_params(manifestRepo)
-    query = MANIFEST_QUERY % {'owner': owner, 'repo': repo}
+    query = MANIFEST_QUERY % {'owner': owner, 'repo': repo, 'file': 'default.xml'}
     raw_manifest = query_api(query, token)
     log.debug("Raw manifest: %s", raw_manifest)
     if not raw_manifest['data']['repository']:
         raise RuntimeError(
             "Couldn't load partner manifest at %s, insufficient permissions ?" %
             manifestRepo
         )
     e = ET.fromstring(raw_manifest['data']['repository']['object']['text'])
@@ -272,38 +273,60 @@ def get_repack_configs(repackRepo, token
         name = sub_config['name']
         for file in sub_config['object'].get('entries', []):
             if file['name'] != 'repack.cfg':
                 continue
             configs[name] = parse_config(file['object']['text'])
     return configs
 
 
+def get_attribution_config(manifestRepo, token):
+    log.debug("Querying for manifest attribution_config.yml in %s", manifestRepo)
+    owner, repo = get_repo_params(manifestRepo)
+    query = MANIFEST_QUERY % {'owner': owner, 'repo': repo, 'file': 'attribution_config.yml'}
+    raw_manifest = query_api(query, token)
+    if not raw_manifest['data']['repository']:
+        raise RuntimeError(
+            "Couldn't load partner manifest at %s, insufficient permissions ?" %
+            manifestRepo
+        )
+    # no file has been set up, gracefully continue
+    if raw_manifest['data']['repository']['object'] is None:
+        log.debug('No attribution_config.yml file found')
+        return {}
+
+    return yaml.safe_load(raw_manifest['data']['repository']['object']['text'])
+
+
 def get_partner_config_by_url(manifest_url, kind, token, partner_subset=None):
     """ Retrieve partner data starting from the manifest url, which points to a repository
     containing a default.xml that is intended to be drive the Google tool 'repo'. It
     descends into each partner repo to lookup and parse the repack.cfg file(s).
 
     If partner_subset is a list of sub_config names only return data for those.
 
     Supports caching data by kind to avoid repeated requests, relying on the related kinds for
     partner repacking, signing, repackage, repackage signing all having the same kind prefix.
     """
     if not manifest_url:
         raise RuntimeError('Manifest url for {} not defined'.format(kind))
     if kind not in partner_configs:
         log.info('Looking up data for %s from %s', kind, manifest_url)
         check_login(token)
-        partners = get_partners(manifest_url, token)
+        if kind == 'release-partner-attribution':
+            partner_configs[kind] = get_attribution_config(manifest_url, token)
+        else:
+            partners = get_partners(manifest_url, token)
 
-        partner_configs[kind] = {}
-        for partner, partner_url in partners.items():
-            if partner_subset and partner not in partner_subset:
-                continue
-            partner_configs[kind][partner] = get_repack_configs(partner_url, token)
+            partner_configs[kind] = {}
+            for partner, partner_url in partners.items():
+                if partner_subset and partner not in partner_subset:
+                    continue
+                partner_configs[kind][partner] = get_repack_configs(partner_url, token)
+
     return partner_configs[kind]
 
 
 def check_if_partners_enabled(config, tasks):
     if (
         config.params['release_enable_partner_repack'] and
         config.kind.startswith('release-partner-repack')
     ) or (
@@ -356,24 +379,33 @@ def _fix_subpartner_locales(orig_config,
 def fix_partner_config(orig_config):
     pc = {}
     with open(LOCALES_FILE, 'r') as fh:
         all_locales = list(json.load(fh).keys())
     # l10n-changesets.json doesn't include en-US, but the repack list does
     if 'en-US' not in all_locales:
         all_locales.append('en-US')
     for kind, kind_config in six.iteritems(orig_config):
-        for partner, partner_config in six.iteritems(kind_config):
-            for subpartner, subpartner_config in six.iteritems(partner_config):
-                # get rid of empty subpartner configs
-                if not subpartner_config:
-                    continue
-                # Make sure our locale list is a subset of all_locales
-                pc.setdefault(kind, {}).setdefault(partner, {})[subpartner] = \
-                    _fix_subpartner_locales(subpartner_config, all_locales)
+        if kind == 'release-partner-attribution':
+            pc[kind] = {}
+            if kind_config:
+                pc[kind] = {"defaults": kind_config["defaults"]}
+                for config in kind_config["configs"]:
+                    # Make sure our locale list is a subset of all_locales
+                    pc[kind].setdefault("configs", []).append(
+                        _fix_subpartner_locales(config, all_locales))
+        else:
+            for partner, partner_config in six.iteritems(kind_config):
+                for subpartner, subpartner_config in six.iteritems(partner_config):
+                    # get rid of empty subpartner configs
+                    if not subpartner_config:
+                        continue
+                    # Make sure our locale list is a subset of all_locales
+                    pc.setdefault(kind, {}).setdefault(partner, {})[subpartner] = \
+                        _fix_subpartner_locales(subpartner_config, all_locales)
     return pc
 
 
 # seems likely this exists elsewhere already
 def get_ftp_platform(platform):
     if platform.startswith('win32'):
         return 'win32'
     elif platform.startswith('win64-aarch64'):
@@ -405,16 +437,18 @@ def get_partner_url_config(parameters, g
         'release-product': parameters['release_product'],
         'release-level': release_level(parameters['project']),
         'release-type': parameters["release_type"]
     }
     resolve_keyed_by(partner_url_config, 'release-eme-free-repack', 'eme-free manifest_url',
                      **substitutions)
     resolve_keyed_by(partner_url_config, 'release-partner-repack', 'partner manifest url',
                      **substitutions)
+    resolve_keyed_by(partner_url_config, 'release-partner-attribution', 'partner attribution url',
+                     **substitutions)
     return partner_url_config
 
 
 def get_repack_ids_by_platform(config, build_platform):
     partner_config = get_partner_config_by_kind(config, config.kind)
     combinations = []
     for partner, subconfigs in partner_config.items():
         for sub_config_name, sub_config in subconfigs.items():