Bug 1343953 - Add support for unsigned/signed tasks r=jlund
authorRail Aliiev <rail@mozilla.com>
Fri, 03 Mar 2017 11:24:55 -0500
changeset 7354 f39221db0f03836526605b29443186583afe701d
parent 7353 6c8a14c61f268afac414ef044cf9a01c9c90a9bc
child 7355 2bcf63ce095bccee48477a5b365a5a8fe7746e85
push id5475
push userraliiev@mozilla.com
push dateFri, 03 Mar 2017 16:26:08 +0000
reviewersjlund
bugs1343953
Bug 1343953 - Add support for unsigned/signed tasks r=jlund
buildfarm/release/release-runner.py
buildfarm/release/releasetasks_graph_gen.py
lib/python/kickoff/__init__.py
lib/python/kickoff/build_status.py
--- a/buildfarm/release/release-runner.py
+++ b/buildfarm/release/release-runner.py
@@ -10,36 +10,36 @@ import subprocess
 import hashlib
 import functools
 import shutil
 import tempfile
 import requests
 from os import path
 from optparse import OptionParser
 from twisted.python.lockfile import FilesystemLock
+from taskcluster import Scheduler, Index, Queue
+from taskcluster.utils import slugId
+import yaml
 
 site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
 
-from kickoff import get_partials, ReleaseRunner, make_task_graph_strict_kwargs, long_revision
-from kickoff import get_l10n_config, get_en_US_config
-from kickoff import email_release_drivers
-from kickoff import bump_version
+from kickoff import (get_partials, ReleaseRunner,
+                     make_task_graph_strict_kwargs, long_revision,
+                     get_l10n_config, get_en_US_config, email_release_drivers,
+                     bump_version)
 from kickoff.sanity.base import SanityException, is_candidate_release
 from kickoff.sanity.revisions import RevisionsSanitizer
 from kickoff.sanity.l10n import L10nSanitizer
 from kickoff.sanity.partials import PartialsSanitizer
 from kickoff.build_status import are_en_us_builds_completed
 from release.info import readBranchConfig
 from release.l10n import parsePlainL10nChangesets
 from release.versions import getAppVersion
-from taskcluster import Scheduler, Index, Queue
-from taskcluster.utils import slugId
 from util.hg import mercurial
 from util.retry import retry
-import yaml
 
 log = logging.getLogger(__name__)
 
 
 # both CHECKSUMS and ALL_FILES have been defined to improve the release sanity
 # en-US binaries timing by whitelisting artifacts of interest - bug 1251761
 CHECKSUMS = set([
     '.checksums',
@@ -48,18 +48,17 @@ CHECKSUMS = set([
 
 
 ALL_FILES = set([
     '.checksums',
     '.checksums.asc',
     '.complete.mar',
     '.exe',
     '.dmg',
-    'i686.tar.bz2',
-    'x86_64.tar.bz2',
+    'tar.bz2',
 ])
 
 CONFIGS_WORKDIR = 'buildbot-configs'
 
 
 def check_and_assign_long_revision(release_runner, release):
     # Revisions must be checked before trying to get the long one.
     RevisionsSanitizer(**release).run()
@@ -273,19 +272,23 @@ def get_hash(path, hash_type="sha512"):
         for chunk in iter(functools.partial(f.read, 4096), ''):
             h.update(chunk)
     return h.hexdigest()
 
 
 def validate_graph_kwargs(queue, gpg_key_path, **kwargs):
     # TODO: to be moved under kickoff soon, once new relpro sanity is in place
     # bug 1282959
-    platforms = kwargs.get('en_US_config', {}).get('platforms', {})
-    for platform in platforms.keys():
-        task_id = platforms.get(platform).get('task_id', {})
+    platforms = kwargs['en_US_config']['platforms']
+    for platform in platforms:
+        # FIXME: enable sanity check later for TC platforms
+        if platforms[platform]["signed_task_id"] != platforms[platform]["unsigned_task_id"]:
+            log.warning("Skipping en-US sanity for %s, TC platform", platform)
+            continue
+        task_id = platforms[platform]['signed_task_id']
         log.info('Performing release sanity for %s en-US binary', platform)
         sanitize_en_US_binary(queue, task_id, gpg_key_path)
 
     log.info("Release sanity for all en-US is now completed!")
 
 
 def main(options):
     log.info('Loading config from %s' % options.config)
@@ -399,35 +402,35 @@ def main(options):
             postrelease_bouncer_aliases_enabled = branchConfig[
                 'postrelease_bouncer_aliases_enabled']
             postrelease_mark_as_shipped_enabled = branchConfig[
                 'postrelease_mark_as_shipped_enabled']
             final_verify_channels = release_channels
             publish_to_balrog_channels = release_channels
             push_to_releases_enabled = True
 
-        ship_it_product_name = release['product']
-        tc_product_name = branchConfig['stage_product'][ship_it_product_name]
         # XXX: Doesn't work with neither Fennec nor Thunderbird
         platforms = branchConfig['release_platforms']
 
         try:
-            if not are_en_us_builds_completed(index, release_name=release['name'], submitted_at=release['submittedAt'],
-                                              branch=release['branchShortName'], revision=release[
-                                                  'mozillaRevision'],
-                                              tc_product_name=tc_product_name, platforms=platforms, queue=queue):
+            graph_id = slugId()
+            done = are_en_us_builds_completed(
+                index=index, release_name=release['name'],
+                submitted_at=release['submittedAt'],
+                revision=release['mozillaRevision'],
+                platforms=platforms, queue=queue,
+                tc_task_indexes=branchConfig['tc_indexes'][release['product']])
+            if not done:
                 log.info(
                     'Builds are not completed yet, skipping release "%s" for now', release['name'])
                 rr.update_status(release, 'Waiting for builds to be completed')
                 continue
 
             log.info('Every build is completed for release: %s',
                      release['name'])
-            graph_id = slugId()
-
             rr.update_status(release, 'Generating task graph')
 
             kwargs = {
                 "public_key": docker_worker_key,
                 "version": release["version"],
                 # ESR should not use "esr" suffix here:
                 "next_version": bump_version(release["version"].replace("esr", "")),
                 "appVersion": getAppVersion(release["version"]),
@@ -443,23 +446,25 @@ def main(options):
                 "branch": release['branchShortName'],
                 "updates_enabled": bool(release["partials"]),
                 "l10n_config": get_l10n_config(
                     index=index, product=release[
                         "product"], branch=release['branchShortName'],
                     revision=release['mozillaRevision'],
                     platforms=branchConfig['platforms'],
                     l10n_platforms=branchConfig['l10n_release_platforms'],
-                    l10n_changesets=release['l10n_changesets']
+                    l10n_changesets=release['l10n_changesets'],
+                    tc_task_indexes=branchConfig['tc_indexes'][release['product']],
                 ),
                 "en_US_config": get_en_US_config(
                     index=index, product=release[
                         "product"], branch=release['branchShortName'],
                     revision=release['mozillaRevision'],
-                    platforms=branchConfig['release_platforms']
+                    platforms=branchConfig['release_platforms'],
+                    tc_task_indexes=branchConfig['tc_indexes'][release['product']],
                 ),
                 "verifyConfigs": {},
                 "balrog_api_root": branchConfig["balrog_api_root"],
                 "funsize_balrog_api_root": branchConfig["funsize_balrog_api_root"],
                 "balrog_username": balrog_username,
                 "balrog_password": balrog_password,
                 "beetmover_aws_access_key_id": beetmover_aws_access_key_id,
                 "beetmover_aws_secret_access_key": beetmover_aws_secret_access_key,
@@ -495,17 +500,17 @@ def main(options):
             }
 
             validate_graph_kwargs(queue, gpg_key_path, **kwargs)
             graph = make_task_graph_strict_kwargs(**kwargs)
             rr.update_status(release, "Submitting task graph")
             log.info("Task graph generated!")
             import pprint
             log.debug(pprint.pformat(graph, indent=4, width=160))
-            print scheduler.createTaskGraph(graph_id, graph)
+            print(scheduler.createTaskGraph(graph_id, graph))
 
             rr.mark_as_completed(release)
             l10n_url = rr.release_l10n_api.getL10nFullUrl(release['name'])
             email_release_drivers(smtp_server=smtp_server, from_=notify_from,
                                   to=notify_to, release=release,
                                   task_group_id=graph_id, l10n_url=l10n_url)
         except Exception as exception:
             # We explicitly do not raise an error here because there's no
--- a/buildfarm/release/releasetasks_graph_gen.py
+++ b/buildfarm/release/releasetasks_graph_gen.py
@@ -88,23 +88,25 @@ def main(release_runner_config, release_
         # in release-runner.py world we have a concept of branchConfig and release (shipit) vars
         # todo fix get_en_US_config and en_US_config helper methods to not require both
         "l10n_config": get_l10n_config(
             index=index, product=release_config["product"],
             branch=release_config["branch"],
             revision=release_config["mozilla_revision"],
             platforms=release_config['platforms'],
             l10n_platforms=release_config['l10n_release_platforms'] or {},
-            l10n_changesets=release_config["l10n_changesets"]
+            l10n_changesets=release_config["l10n_changesets"],
+            tc_task_indexes=None,
         ),
         "en_US_config": get_en_US_config(
             index=index, product=release_config["product"],
             branch=release_config["branch"],
             revision=release_config["mozilla_revision"],
-            platforms=release_config['platforms']
+            platforms=release_config['platforms'],
+            tc_task_indexes=None,
         ),
         "extra_balrog_submitter_params": release_config['extra_balrog_submitter_params'],
         "publish_to_balrog_channels": release_config["publish_to_balrog_channels"],
         "postrelease_mark_as_shipped_enabled": release_config["postrelease_mark_as_shipped_enabled"],
         # TODO: use [] when snaps_enabled is landed
         "snap_enabled": release_config.get("snap_enabled", False),
         "update_verify_channel": release_config["update_verify_channel"],
         "update_verify_requires_cdn_push": release_config["update_verify_requires_cdn_push"],
--- a/lib/python/kickoff/__init__.py
+++ b/lib/python/kickoff/__init__.py
@@ -13,19 +13,20 @@ log = logging.getLogger(__name__)
 # temporary regex to filter out anything but mozilla-beta and mozilla-release
 # within release promotion. Once migration to release promotion is completed
 # for all types of releases, we will backout this filtering
 # regex beta tracking bug is 1252333,
 # regex release tracking bug is 1263976
 # This is now a default
 RELEASE_PATTERNS = [
     r"Firefox-.*",
-#     r"Fennec-.*",
+    # r"Fennec-.*",
 ]
 
+
 def matches(name, patterns):
     return any([re.search(p, name) for p in patterns])
 
 
 def long_revision(repo, revision):
     """Convert short revision to long using JSON API
 
     >>> long_revision("releases/mozilla-beta", "59f372c35b24")
@@ -75,21 +76,21 @@ class ReleaseRunner(object):
 
     def get_release_l10n(self, release):
         return self.release_l10n_api.getL10n(release)
 
     def update_status(self, release, status):
         log.info('updating status for %s to %s' % (release['name'], status))
         try:
             self.release_api.update(release['name'], status=status)
-        except requests.HTTPError, e:
+        except requests.HTTPError as e:
             log.warning('Caught HTTPError: %s' % e.response.content)
             log.warning('status update failed, continuing...', exc_info=True)
 
-    def mark_as_completed(self, release):#, enUSPlatforms):
+    def mark_as_completed(self, release):
         log.info('mark as completed %s' % release['name'])
         self.release_api.update(release['name'], complete=True,
                                 status='Started')
 
     def mark_as_failed(self, release, why):
         log.info('mark as failed %s' % release['name'])
         self.release_api.update(release['name'], ready=False, status=why)
 
@@ -151,48 +152,60 @@ def get_platform_locales(l10n_changesets
     if platform == "macosx64":
         ignore = "ja"
     else:
         ignore = "ja-JP-mac"
 
     return [l for l in l10n_changesets.keys() if l != ignore]
 
 
-def task_for_revision(index, branch, revision, product, platform):
-    return index.findTask(
-        "gecko.v2.{branch}.revision.{rev}.{product}.{platform}-opt".format(
-        rev=revision, branch=branch, product=product, platform=platform))
-
-
 def get_l10n_config(index, product, branch, revision, platforms,
-                    l10n_platforms, l10n_changesets):
+                    l10n_platforms, l10n_changesets, tc_task_indexes):
     l10n_platform_configs = {}
     for platform in l10n_platforms:
-        task = task_for_revision(index, branch, revision, product, platform)
-        url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
-            taskid=task["taskId"]
+        route = tc_task_indexes[platform]['signed'].format(rev=revision)
+        signed_task = index.findTask(route)
+        en_us_binary_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
+            taskid=signed_task["taskId"]
         )
+        if tc_task_indexes[platform]['unsigned'] != tc_task_indexes[platform]['signed']:
+            # TC based builds use different tasks for en-US (signed) and
+            # martools (unsigned)
+            unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision)
+            unsigned_task = index.findTask(unsigned_route)
+            mar_tools_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build/host/bin".format(
+                taskid=unsigned_task["taskId"]
+            )
+        else:
+            mar_tools_url = en_us_binary_url
+
         l10n_platform_configs[platform] = {
             "locales": get_platform_locales(l10n_changesets, platform),
-            "en_us_binary_url": url,
+            "en_us_binary_url": en_us_binary_url,
+            "mar_tools_url": mar_tools_url,
             "chunks": platforms[platform].get("l10n_chunks", 10),
         }
 
     return {
         "platforms": l10n_platform_configs,
         "changesets": l10n_changesets,
     }
 
 
-def get_en_US_config(index, product, branch, revision, platforms):
+def get_en_US_config(index, product, branch, revision, platforms,
+                     tc_task_indexes):
     platform_configs = {}
     for platform in platforms:
-        task = task_for_revision(index, branch, revision, product, platform)
+        signed_route = tc_task_indexes[platform]['signed'].format(rev=revision)
+        signed_task = index.findTask(signed_route)
+        unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision)
+        unsigned_task = index.findTask(unsigned_route)
         platform_configs[platform] = {
-            "task_id": task["taskId"]
+            "signed_task_id": signed_task["taskId"],
+            "unsigned_task_id": unsigned_task["taskId"],
         }
 
     return {
         "platforms": platform_configs,
     }
 
 
 # FIXME: the following function should be removed and we should use
--- a/lib/python/kickoff/build_status.py
+++ b/lib/python/kickoff/build_status.py
@@ -1,99 +1,111 @@
 from datetime import datetime, timedelta
 from dateutil import parser, tz
 from taskcluster.exceptions import TaskclusterRestFailure
 
-from kickoff import task_for_revision
-
 import logging
 log = logging.getLogger(__name__)
 
 
 _BUILD_WATCHERS = {}
 
 
 # TODO: Bug 1300147. Avoid having 7 parameters by using a release object that
 # contains only what's needed.
-def are_en_us_builds_completed(index, release_name, submitted_at, branch,
-                               revision, tc_product_name, platforms, queue):
+def are_en_us_builds_completed(index, release_name, submitted_at, revision,
+                               platforms, queue, tc_task_indexes):
     try:
         watcher = _BUILD_WATCHERS[release_name]
     except KeyError:
-        watcher = EnUsBuildsWatcher(index, release_name, submitted_at, branch,
-                                    revision, tc_product_name, platforms, queue)
+        watcher = EnUsBuildsWatcher(index, release_name, submitted_at,
+                                    revision, platforms, queue,
+                                    tc_task_indexes)
         _BUILD_WATCHERS[release_name] = watcher
         log.debug('New watcher created for "%s"', release_name)
 
     result = watcher.are_builds_completed()
 
     if result is True:
         del _BUILD_WATCHERS[release_name]
-        log.debug('Builds for "%s" are completed. Watcher deleted', release_name)
+        log.debug('Builds for "%s" are completed. Watcher deleted',
+                  release_name)
 
     return result
 
 
 class LoggedError(Exception):
     def __init__(self, message):
         log.exception(message)
         Exception.__init__(self, message)
 
 
 class EnUsBuildsWatcher:
     # TODO: Bug 1300147 as well
-    def __init__(self, index, release_name, submitted_at, branch, revision,
-                 tc_product_name, platforms, queue):
+    def __init__(self, index, release_name, submitted_at, revision,
+                 platforms, queue, tc_task_indexes):
         self.taskcluster_index = index
-        self.taskcluster_product_name = tc_product_name
 
         self.release_name = release_name
-        self.branch = branch
         self.revision = revision
         self.task_per_platform = {p: None for p in platforms}
         self.queue = queue
+        self.tc_task_indexes = tc_task_indexes
 
         self._timeout_watcher = TimeoutWatcher(start_timestamp=submitted_at)
 
     def are_builds_completed(self):
         if self._timeout_watcher.timed_out:
-            raise TimeoutWatcher.TimeoutError(self.release_name, self._timeout_watcher.start_timestamp)
+            raise TimeoutWatcher.TimeoutError(
+                self.release_name, self._timeout_watcher.start_timestamp)
 
         self._fetch_completed_tasks()
 
         return len(self._platforms_with_no_task) == 0
 
     def _fetch_completed_tasks(self):
         platforms_with_no_task = self._platforms_with_no_task
-        log.debug('Release "%s" still has to find tasks for %s', self.release_name, platforms_with_no_task)
+        log.debug('Release "%s" still has to find tasks for %s',
+                  self.release_name, platforms_with_no_task)
 
         for platform in platforms_with_no_task:
             try:
-                # Tasks are always completed if they are referenced in the index
-                # https://docs.taskcluster.net/reference/core/index
-                task_id = task_for_revision(
-                    self.taskcluster_index, self.branch, self.revision, self.taskcluster_product_name, platform
-                )['taskId']
-                # Bug 1307326 - consider only tasks indexed with rank > 0
+                # Tasks are always completed if they are referenced in the
+                # index https://docs.taskcluster.net/reference/core/index
+                # Assuming that the signed tasks are completed after their
+                # unsigned counterparts
+                route = self.tc_task_indexes[platform]['signed'].format(
+                    rev=self.revision)
+                task_id = self.taskcluster_index.findTask(route)['taskId']
+                # Bug 1307326 - consider only tasks indexed with rank > 0.
+                # If `rank` is unknown use tier-1 tasks.
                 task = self.queue.task(task_id)
-                rank = task["extra"]["index"]["rank"]
-                if rank == 0:
-                    log.debug("Ignoring task %s because the rank is set to 0",
-                              task_id)
+                rank = task["extra"].get("index", {}).get("rank")
+                tier = task["extra"].get("treeherder", {}).get("tier")
+                if rank is None:
+                    eligible = tier == 1
+                else:
+                    eligible = rank != 0
+                if  not eligible:
+                    log.debug("Ignoring task %s because rank (%s) or tier (%s)",
+                              task_id, rank, tier)
                     continue
             except TaskclusterRestFailure:
-                log.debug('Task for platform "%s" is not yet created for release "%s"', platform, self.release_name)
+                log.debug('Task for platform "%s" is not yet created for '
+                          'release "%s"', platform, self.release_name)
                 continue
 
-            log.debug('Task "%s" was found for release "%s" and platform "%s"', task_id, self.release_name, platform)
+            log.debug('Task "%s" was found for release "%s" and platform "%s"',
+                      task_id, self.release_name, platform)
             self.task_per_platform[platform] = task_id
 
     @property
     def _platforms_with_no_task(self):
-        return [platform for platform, task in self.task_per_platform.iteritems() if task is None]
+        return [platform for platform, task in
+                self.task_per_platform.iteritems() if task is None]
 
 
 class TimeoutWatcher:
     MAX_WAITING_TIME = timedelta(days=1)
 
     def __init__(self, start_timestamp):
         self.start_timestamp = parser.parse(start_timestamp)
 
@@ -105,15 +117,17 @@ class TimeoutWatcher:
     @property
     def timed_out(self):
         return self._now() - self.start_timestamp >= self.MAX_WAITING_TIME
 
     class TimeoutError(LoggedError):
         def __init__(self, release_name, start_timestamp):
             LoggedError.__init__(
                 self,
-                '{} has spent more than {} between the release being submitted on ship-it (at {} [UTC]) and now.'
-                .format(release_name, TimeoutWatcher.MAX_WAITING_TIME, start_timestamp)
+                '{} has spent more than {} between the release being submitted'
+                ' on ship-it (at {} [UTC]) and now.'.format(
+                    release_name, TimeoutWatcher.MAX_WAITING_TIME,
+                    start_timestamp)
             )
 
     class AlreadyStartedError(Exception):
         # Common error, there's no need to log it.
         pass