Support Fennec and get rid of artifacts checks draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Thu, 01 Sep 2016 17:20:51 +0200
changeset 7133 7ca6cb5bc2f7c3b52bd0dc4dc8a98e99f8415d65
parent 7132 f78c86129ca5f6f6203787cbab2f57309b896efc
child 7134 173f17231c9d80bf52a102d842f28fb64275b5da
push id117
push userjlorenzo@mozilla.com
push dateThu, 01 Sep 2016 15:21:04 +0000
Support Fennec and get rid of artifacts checks MozReview-Commit-ID: 2fJcHyBVYvH
buildfarm/release/build_status.py
buildfarm/release/release-runner.py
--- a/buildfarm/release/build_status.py
+++ b/buildfarm/release/build_status.py
@@ -1,65 +1,37 @@
 import taskcluster
 
 from kickoff import task_for_revision
 
 import logging
 log = logging.getLogger(__name__)
 
-PLATFORMS_TO_WAIT_ON = (
-    {'index_platform_name': 'linux', 'artifact_suffix': 'linux-i686.tar.bz2'},
-    {'index_platform_name': 'linux64', 'artifact_suffix': 'linux-x86_64.tar.bz2'},
-    {'index_platform_name': 'macosx64', 'artifact_suffix': 'mac.dmg'},
-    {'index_platform_name': 'win32', 'artifact_suffix': 'win32.zip'},
-    {'index_platform_name': 'win64', 'artifact_suffix': 'win64.zip'},
-)
+PLATFORMS_TO_WAIT_ON = {
+    'firefox': ['linux', 'linux64', 'macosx64', 'win32', 'win64'],
+    'mobile': ['android-x86', 'android-api-9'],
+}
 
 
-def are_en_us_builds_done(index, queue, release):
-    def map_tasks_ids_and_expected_artifacts(platform):
-        task = task_for_revision(index, branch=release['branch'], revision=release['mozillaRevision'],
-                                 product=release['product'], platform=platform['index_platform_name'])
-
-        expected_artifact = 'public/build/{}-{}.en-US.{}'.format(
-            release['product'],
-            release['version'],
-            platform['artifact_suffix']
-        )
-
-        return {'taskId': task['taskId'], 'expected_artifact': expected_artifact}
+def are_en_us_builds_complete(index, queue, release):
+    product = release['product']
+    platforms = PLATFORMS_TO_WAIT_ON[product]
+    log.debug('product %s, platforms %s', product, platforms)
 
     try:
-        tasks_to_watch = map(map_tasks_ids_and_expected_artifacts, PLATFORMS_TO_WAIT_ON)
+        tasks_to_watch = [
+            task_for_revision(
+                index, branch=release['branch'], revision=release['mozillaRevision'], product=product, platform=platform
+            )['taskId']
+            for platform in platforms
+        ]
     except taskcluster.exceptions.TaskclusterRestFailure:
         # Tasks not created yet
         return False
 
     log.debug('tasks found %s', tasks_to_watch)
-
-    if _are_all_tasks_completed(queue, tasks_to_watch) is False:
-        log.debug('some tasks not ready yet')
-        return False
-
-    return _are_all_artifacts_present(queue, tasks_to_watch)
-
-
-def _are_all_tasks_completed(queue, tasks):
-    tasks_not_done_yet = [task for task in tasks if queue.status(task['taskId'])['state'] != 'completed']
-    log.debug('tasks not done yet %s', tasks_not_done_yet)
-    return len(tasks_not_done_yet) == 0
+    return _are_all_tasks_completed(queue, tasks_to_watch)
 
 
-def _are_all_artifacts_present(queue, tasks):
-    for task in tasks:
-        artifacts = queue.listLatestArtifacts(task['taskId'])
-        log.debug('artifacts found for task %s: %s', task['taskId'], artifacts)
-        expected_artifacts = [artifact for artifact in artifacts if artifact['name'] == task['expected_artifact']]
-        log.debug('expected artifacts: %s', expected_artifacts)
-
-        number_of_artifacts = len(expected_artifacts)
-        if number_of_artifacts > 1:
-            raise Exception('Too many artifacts found for %s' % task['taskId'])
-        elif number_of_artifacts == 0:
-            return False
-
-    log.debug('all artifacts are here')
-    return True
+def _are_all_tasks_completed(queue, taskIds):
+    tasks_not_done_yet = [taskId for taskId in taskIds if queue.status(taskId)['state'] != 'completed']
+    log.debug('tasks not done yet %s', tasks_not_done_yet)
+    return len(tasks_not_done_yet) == 0
--- a/buildfarm/release/release-runner.py
+++ b/buildfarm/release/release-runner.py
@@ -26,17 +26,17 @@ from kickoff.sanity import ReleaseSaniti
 from release.info import readBranchConfig
 from release.l10n import parsePlainL10nChangesets
 from release.versions import getAppVersion
 from taskcluster import Scheduler, Index, Queue
 from taskcluster.utils import slugId
 from util.hg import mercurial
 from util.retry import retry
 from util.file import load_config, get_config
-from build_status import are_en_us_builds_done
+from build_status import are_en_us_builds_complete
 
 log = logging.getLogger(__name__)
 
 
 # both CHECKSUMS and ALL_FILES have been defined to improve the release sanity
 # en-US binaries timing by whitelisting artifacts of interest - bug 1251761
 CHECKSUMS = set([
     '.checksums',
@@ -332,21 +332,22 @@ def main(options):
         postrelease_enabled = branchConfig['postrelease_version_bump_enabled']
         postrelease_bouncer_aliases_enabled = branchConfig['postrelease_bouncer_aliases_enabled']
         final_verify_channels = release_channels
         publish_to_balrog_channels = release_channels
         push_to_releases_enabled = True
 
     rc = 0
     for release in rr.new_releases:
-        if are_en_us_builds_done(index, queue, release) is False:
-            log.info('Builds for release "%s" are not done yet. Skipping this one for now')
-            release['status'] = 'waiting for en_us_builds'
+        if not are_en_us_builds_complete(index, queue, release):
+            log.info('Builds for release "%s" are complete done yet. Skipping this one for now')
+            # TODO: Mark as "waiting" in ship-it
             continue
 
+        log.info('Every build for release "%s" is complete!')
         graph_id = slugId()
         try:
             rr.update_status(release, 'Generating task graph')
             l10n_changesets = parsePlainL10nChangesets(rr.get_release_l10n(release["name"]))
 
             kwargs = {
                 "public_key": docker_worker_key,
                 "version": release["version"],