Bug 1343393 - add graph logic to handle a fennec beta build. r=mtabara DONTBUILD
authorAki Sasaki <asasaki@mozilla.com>
Wed, 01 Mar 2017 14:54:15 +0000
changeset 345314 232a19e40a52117301918832dbd35a00b2e440ae
parent 345313 7b406378bfe5fb840c633fe63262e30c8cfc60e3
child 345315 82001a435bce84794ea9c928d5643508f2044b47
push id31436
push userkwierso@gmail.com
push dateThu, 02 Mar 2017 01:18:52 +0000
treeherdermozilla-central@e91de6fb2b3d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1343393
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1343393 - add graph logic to handle a fennec beta build. r=mtabara DONTBUILD MozReview-Commit-ID: 4c6H5hiOiKj
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/scriptworker.py
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -218,16 +218,28 @@ def target_tasks_mozilla_beta(full_task_
 @_target_task('mozilla_release_tasks')
 def target_tasks_mozilla_release(full_task_graph, parameters):
     """Select the set of tasks required for a promotable beta or release build
     of linux, plus android CI. The candidates build process involves a pipeline
     of builds and signing, but does not include beetmover or balrog jobs."""
     return target_tasks_mozilla_beta(full_task_graph, parameters)
 
 
+@_target_task('candidates_fennec')
+def target_tasks_candidates_fennec(full_task_graph, parameters):
+    """Select the set of tasks required for a candidates build of fennec. The
+    nightly build process involves a pipeline of builds, signing,
+    and, eventually, uploading the tasks to balrog."""
+    filtered_for_project = target_tasks_nightly(full_task_graph, parameters)
+    def filter(task):
+        if task.kind not in ['balrog']:
+            return task.attributes.get('nightly', False)
+    return [l for l in filtered_for_project if filter(full_task_graph[l])]
+
+
 @_target_task('pine_tasks')
 def target_tasks_pine(full_task_graph, parameters):
     """Bug 1339179 - no mobile automation needed on pine"""
     def filter(task):
         platform = task.attributes.get('build_platform')
         # disable mobile jobs
         if str(platform).startswith('android'):
             return False
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -11,16 +11,17 @@ complexities of worker implementations, 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import time
 
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema
+from taskgraph.util.scriptworker import get_release_build_number
 from voluptuous import Schema, Any, Required, Optional, Extra
 
 from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
 
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
     basestring,
@@ -564,24 +565,27 @@ def build_scriptworker_signing_payload(c
         'maxRunTime': worker['max-run-time'],
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
 
 
 @payload_builder('beetmover')
 def build_beetmover_payload(config, task, task_def):
     worker = task['worker']
+    build_number = get_release_build_number(config)
 
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
         'upload_date': config.params['build_date'],
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
     if worker.get('locale'):
         task_def['payload']['locale'] = worker['locale']
+    if build_number:
+        task_def['payload']['build_number'] = build_number
 
 
 @payload_builder('balrog')
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'upstreamArtifacts':  worker['upstream-artifacts']
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -10,16 +10,17 @@ these scopes automatically by project; t
 project branch, and merge day uplifts more user friendly.
 
 In the future, we may adjust scopes by other settings as well, e.g. different
 scopes for `push-to-candidates` rather than `push-to-releases`, even if both
 happen on mozilla-beta and mozilla-release.
 """
 from __future__ import absolute_import, print_function, unicode_literals
 import functools
+import os
 
 
 """Map signing scope aliases to sets of projects.
 
 Currently m-c and m-a use nightly signing; m-b and m-r use release signing.
 We will need to add esr support at some point. Eventually we want to add
 nuance so certain m-b and m-r tasks use dep or nightly signing, and we only
 release sign when we have a signed-off set of candidate builds.  This current
@@ -48,46 +49,58 @@ SIGNING_CERT_SCOPES = {
 }
 
 """Map beetmover scope aliases to sets of projects.
 """
 BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [[
     'all-nightly-branches', set([
         'mozilla-central',
         'mozilla-aurora',
-        'jamun',
         'mozilla-beta',
         'mozilla-release',
     ])
 ], [
     'all-release-branches', set([
+        'mozilla-beta',
+        'mozilla-release',
     ])
 ]]
 
+"""The set of all beetmover release target tasks.
+
+Used for both `BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK` and `get_release_build_number`
+"""
+BEETMOVER_RELEASE_TARGET_TASKS = set([
+    'candidates_fennec',
+])
+
 """Map beetmover tasks aliases to sets of target task methods.
 
 This is a list of list-pairs, for ordering.
 """
 BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK = [[
     'all-nightly-tasks', set([
         'nightly_fennec',
         'nightly_linux',
         'mozilla_beta_tasks',
         'mozilla_release_tasks',
     ])
 ], [
-    'all-release-tasks', set([
-    ])
+    'all-release-tasks', BEETMOVER_RELEASE_TARGET_TASKS
 ]]
 
 """Map the beetmover scope aliases to the actual scopes.
 """
 BEETMOVER_BUCKET_SCOPES = {
-    'all-release-branches': 'project:releng:beetmover:bucket:release',
-    'all-nightly-branches': 'project:releng:beetmover:bucket:nightly',
+    'all-release-tasks': {
+        'all-release-branches': 'project:releng:beetmover:bucket:release',
+    },
+    'all-nightly-tasks': {
+        'all-nightly-branches': 'project:releng:beetmover:bucket:nightly',
+    },
     'default': 'project:releng:beetmover:bucket:dep',
 }
 
 """Map the beetmover tasks aliases to the actual action scopes.
 """
 BEETMOVER_ACTION_SCOPES = {
     'all-release-tasks': 'project:releng:beetmover:action:push-to-candidates',
     'all-nightly-tasks': 'project:releng:beetmover:action:push-to-nightly',
@@ -122,22 +135,23 @@ BALROG_SERVER_SCOPES = {
     'nightly': 'project:releng:balrog:nightly',
     'aurora': 'project:releng:balrog:nightly',
     'beta': 'project:releng:balrog:nightly',
     'release': 'project:releng:balrog:nightly',
     'default': 'project:releng:balrog:nightly',
 }
 
 
+# scope functions {{{1
 def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['project']`.
 
     Args:
         alias_to_project_map (list of lists): each list pair contains the
-            alias alias and the set of projects that match.  This is ordered.
+            alias and the set of projects that match.  This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
         config (dict): the task config that defines the project.
 
     Returns:
         string: the scope to use.
     """
     for alias, projects in alias_to_project_map:
         if config.params['project'] in projects and alias in alias_to_scope_map:
@@ -145,44 +159,98 @@ def get_scope_from_project(alias_to_proj
     return alias_to_scope_map['default']
 
 
 def get_scope_from_target_method(alias_to_tasks_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['target_tasks_method']`.
 
     Args:
         alias_to_tasks_map (list of lists): each list pair contains the
-            alias alias and the set of target methods that match. This is ordered.
+            alias and the set of target methods that match. This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
         config (dict): the task config that defines the target task method.
 
     Returns:
         string: the scope to use.
     """
     for alias, tasks in alias_to_tasks_map:
         if config.params['target_tasks_method'] in tasks and alias in alias_to_scope_map:
             return alias_to_scope_map[alias]
     return alias_to_scope_map['default']
 
 
+def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_project_map,
+                                             aliases_to_scope_map, config):
+    """Determine the restricted scope from both `target_tasks_method` and `project`.
+
+    On certain branches, we'll need differing restricted scopes based on
+    `target_tasks_method`.  However, we can't key solely on that, since that
+    `target_tasks_method` might be run on an unprivileged branch.  This method
+    checks both.
+
+    Args:
+        alias_to_tasks_map (list of lists): each list pair contains the
+            alias and the set of target methods that match. This is ordered.
+        alias_to_project_map (list of lists): each list pair contains the
+            alias and the set of projects that match.  This is ordered.
+        aliases_to_scope_map (dict of dicts): the task alias to project alias to scope
+        config (dict): the task config that defines the target task method and project.
+
+    Returns:
+        string: the scope to use.
+    """
+    project = config.params['project']
+    target = config.params['target_tasks_method']
+    for alias1, tasks in alias_to_tasks_map:
+        for alias2, projects in alias_to_project_map:
+            if target in tasks and project in projects and \
+                    aliases_to_scope_map.get(alias1, {}).get(alias2):
+                return aliases_to_scope_map[alias1][alias2]
+    return aliases_to_scope_map['default']
+
+
 get_signing_cert_scope = functools.partial(
     get_scope_from_project,
     SIGNING_SCOPE_ALIAS_TO_PROJECT,
     SIGNING_CERT_SCOPES
 )
 
 get_beetmover_bucket_scope = functools.partial(
-    get_scope_from_project,
+    get_scope_from_target_method_and_project,
+    BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
     BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
     BEETMOVER_BUCKET_SCOPES
 )
 
 get_beetmover_action_scope = functools.partial(
     get_scope_from_target_method,
     BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
     BEETMOVER_ACTION_SCOPES
 )
 
 get_balrog_server_scope = functools.partial(
     get_scope_from_project,
     BALROG_SCOPE_ALIAS_TO_PROJECT,
     BALROG_SERVER_SCOPES
 )
+
+
+# build_number {{{1
+def get_release_build_number(config):
+    """Get the build number for a release task.
+
+    Currently only applies to beetmover tasks.
+
+    Args:
+        config (dict): the task config that defines the target task method.
+
+    Raises:
+        ValueError: if a release graph doesn't define a valid
+            `os.environ['BUILD_NUMBER']`
+
+    Returns:
+        int: the build number, if applicable.
+    """
+    if config.params['target_tasks_method'] in BEETMOVER_RELEASE_TARGET_TASKS:
+        build_number = str(os.environ.get("BUILD_NUMBER", ""))
+        if not build_number.isdigit():
+            raise ValueError("Release graphs must specify `BUILD_NUMBER` in the environment!")
+        return int(build_number)