bug 1343393 - add graph logic to handle a fennec beta build. r=mtabara a=release DONTBUILD
authorAki Sasaki <asasaki@mozilla.com>
Wed, 01 Mar 2017 18:01:18 -0800
changeset 378805 15eece89ef862d2b51357ad6f7cf6ed3476f4981
parent 378804 a89019f9eec3011c42eaff160b47c5593ca5d7ec
child 378806 9e486bc61d632b88dd28bc948f5e31497612eca2
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara, release
bugs1343393
milestone53.0a2
bug 1343393 - add graph logic to handle a fennec beta build. r=mtabara a=release DONTBUILD
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/scriptworker.py
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -155,8 +155,22 @@ def target_tasks_mozilla_beta(full_task_
 
 
 @_target_task('mozilla_release_tasks')
 def target_tasks_mozilla_release(full_task_graph, parameters):
     """Select the set of tasks required for a promotable beta or release build
     of linux, plus android CI. The candidates build process involves a pipeline
     of builds and signing, but does not include beetmover or balrog jobs."""
     return target_tasks_mozilla_beta(full_task_graph, parameters)
+
+
+@_target_task('candidates_fennec')
+def target_tasks_candidates_fennec(full_task_graph, parameters):
+    """Select the set of tasks required for a candidates build of fennec. The
+    nightly build process involves a pipeline of builds, signing,
+    and, eventually, uploading the tasks to balrog."""
+    filtered_for_project = target_tasks_nightly(full_task_graph, parameters)
+
+    def filter(task):
+        if task.kind not in ['balrog']:
+            return task.attributes.get('nightly', False)
+
+    return [l for l in filtered_for_project if filter(full_task_graph[l])]
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -11,16 +11,17 @@ complexities of worker implementations, 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import time
 
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema
+from taskgraph.util.scriptworker import get_release_build_number
 from voluptuous import Schema, Any, Required, Optional, Extra
 
 from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
 
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
     basestring,
@@ -547,24 +548,27 @@ def build_scriptworker_signing_payload(c
         'maxRunTime': worker['max-run-time'],
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
 
 
 @payload_builder('beetmover')
 def build_beetmover_payload(config, task, task_def):
     worker = task['worker']
+    build_number = get_release_build_number(config)
 
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
         'upload_date': config.params['build_date'],
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
     if worker.get('locale'):
         task_def['payload']['locale'] = worker['locale']
+    if build_number:
+        task_def['payload']['build_number'] = build_number
 
 
 @payload_builder('balrog')
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'upstreamArtifacts':  worker['upstream-artifacts']
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -10,16 +10,17 @@ these scopes automatically by project; t
 project branch, and merge day uplifts more user friendly.
 
 In the future, we may adjust scopes by other settings as well, e.g. different
 scopes for `push-to-candidates` rather than `push-to-releases`, even if both
 happen on mozilla-beta and mozilla-release.
 """
 from __future__ import absolute_import, print_function, unicode_literals
 import functools
+import os
 
 
 """Map signing scope aliases to sets of projects.
 
 Currently m-c and m-a use nightly signing; m-b and m-r use release signing.
 We will need to add esr support at some point. Eventually we want to add
 nuance so certain m-b and m-r tasks use dep or nightly signing, and we only
 release sign when we have a signed-off set of candidate builds.  This current
@@ -48,46 +49,58 @@ SIGNING_CERT_SCOPES = {
 }
 
 """Map beetmover scope aliases to sets of projects.
 """
 BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [[
     'all-nightly-branches', set([
         'mozilla-central',
         'mozilla-aurora',
-        'jamun',
         'mozilla-beta',
         'mozilla-release',
     ])
 ], [
     'all-release-branches', set([
+        'mozilla-beta',
+        'mozilla-release',
     ])
 ]]
 
+"""The set of all beetmover release target tasks.
+
+Used for both `BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK` and `get_release_build_number`
+"""
+BEETMOVER_RELEASE_TARGET_TASKS = set([
+    'candidates_fennec',
+])
+
 """Map beetmover tasks aliases to sets of target task methods.
 
 This is a list of list-pairs, for ordering.
 """
 BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK = [[
     'all-nightly-tasks', set([
         'nightly_fennec',
         'nightly_linux',
         'mozilla_beta_tasks',
         'mozilla_release_tasks',
     ])
 ], [
-    'all-release-tasks', set([
-    ])
+    'all-release-tasks', BEETMOVER_RELEASE_TARGET_TASKS
 ]]
 
 """Map the beetmover scope aliases to the actual scopes.
 """
 BEETMOVER_BUCKET_SCOPES = {
-    'all-release-branches': 'project:releng:beetmover:bucket:release',
-    'all-nightly-branches': 'project:releng:beetmover:bucket:nightly',
+    'all-release-tasks': {
+        'all-release-branches': 'project:releng:beetmover:bucket:release',
+    },
+    'all-nightly-tasks': {
+        'all-nightly-branches': 'project:releng:beetmover:bucket:nightly',
+    },
     'default': 'project:releng:beetmover:bucket:dep',
 }
 
 """Map the beetmover tasks aliases to the actual action scopes.
 """
 BEETMOVER_ACTION_SCOPES = {
     'all-release-tasks': 'project:releng:beetmover:action:push-to-candidates',
     'all-nightly-tasks': 'project:releng:beetmover:action:push-to-nightly',
@@ -122,22 +135,23 @@ BALROG_SERVER_SCOPES = {
     'nightly': 'project:releng:balrog:nightly',
     'aurora': 'project:releng:balrog:nightly',
     'beta': 'project:releng:balrog:nightly',
     'release': 'project:releng:balrog:nightly',
     'default': 'project:releng:balrog:nightly',
 }
 
 
+# scope functions {{{1
 def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['project']`.
 
     Args:
         alias_to_project_map (list of lists): each list pair contains the
-            alias alias and the set of projects that match.  This is ordered.
+            alias and the set of projects that match.  This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
         config (dict): the task config that defines the project.
 
     Returns:
         string: the scope to use.
     """
     for alias, projects in alias_to_project_map:
         if config.params['project'] in projects and alias in alias_to_scope_map:
@@ -145,44 +159,98 @@ def get_scope_from_project(alias_to_proj
     return alias_to_scope_map['default']
 
 
 def get_scope_from_target_method(alias_to_tasks_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['target_tasks_method']`.
 
     Args:
         alias_to_tasks_map (list of lists): each list pair contains the
-            alias alias and the set of target methods that match. This is ordered.
+            alias and the set of target methods that match. This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
         config (dict): the task config that defines the target task method.
 
     Returns:
         string: the scope to use.
     """
     for alias, tasks in alias_to_tasks_map:
         if config.params['target_tasks_method'] in tasks and alias in alias_to_scope_map:
             return alias_to_scope_map[alias]
     return alias_to_scope_map['default']
 
 
+def get_scope_from_target_method_and_project(alias_to_tasks_map, alias_to_project_map,
+                                             aliases_to_scope_map, config):
+    """Determine the restricted scope from both `target_tasks_method` and `project`.
+
+    On certain branches, we'll need differing restricted scopes based on
+    `target_tasks_method`.  However, we can't key solely on that, since that
+    `target_tasks_method` might be run on an unprivileged branch.  This method
+    checks both.
+
+    Args:
+        alias_to_tasks_map (list of lists): each list pair contains the
+            alias and the set of target methods that match. This is ordered.
+        alias_to_project_map (list of lists): each list pair contains the
+            alias and the set of projects that match.  This is ordered.
+        aliases_to_scope_map (dict of dicts): the task alias to project alias to scope
+        config (dict): the task config that defines the target task method and project.
+
+    Returns:
+        string: the scope to use.
+    """
+    project = config.params['project']
+    target = config.params['target_tasks_method']
+    for alias1, tasks in alias_to_tasks_map:
+        for alias2, projects in alias_to_project_map:
+            if target in tasks and project in projects and \
+                    aliases_to_scope_map.get(alias1, {}).get(alias2):
+                return aliases_to_scope_map[alias1][alias2]
+    return aliases_to_scope_map['default']
+
+
 get_signing_cert_scope = functools.partial(
     get_scope_from_project,
     SIGNING_SCOPE_ALIAS_TO_PROJECT,
     SIGNING_CERT_SCOPES
 )
 
 get_beetmover_bucket_scope = functools.partial(
-    get_scope_from_project,
+    get_scope_from_target_method_and_project,
+    BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
     BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
     BEETMOVER_BUCKET_SCOPES
 )
 
 get_beetmover_action_scope = functools.partial(
     get_scope_from_target_method,
     BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
     BEETMOVER_ACTION_SCOPES
 )
 
 get_balrog_server_scope = functools.partial(
     get_scope_from_project,
     BALROG_SCOPE_ALIAS_TO_PROJECT,
     BALROG_SERVER_SCOPES
 )
+
+
+# build_number {{{1
+def get_release_build_number(config):
+    """Get the build number for a release task.
+
+    Currently only applies to beetmover tasks.
+
+    Args:
+        config (dict): the task config that defines the target task method.
+
+    Raises:
+        ValueError: if a release graph doesn't define a valid
+            `os.environ['BUILD_NUMBER']`
+
+    Returns:
+        int: the build number, if applicable.
+    """
+    if config.params['target_tasks_method'] in BEETMOVER_RELEASE_TARGET_TASKS:
+        build_number = str(os.environ.get("BUILD_NUMBER", ""))
+        if not build_number.isdigit():
+            raise ValueError("Release graphs must specify `BUILD_NUMBER` in the environment!")
+        return int(build_number)