Bug 1352477 - taskgraph ignores run-on-project for fennec-nightlies r=aki a=release CLOSED TREE
authorJohan Lorenzo <jlorenzo@mozilla.com>
Tue, 18 Apr 2017 15:30:26 +0200
changeset 375972 4dae386f2c331851b192a3ef493cc139fd562a44
parent 375971 557af4db9a7807c5a956d19d1847349dd5432169
child 375973 94b7e538af7dbb9a6567cad9125c2f5411e7fb80
push id11072
push userjlorenzo@mozilla.com
push dateTue, 18 Apr 2017 13:37:27 +0000
treeherdermozilla-aurora@4dae386f2c33 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki, release
bugs1352477
milestone54.0a2
Bug 1352477 - taskgraph ignores run-on-project for fennec-nightlies r=aki a=release CLOSED TREE MozReview-Commit-ID: B4vCAoRye7F
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/ci/upload-symbols/job-template.yml
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/l10n.py
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -23,16 +23,20 @@ job-template:
       by-build-platform:
          default: Localization
          android-api-15-nightly: Single Locale Repack
    locales-file:
       by-build-platform:
          default: browser/locales/all-locales
          android-api-15-nightly: mobile/locales/l10n-changesets.json
    chunks: 10
+   run-on-projects:
+     - mozilla-central
+     - mozilla-aurora
+     - mozilla-beta
    run-time:
       by-build-platform:
          default: 36000
          android-api-15-nightly: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-nightly: internal
--- a/taskcluster/ci/upload-symbols/job-template.yml
+++ b/taskcluster/ci/upload-symbols/job-template.yml
@@ -1,17 +1,16 @@
 label: # see transforms
 description: Upload Symbols
 dependencies: # see transforms
 expires-after: 7 days
 deadline-after: 24 hours
 run-on-projects:
     - try
-    - mozilla-beta
-    - mozilla-release
+    - release
 worker-type: aws-provisioner-v1/gecko-symbol-upload
 worker:
     implementation: docker-worker
     max-run-time: 600
     command: ["/bin/bash", "bin/upload.sh"]
     docker-image: taskclusterprivate/upload_symbols:0.0.4
     env:
         GECKO_HEAD_REPOSITORY: # see transforms
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -18,24 +18,41 @@ def _target_task(name):
     return wrap
 
 
 def get_method(method):
     """Get a target_task_method to pass to a TaskGraphGenerator."""
     return _target_task_methods[method]
 
 
+def filter_on_nightly(task, parameters):
+    return not task.attributes.get('nightly') or parameters.get('include_nightly')
+
+
 def filter_for_project(task, parameters):
     """Filter tasks by project.  Optionally enable nightlies."""
-    if task.attributes.get('nightly') and not parameters.get('include_nightly'):
-        return False
     run_on_projects = set(task.attributes.get('run_on_projects', []))
     return match_run_on_projects(parameters['project'], run_on_projects)
 
 
+def filter_upload_symbols(task, parameters):
+    # Filters out symbols when there are not part of a nightly or a release build
+    # TODO Remove this too specific filter (bug 1353296)
+    return '-upload-symbols' not in task.label or \
+        task.attributes.get('nightly') or \
+        parameters.get('project') in ('mozilla-beta', 'mozilla-release')
+
+
+def standard_filter(task, parameters):
+    return all(
+        filter_func(task, parameters) for filter_func in
+        (filter_on_nightly, filter_for_project, filter_upload_symbols)
+    )
+
+
 @_target_task('try_option_syntax')
 def target_tasks_try_option_syntax(full_task_graph, parameters):
     """Generate a list of target tasks based on try syntax in
     parameters['message'] and, for context, the full task graph."""
     options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph)
     target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues()
                            if options.task_matches(t.attributes)]
 
@@ -81,17 +98,17 @@ def target_tasks_try_option_syntax(full_
 
 
 @_target_task('default')
 def target_tasks_default(full_task_graph, parameters):
     """Target the tasks which have indicated they should be run on this project
     via the `run_on_projects` attributes."""
 
     return [l for l, t in full_task_graph.tasks.iteritems()
-            if filter_for_project(t, parameters)]
+            if standard_filter(t, parameters)]
 
 
 @_target_task('ash_tasks')
 def target_tasks_ash(full_task_graph, parameters):
     """Target tasks that only run on the ash branch."""
     def filter(task):
         platform = task.attributes.get('build_platform')
         # Early return if platform is None
@@ -186,17 +203,19 @@ def target_tasks_code_coverage(full_task
 @_target_task('nightly_fennec')
 def target_tasks_nightly_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     def filter(task):
         platform = task.attributes.get('build_platform')
         if platform in ('android-api-15-nightly', 'android-x86-nightly', 'android-nightly'):
-            return task.attributes.get('nightly', False)
+            if not task.attributes.get('nightly', False):
+                return False
+            return filter_for_project(task, parameters)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_linux')
 def target_tasks_nightly_linux(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of linux. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
@@ -209,17 +228,17 @@ def target_tasks_nightly_linux(full_task
 
 @_target_task('mozilla_beta_tasks')
 def target_tasks_mozilla_beta(full_task_graph, parameters):
     """Select the set of tasks required for a promotable beta or release build
     of linux, plus android CI. The candidates build process involves a pipeline
     of builds and signing, but does not include beetmover or balrog jobs."""
 
     def filter(task):
-        if not filter_for_project(task, parameters):
+        if not standard_filter(task, parameters):
             return False
         platform = task.attributes.get('build_platform')
         if platform in ('linux64-pgo', 'linux-pgo', 'win32-pgo', 'win64-pgo',
                         'android-api-15-nightly', 'android-x86-nightly',
                         'win32', 'win64', 'macosx64'):
             return False
         if platform in ('linux64', 'linux'):
             if task.attributes['build_type'] == 'opt':
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import, 
 
 import copy
 import json
 
 from mozbuild.chunkify import chunkify
 from taskgraph.transforms.base import (
     TransformSequence,
 )
+from taskgraph.transforms.job import job_description_schema
 from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
     resolve_keyed_by,
 )
 from taskgraph.util.treeherder import split_symbol, join_symbol
 from voluptuous import (
     Any,
@@ -32,16 +33,20 @@ from voluptuous import (
 def _by_platform(arg):
     return optionally_keyed_by('build-platform', arg)
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
     basestring,
     {Required('task-reference'): basestring})
 
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
+
 l10n_description_schema = Schema({
     # Name for this job, inferred from the dependent job before validation
     Required('name'): basestring,
 
     # build-platform, inferred from dependent job before validation
     Required('build-platform'): basestring,
 
     # max run time of the task
@@ -73,16 +78,18 @@ l10n_description_schema = Schema({
         Required('job-name'): _by_platform(basestring),
 
         # Type of index
         Optional('type'): basestring,
     },
     # Description of the localized task
     Required('description'): _by_platform(basestring),
 
+    Optional('run-on-projects'): job_description_schema['run-on-projects'],
+
     # task object of the dependent task
     Required('dependent-task'): object,
 
     # worker-type to utilize
     Required('worker-type'): _by_platform(basestring),
 
     # File which contains the used locales
     Required('locales-file'): _by_platform(basestring),
@@ -371,17 +378,17 @@ def make_job_description(config, jobs):
             },
             'attributes': job['attributes'],
             'treeherder': {
                 'kind': 'build',
                 'tier': job['treeherder']['tier'],
                 'symbol': job['treeherder']['symbol'],
                 'platform': job['treeherder']['platform'],
             },
-            'run-on-projects': [],
+            'run-on-projects': job.get('run-on-projects') if job.get('run-on-projects') else [],
         }
 
         if job.get('index'):
             job_description['index'] = {
                 'product': job['index']['product'],
                 'job-name': job['index']['job-name'],
                 'type': job['index'].get('type', 'generic'),
             }