Bug 1352477 - taskgraph ignores run-on-project for fennec-nightlies r=aki a=release CLOSED TREE
MozReview-Commit-ID: B4vCAoRye7F
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -23,16 +23,20 @@ job-template:
by-build-platform:
default: Localization
android-api-15-nightly: Single Locale Repack
locales-file:
by-build-platform:
default: browser/locales/all-locales
android-api-15-nightly: mobile/locales/l10n-changesets.json
chunks: 10
+ run-on-projects:
+ - mozilla-central
+ - mozilla-aurora
+ - mozilla-beta
run-time:
by-build-platform:
default: 36000
android-api-15-nightly: 18000
tooltool:
by-build-platform:
default: public
android-api-15-nightly: internal
--- a/taskcluster/ci/upload-symbols/job-template.yml
+++ b/taskcluster/ci/upload-symbols/job-template.yml
@@ -1,17 +1,16 @@
label: # see transforms
description: Upload Symbols
dependencies: # see transforms
expires-after: 7 days
deadline-after: 24 hours
run-on-projects:
- try
- - mozilla-beta
- - mozilla-release
+ - release
worker-type: aws-provisioner-v1/gecko-symbol-upload
worker:
implementation: docker-worker
max-run-time: 600
command: ["/bin/bash", "bin/upload.sh"]
docker-image: taskclusterprivate/upload_symbols:0.0.4
env:
GECKO_HEAD_REPOSITORY: # see transforms
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -18,24 +18,41 @@ def _target_task(name):
return wrap
def get_method(method):
"""Get a target_task_method to pass to a TaskGraphGenerator."""
return _target_task_methods[method]
+def filter_on_nightly(task, parameters):
+ return not task.attributes.get('nightly') or parameters.get('include_nightly')
+
+
def filter_for_project(task, parameters):
"""Filter tasks by project. Optionally enable nightlies."""
- if task.attributes.get('nightly') and not parameters.get('include_nightly'):
- return False
run_on_projects = set(task.attributes.get('run_on_projects', []))
return match_run_on_projects(parameters['project'], run_on_projects)
+def filter_upload_symbols(task, parameters):
+ # Filters out symbols when there are not part of a nightly or a release build
+ # TODO Remove this too specific filter (bug 1353296)
+ return '-upload-symbols' not in task.label or \
+ task.attributes.get('nightly') or \
+ parameters.get('project') in ('mozilla-beta', 'mozilla-release')
+
+
+def standard_filter(task, parameters):
+ return all(
+ filter_func(task, parameters) for filter_func in
+ (filter_on_nightly, filter_for_project, filter_upload_symbols)
+ )
+
+
@_target_task('try_option_syntax')
def target_tasks_try_option_syntax(full_task_graph, parameters):
"""Generate a list of target tasks based on try syntax in
parameters['message'] and, for context, the full task graph."""
options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph)
target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues()
if options.task_matches(t.attributes)]
@@ -81,17 +98,17 @@ def target_tasks_try_option_syntax(full_
@_target_task('default')
def target_tasks_default(full_task_graph, parameters):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
return [l for l, t in full_task_graph.tasks.iteritems()
- if filter_for_project(t, parameters)]
+ if standard_filter(t, parameters)]
@_target_task('ash_tasks')
def target_tasks_ash(full_task_graph, parameters):
"""Target tasks that only run on the ash branch."""
def filter(task):
platform = task.attributes.get('build_platform')
# Early return if platform is None
@@ -186,17 +203,19 @@ def target_tasks_code_coverage(full_task
@_target_task('nightly_fennec')
def target_tasks_nightly_fennec(full_task_graph, parameters):
"""Select the set of tasks required for a nightly build of fennec. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
def filter(task):
platform = task.attributes.get('build_platform')
if platform in ('android-api-15-nightly', 'android-x86-nightly', 'android-nightly'):
- return task.attributes.get('nightly', False)
+ if not task.attributes.get('nightly', False):
+ return False
+ return filter_for_project(task, parameters)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_linux')
def target_tasks_nightly_linux(full_task_graph, parameters):
"""Select the set of tasks required for a nightly build of linux. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
@@ -209,17 +228,17 @@ def target_tasks_nightly_linux(full_task
@_target_task('mozilla_beta_tasks')
def target_tasks_mozilla_beta(full_task_graph, parameters):
"""Select the set of tasks required for a promotable beta or release build
of linux, plus android CI. The candidates build process involves a pipeline
of builds and signing, but does not include beetmover or balrog jobs."""
def filter(task):
- if not filter_for_project(task, parameters):
+ if not standard_filter(task, parameters):
return False
platform = task.attributes.get('build_platform')
if platform in ('linux64-pgo', 'linux-pgo', 'win32-pgo', 'win64-pgo',
'android-api-15-nightly', 'android-x86-nightly',
'win32', 'win64', 'macosx64'):
return False
if platform in ('linux64', 'linux'):
if task.attributes['build_type'] == 'opt':
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import,
import copy
import json
from mozbuild.chunkify import chunkify
from taskgraph.transforms.base import (
TransformSequence,
)
+from taskgraph.transforms.job import job_description_schema
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
resolve_keyed_by,
)
from taskgraph.util.treeherder import split_symbol, join_symbol
from voluptuous import (
Any,
@@ -32,16 +33,20 @@ from voluptuous import (
def _by_platform(arg):
return optionally_keyed_by('build-platform', arg)
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
+
l10n_description_schema = Schema({
# Name for this job, inferred from the dependent job before validation
Required('name'): basestring,
# build-platform, inferred from dependent job before validation
Required('build-platform'): basestring,
# max run time of the task
@@ -73,16 +78,18 @@ l10n_description_schema = Schema({
Required('job-name'): _by_platform(basestring),
# Type of index
Optional('type'): basestring,
},
# Description of the localized task
Required('description'): _by_platform(basestring),
+ Optional('run-on-projects'): job_description_schema['run-on-projects'],
+
# task object of the dependent task
Required('dependent-task'): object,
# worker-type to utilize
Required('worker-type'): _by_platform(basestring),
# File which contains the used locales
Required('locales-file'): _by_platform(basestring),
@@ -371,17 +378,17 @@ def make_job_description(config, jobs):
},
'attributes': job['attributes'],
'treeherder': {
'kind': 'build',
'tier': job['treeherder']['tier'],
'symbol': job['treeherder']['symbol'],
'platform': job['treeherder']['platform'],
},
- 'run-on-projects': [],
+ 'run-on-projects': job.get('run-on-projects') if job.get('run-on-projects') else [],
}
if job.get('index'):
job_description['index'] = {
'product': job['index']['product'],
'job-name': job['index']['job-name'],
'type': job['index'].get('type', 'generic'),
}