Bug 1465117 - Add additional options to backfilling action task r=dustin,jmaher
☠☠ backed out by e16a3d8427b8 ☠ ☠
authorBrian Stack <bstack@mozilla.com>
Fri, 01 Jun 2018 16:44:16 -0700
changeset 421627 3d3fe54d0cb63a2b845f4de479aafb0df517f8d6
parent 421626 6d5d80cfaaae6c787d67ac508eba04d2d77de689
child 421628 ec66192b6be9139037a7a40145570ddd6598bb19
push id34099
push userncsoregi@mozilla.com
push dateWed, 06 Jun 2018 22:00:08 +0000
treeherdermozilla-central@1ab062fd31db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, jmaher
bugs1465117
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1465117 - Add additional options to backfilling action task r=dustin,jmaher MozReview-Commit-ID: FMGjhQbg4im
taskcluster/mach_commands.py
taskcluster/taskgraph/actions/backfill.py
taskcluster/taskgraph/actions/mochitest_retrigger.py
taskcluster/taskgraph/actions/util.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -267,53 +267,45 @@ class MachCommands(MachCommandBase):
                      help='Action input (.yml or .json)')
     @CommandArgument('--task', default=None,
                      help='Task definition (.yml or .json; if omitted, the task will be'
                           'fetched from the queue)')
     @CommandArgument('callback', default=None,
                      help='Action callback name (Python function name)')
     def test_action_callback(self, **options):
         import taskgraph.parameters
-        from taskgraph.util.taskcluster import get_task_definition
         import taskgraph.actions
         import yaml
 
         def load_data(filename):
             with open(filename) as f:
                 if filename.endswith('.yml'):
                     return yaml.safe_load(f)
                 elif filename.endswith('.json'):
                     return json.load(f)
                 else:
                     raise Exception("unknown filename {}".format(filename))
 
         try:
             self.setup_logging()
             task_id = options['task_id']
-            if options['task']:
-                task = load_data(options['task'])
-            elif task_id:
-                task = get_task_definition(task_id)
-            else:
-                task = None
 
             if options['input']:
                 input = load_data(options['input'])
             else:
                 input = None
 
             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
             parameters.check()
 
             root = options['root']
 
             return taskgraph.actions.trigger_action_callback(
                     task_group_id=options['task_group_id'],
                     task_id=task_id,
-                    task=task,
                     input=input,
                     callback=options['callback'],
                     parameters=parameters,
                     root=root,
                     test=True)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
--- a/taskcluster/taskgraph/actions/backfill.py
+++ b/taskcluster/taskgraph/actions/backfill.py
@@ -7,17 +7,18 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
 
 import requests
 from requests.exceptions import HTTPError
 
 from .registry import register_callback_action
-from .util import find_decision_task, create_tasks
+from .util import find_decision_task, create_task_from_def, fix_task_dependencies
+from slugid import nice as slugid
 from taskgraph.util.taskcluster import get_artifact_from_index
 from taskgraph.taskgraph import TaskGraph
 
 PUSHLOG_TMPL = '{}/json-pushes?version=2&startID={}&endID={}'
 INDEX_TMPL = 'gecko.v2.{}.pushlog-id.{}.decision'
 
 logger = logging.getLogger(__name__)
 
@@ -39,27 +40,46 @@ logger = logging.getLogger(__name__)
             'depth': {
                 'type': 'integer',
                 'default': 5,
                 'minimum': 1,
                 'maximum': 10,
                 'title': 'Depth',
                 'description': ('The number of previous pushes before the current '
                                 'push to attempt to trigger this task on.')
+            },
+            'inclusive': {
+                'type': 'boolean',
+                'default': False,
+                'title': 'Inclusive Range',
+                'description': ('If true, the backfill will also retrigger the task '
+                                'on the selected push.')
+            },
+            'addGeckoProfile': {
+                'type': 'boolean',
+                'default': False,
+                'title': 'Add Gecko Profile',
+                'description': 'If true, appends --geckoProfile to mozharness options.'
+            },
+            'testPath': {
+                'type': 'string',
+                'title': 'Test Path',
+                'description': 'If specified, set MOZHARNESS_TEST_PATHS to this value.'
             }
         },
         'additionalProperties': False
     },
     available=lambda parameters: parameters.get('project', None) != 'try'
 )
 def backfill_action(parameters, graph_config, input, task_group_id, task_id, task):
     label = task['metadata']['name']
     pushes = []
-    depth = input.get('depth', 5)
-    end_id = int(parameters['pushlog_id']) - 1
+    inclusive_tweak = 1 if input.get('inclusive') else 0
+    depth = input.get('depth', 5) + inclusive_tweak
+    end_id = int(parameters['pushlog_id']) - (1 - inclusive_tweak)
 
     while True:
         start_id = max(end_id - depth, 0)
         pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], start_id, end_id)
         r = requests.get(pushlog_url)
         r.raise_for_status()
         pushes = pushes + r.json()['pushes'].keys()
         if len(pushes) >= depth:
@@ -85,13 +105,26 @@ def backfill_action(parameters, graph_co
                     INDEX_TMPL.format(parameters['project'], push),
                     'public/parameters.yml')
             push_decision_task_id = find_decision_task(push_params, graph_config)
         except HTTPError as e:
             logger.info('Skipping {} due to missing index artifacts! Error: {}'.format(push, e))
             continue
 
         if label in full_task_graph.tasks.keys():
-            create_tasks(
-                    [label], full_task_graph, label_to_taskid,
-                    push_params, push_decision_task_id, push)
+            task_def = fix_task_dependencies(full_task_graph.tasks[label], label_to_taskid)
+            task_def['taskGroupId'] = push_decision_task_id
+
+            if input.get('addGeckoProfile'):
+                mh_options = task_def['payload'].setdefault('env', {}) \
+                                                .get('MOZHARNESS_OPTIONS', '')
+                task_def['payload']['env']['MOZHARNESS_OPTIONS'] = mh_options + ' --geckoProfile'
+                task_def['extra']['treeherder']['symbol'] += '-p'
+
+            if input.get('testPath'):
+                env = task_def['payload'].setdefault('env', {})
+                env['MOZHARNESS_TEST_PATHS'] = input.get('testPath')
+                task_def['extra']['treeherder']['symbol'] += '-b'
+
+            new_task_id = slugid()
+            create_task_from_def(new_task_id, task_def, parameters['level'])
         else:
             logging.info('Could not find {} on {}. Skipping.'.format(label, push))
--- a/taskcluster/taskgraph/actions/mochitest_retrigger.py
+++ b/taskcluster/taskgraph/actions/mochitest_retrigger.py
@@ -6,19 +6,18 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import logging
 
 from slugid import nice as slugid
 
-from .util import (create_task_from_def, fetch_graph_and_labels)
+from .util import (create_task_from_def, fetch_graph_and_labels, fix_task_dependencies)
 from .registry import register_callback_action
-from taskgraph.util.parameterization import resolve_task_references
 
 TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task"
 
 logger = logging.getLogger(__name__)
 
 
 @register_callback_action(
     name='retrigger-mochitest-reftest-with-options',
@@ -79,23 +78,17 @@ logger = logging.getLogger(__name__)
         'required': ['path']
     }
 )
 def mochitest_retrigger_action(parameters, graph_config, input, task_group_id, task_id, task):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
         parameters, graph_config)
 
     pre_task = full_task_graph.tasks[task['metadata']['name']]
-
-    # fix up the task's dependencies, similar to how optimization would
-    # have done in the decision
-    dependencies = {name: label_to_taskid[label]
-                    for name, label in pre_task.dependencies.iteritems()}
-    new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies)
-    new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
+    new_task_definition = fix_task_dependencies(pre_task, label_to_taskid)
 
     # don't want to run mozharness tests, want a custom mach command instead
     new_task_definition['payload']['command'] += ['--no-run-tests']
 
     custom_mach_command = [task['tags']['test-type']]
 
     # mochitests may specify a flavor
     if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
--- a/taskcluster/taskgraph/actions/util.py
+++ b/taskcluster/taskgraph/actions/util.py
@@ -13,16 +13,17 @@ import os
 
 from requests.exceptions import HTTPError
 
 from taskgraph import create
 from taskgraph.decision import write_artifact
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.optimize import optimize_task_graph
 from taskgraph.util.taskcluster import get_session, find_task_id, get_artifact, list_tasks
+from taskgraph.util.parameterization import resolve_task_references
 
 logger = logging.getLogger(__name__)
 
 PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1'
 
 
 def find_decision_task(parameters, graph_config):
     """Given the parameters for this action, find the taskId of the decision
@@ -100,16 +101,26 @@ def create_task_from_def(task_id, task_d
     it to this function. No dependencies will be scheduled. You must handle
     this yourself. Seeing how create_tasks handles it might prove helpful."""
     task_def['schedulerId'] = 'gecko-level-{}'.format(level)
     label = task_def['metadata']['name']
     session = get_session()
     create.create_task(session, task_id, label, task_def)
 
 
+def fix_task_dependencies(task_def, label_to_taskid):
+    """fix up the task's dependencies, similar to how optimization would
+    have done in the decision"""
+    dependencies = {name: label_to_taskid[label]
+                    for name, label in task_def.dependencies.iteritems()}
+    new_task_definition = resolve_task_references(task_def.label, task_def.task, dependencies)
+    new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
+    return new_task_definition
+
+
 def update_parent(task, graph):
     task.task.setdefault('extra', {})['parent'] = os.environ.get('TASK_ID', '')
     return task
 
 
 def create_tasks(to_run, full_task_graph, label_to_taskid,
                  params, decision_task_id=None, suffix=''):
     """Create new tasks.  The task definition will have {relative-datestamp':