Bug 1465117 - Take two at updating backfill task. Use modifier this time. r=dustin,jmaher
authorBrian Stack <bstack@mozilla.com>
Fri, 08 Jun 2018 12:08:34 -0700
changeset 479867 2731368a39e57a7fda79ebe8b33a9251a327fe5d
parent 479866 1c119911fc842731dbbf4c207ec84e3b3a09f59e
child 479927 9e214df80589d11213cdd0db69794b29063baec8
push id1757
push userffxbld-merge
push dateFri, 24 Aug 2018 17:02:43 +0000
treeherdermozilla-release@736023aebdb1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, jmaher
bugs1465117
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1465117 - Take two at updating backfill task. Use modifier this time. r=dustin,jmaher MozReview-Commit-ID: CAs0WRb839r
taskcluster/mach_commands.py
taskcluster/taskgraph/actions/backfill.py
taskcluster/taskgraph/actions/util.py
taskcluster/taskgraph/task.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -267,53 +267,45 @@ class MachCommands(MachCommandBase):
                      help='Action input (.yml or .json)')
     @CommandArgument('--task', default=None,
                      help='Task definition (.yml or .json; if omitted, the task will be'
                           'fetched from the queue)')
     @CommandArgument('callback', default=None,
                      help='Action callback name (Python function name)')
     def test_action_callback(self, **options):
         import taskgraph.parameters
-        from taskgraph.util.taskcluster import get_task_definition
         import taskgraph.actions
         import yaml
 
         def load_data(filename):
             with open(filename) as f:
                 if filename.endswith('.yml'):
                     return yaml.safe_load(f)
                 elif filename.endswith('.json'):
                     return json.load(f)
                 else:
                     raise Exception("unknown filename {}".format(filename))
 
         try:
             self.setup_logging()
             task_id = options['task_id']
-            if options['task']:
-                task = load_data(options['task'])
-            elif task_id:
-                task = get_task_definition(task_id)
-            else:
-                task = None
 
             if options['input']:
                 input = load_data(options['input'])
             else:
                 input = None
 
             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
             parameters.check()
 
             root = options['root']
 
             return taskgraph.actions.trigger_action_callback(
                     task_group_id=options['task_group_id'],
                     task_id=task_id,
-                    task=task,
                     input=input,
                     callback=options['callback'],
                     parameters=parameters,
                     root=root,
                     test=True)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
--- a/taskcluster/taskgraph/actions/backfill.py
+++ b/taskcluster/taskgraph/actions/backfill.py
@@ -39,27 +39,45 @@ logger = logging.getLogger(__name__)
             'depth': {
                 'type': 'integer',
                 'default': 5,
                 'minimum': 1,
                 'maximum': 10,
                 'title': 'Depth',
                 'description': ('The number of previous pushes before the current '
                                 'push to attempt to trigger this task on.')
+            },
+            'inclusive': {
+                'type': 'boolean',
+                'default': False,
+                'title': 'Inclusive Range',
+                'description': ('If true, the backfill will also retrigger the task '
+                                'on the selected push.')
+            },
+            'addGeckoProfile': {
+                'type': 'boolean',
+                'default': False,
+                'title': 'Add Gecko Profile',
+                'description': 'If true, appends --geckoProfile to mozharness options.'
+            },
+            'testPath': {
+                'type': 'string',
+                'title': 'Test Path',
             }
         },
         'additionalProperties': False
     },
     available=lambda parameters: parameters.get('project', None) != 'try'
 )
 def backfill_action(parameters, graph_config, input, task_group_id, task_id, task):
     label = task['metadata']['name']
     pushes = []
-    depth = input.get('depth', 5)
-    end_id = int(parameters['pushlog_id']) - 1
+    inclusive_tweak = 1 if input.get('inclusive') else 0
+    depth = input.get('depth', 5) + inclusive_tweak
+    end_id = int(parameters['pushlog_id']) - (1 - inclusive_tweak)
 
     while True:
         start_id = max(end_id - depth, 0)
         pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], start_id, end_id)
         r = requests.get(pushlog_url)
         r.raise_for_status()
         pushes = pushes + r.json()['pushes'].keys()
         if len(pushes) >= depth:
@@ -85,13 +103,27 @@ def backfill_action(parameters, graph_co
                     INDEX_TMPL.format(parameters['project'], push),
                     'public/parameters.yml')
             push_decision_task_id = find_decision_task(push_params, graph_config)
         except HTTPError as e:
             logger.info('Skipping {} due to missing index artifacts! Error: {}'.format(push, e))
             continue
 
         if label in full_task_graph.tasks.keys():
-            create_tasks(
-                    [label], full_task_graph, label_to_taskid,
-                    push_params, push_decision_task_id, push)
+            def modifier(task):
+                if task.label != label:
+                    return task
+                if input.get('addGeckoProfile'):
+                    mh = task.task['payload'].setdefault('env', {}) \
+                                                    .get('MOZHARNESS_OPTIONS', '')
+                    task.task['payload']['env']['MOZHARNESS_OPTIONS'] = mh + ' --geckoProfile'
+                    task.task['extra']['treeherder']['symbol'] += '-p'
+
+                if input.get('testPath'):
+                    env = task.task['payload'].setdefault('env', {})
+                    env['MOZHARNESS_TEST_PATHS'] = input.get('testPath')
+                    task.task['extra']['treeherder']['symbol'] += '-b'
+                return task
+
+            create_tasks([label], full_task_graph, label_to_taskid,
+                         push_params, push_decision_task_id, push, modifier=modifier)
         else:
             logging.info('Could not find {} on {}. Skipping.'.format(label, push))
--- a/taskcluster/taskgraph/actions/util.py
+++ b/taskcluster/taskgraph/actions/util.py
@@ -106,35 +106,40 @@ def create_task_from_def(task_id, task_d
 
 
 def update_parent(task, graph):
     task.task.setdefault('extra', {})['parent'] = os.environ.get('TASK_ID', '')
     return task
 
 
 def create_tasks(to_run, full_task_graph, label_to_taskid,
-                 params, decision_task_id=None, suffix=''):
+                 params, decision_task_id=None, suffix='', modifier=lambda t: t):
     """Create new tasks.  The task definition will have {relative-datestamp':
     '..'} rendered just like in a decision task.  Action callbacks should use
     this function to create new tasks,
     allowing easy debugging with `mach taskgraph action-callback --test`.
     This builds up all required tasks to run in order to run the tasks requested.
 
+    Optionally this function takes a `modifier` function that is passed in each
+    task before it is put into a new graph. It should return a valid task. Note
+    that this is passed _all_ tasks in the graph, not just the set in to_run. You
+    may want to skip modifying tasks not in your to_run list.
+
     If you wish to create the tasks in a new group, leave out decision_task_id."""
     if suffix != '':
         suffix = '-{}'.format(suffix)
     to_run = set(to_run)
 
     #  Copy to avoid side-effects later
     full_task_graph = copy.deepcopy(full_task_graph)
     label_to_taskid = label_to_taskid.copy()
 
     target_graph = full_task_graph.graph.transitive_closure(to_run)
     target_task_graph = TaskGraph(
-        {l: full_task_graph[l] for l in target_graph.nodes},
+        {l: modifier(full_task_graph[l]) for l in target_graph.nodes},
         target_graph)
     target_task_graph.for_each_task(update_parent)
     optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
                                                                 params,
                                                                 to_run,
                                                                 label_to_taskid)
     write_artifact('task-graph{}.json'.format(suffix), optimized_task_graph.to_json())
     write_artifact('label-to-taskid{}.json'.format(suffix), label_to_taskid)
--- a/taskcluster/taskgraph/task.py
+++ b/taskcluster/taskgraph/task.py
@@ -16,17 +16,17 @@ class Task(object):
     - optimization: optimization to apply to the task (see taskgraph.optimize)
     - dependencies: tasks this one depends on, in the form {name: label}, for example
       {'build': 'build-linux64/opt', 'docker-image': 'build-docker-image-desktop-test'}
 
     And later, as the task-graph processing proceeds:
 
     - task_id -- TaskCluster taskId under which this task will be created
 
-    This class is just a convenience wraper for the data type and managing
+    This class is just a convenience wrapper for the data type and managing
     display, comparison, serialization, etc. It has no functionality of its own.
     """
     def __init__(self, kind, label, attributes, task,
                  optimization=None, dependencies=None):
         self.kind = kind
         self.label = label
         self.attributes = attributes
         self.task = task