Backed out 1 changesets (bug 1465117) for fix_task_dependencies not working as expected a=backout
authorCiure Andrei <aciure@mozilla.com>
Fri, 08 Jun 2018 00:00:20 +0300
changeset 478437 e16a3d8427b8bb6ceeb0b970702fc2eaccd997b8
parent 478436 199a085199815cc99daa658956a7c9436e1d436b
child 478491 68114b4c0d76f086a3a6a6e8106982ef4023bc13
child 478583 48cfb52f99fd8a55f56842f41494c5220394f9cc
child 478620 8e386f33143372071bfbfeca7596a2144cf0ca85
push id1757
push userffxbld-merge
push dateFri, 24 Aug 2018 17:02:43 +0000
treeherdermozilla-release@736023aebdb1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1465117
milestone62.0a1
backs out3d3fe54d0cb63a2b845f4de479aafb0df517f8d6
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 1 changesets (bug 1465117) for fix_task_dependencies not working as expected a=backout Backed out changeset 3d3fe54d0cb6 (bug 1465117)
taskcluster/mach_commands.py
taskcluster/taskgraph/actions/backfill.py
taskcluster/taskgraph/actions/mochitest_retrigger.py
taskcluster/taskgraph/actions/util.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -267,45 +267,53 @@ class MachCommands(MachCommandBase):
                      help='Action input (.yml or .json)')
     @CommandArgument('--task', default=None,
                      help='Task definition (.yml or .json; if omitted, the task will be'
                           'fetched from the queue)')
     @CommandArgument('callback', default=None,
                      help='Action callback name (Python function name)')
     def test_action_callback(self, **options):
         import taskgraph.parameters
+        from taskgraph.util.taskcluster import get_task_definition
         import taskgraph.actions
         import yaml
 
         def load_data(filename):
             with open(filename) as f:
                 if filename.endswith('.yml'):
                     return yaml.safe_load(f)
                 elif filename.endswith('.json'):
                     return json.load(f)
                 else:
                     raise Exception("unknown filename {}".format(filename))
 
         try:
             self.setup_logging()
             task_id = options['task_id']
+            if options['task']:
+                task = load_data(options['task'])
+            elif task_id:
+                task = get_task_definition(task_id)
+            else:
+                task = None
 
             if options['input']:
                 input = load_data(options['input'])
             else:
                 input = None
 
             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
             parameters.check()
 
             root = options['root']
 
             return taskgraph.actions.trigger_action_callback(
                     task_group_id=options['task_group_id'],
                     task_id=task_id,
+                    task=task,
                     input=input,
                     callback=options['callback'],
                     parameters=parameters,
                     root=root,
                     test=True)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
--- a/taskcluster/taskgraph/actions/backfill.py
+++ b/taskcluster/taskgraph/actions/backfill.py
@@ -7,18 +7,17 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
 
 import requests
 from requests.exceptions import HTTPError
 
 from .registry import register_callback_action
-from .util import find_decision_task, create_task_from_def, fix_task_dependencies
-from slugid import nice as slugid
+from .util import find_decision_task, create_tasks
 from taskgraph.util.taskcluster import get_artifact_from_index
 from taskgraph.taskgraph import TaskGraph
 
 PUSHLOG_TMPL = '{}/json-pushes?version=2&startID={}&endID={}'
 INDEX_TMPL = 'gecko.v2.{}.pushlog-id.{}.decision'
 
 logger = logging.getLogger(__name__)
 
@@ -40,46 +39,27 @@ logger = logging.getLogger(__name__)
             'depth': {
                 'type': 'integer',
                 'default': 5,
                 'minimum': 1,
                 'maximum': 10,
                 'title': 'Depth',
                 'description': ('The number of previous pushes before the current '
                                 'push to attempt to trigger this task on.')
-            },
-            'inclusive': {
-                'type': 'boolean',
-                'default': False,
-                'title': 'Inclusive Range',
-                'description': ('If true, the backfill will also retrigger the task '
-                                'on the selected push.')
-            },
-            'addGeckoProfile': {
-                'type': 'boolean',
-                'default': False,
-                'title': 'Add Gecko Profile',
-                'description': 'If true, appends --geckoProfile to mozharness options.'
-            },
-            'testPath': {
-                'type': 'string',
-                'title': 'Test Path',
-                'description': 'If specified, set MOZHARNESS_TEST_PATHS to this value.'
             }
         },
         'additionalProperties': False
     },
     available=lambda parameters: parameters.get('project', None) != 'try'
 )
 def backfill_action(parameters, graph_config, input, task_group_id, task_id, task):
     label = task['metadata']['name']
     pushes = []
-    inclusive_tweak = 1 if input.get('inclusive') else 0
-    depth = input.get('depth', 5) + inclusive_tweak
-    end_id = int(parameters['pushlog_id']) - (1 - inclusive_tweak)
+    depth = input.get('depth', 5)
+    end_id = int(parameters['pushlog_id']) - 1
 
     while True:
         start_id = max(end_id - depth, 0)
         pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], start_id, end_id)
         r = requests.get(pushlog_url)
         r.raise_for_status()
         pushes = pushes + r.json()['pushes'].keys()
         if len(pushes) >= depth:
@@ -105,26 +85,13 @@ def backfill_action(parameters, graph_co
                     INDEX_TMPL.format(parameters['project'], push),
                     'public/parameters.yml')
             push_decision_task_id = find_decision_task(push_params, graph_config)
         except HTTPError as e:
             logger.info('Skipping {} due to missing index artifacts! Error: {}'.format(push, e))
             continue
 
         if label in full_task_graph.tasks.keys():
-            task_def = fix_task_dependencies(full_task_graph.tasks[label], label_to_taskid)
-            task_def['taskGroupId'] = push_decision_task_id
-
-            if input.get('addGeckoProfile'):
-                mh_options = task_def['payload'].setdefault('env', {}) \
-                                                .get('MOZHARNESS_OPTIONS', '')
-                task_def['payload']['env']['MOZHARNESS_OPTIONS'] = mh_options + ' --geckoProfile'
-                task_def['extra']['treeherder']['symbol'] += '-p'
-
-            if input.get('testPath'):
-                env = task_def['payload'].setdefault('env', {})
-                env['MOZHARNESS_TEST_PATHS'] = input.get('testPath')
-                task_def['extra']['treeherder']['symbol'] += '-b'
-
-            new_task_id = slugid()
-            create_task_from_def(new_task_id, task_def, parameters['level'])
+            create_tasks(
+                    [label], full_task_graph, label_to_taskid,
+                    push_params, push_decision_task_id, push)
         else:
             logging.info('Could not find {} on {}. Skipping.'.format(label, push))
--- a/taskcluster/taskgraph/actions/mochitest_retrigger.py
+++ b/taskcluster/taskgraph/actions/mochitest_retrigger.py
@@ -6,18 +6,19 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import logging
 
 from slugid import nice as slugid
 
-from .util import (create_task_from_def, fetch_graph_and_labels, fix_task_dependencies)
+from .util import (create_task_from_def, fetch_graph_and_labels)
 from .registry import register_callback_action
+from taskgraph.util.parameterization import resolve_task_references
 
 TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task"
 
 logger = logging.getLogger(__name__)
 
 
 @register_callback_action(
     name='retrigger-mochitest-reftest-with-options',
@@ -78,17 +79,23 @@ logger = logging.getLogger(__name__)
         'required': ['path']
     }
 )
 def mochitest_retrigger_action(parameters, graph_config, input, task_group_id, task_id, task):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
         parameters, graph_config)
 
     pre_task = full_task_graph.tasks[task['metadata']['name']]
-    new_task_definition = fix_task_dependencies(pre_task, label_to_taskid)
+
+    # fix up the task's dependencies, similar to how optimization would
+    # have done in the decision
+    dependencies = {name: label_to_taskid[label]
+                    for name, label in pre_task.dependencies.iteritems()}
+    new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies)
+    new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
 
     # don't want to run mozharness tests, want a custom mach command instead
     new_task_definition['payload']['command'] += ['--no-run-tests']
 
     custom_mach_command = [task['tags']['test-type']]
 
     # mochitests may specify a flavor
     if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
--- a/taskcluster/taskgraph/actions/util.py
+++ b/taskcluster/taskgraph/actions/util.py
@@ -13,17 +13,16 @@ import os
 
 from requests.exceptions import HTTPError
 
 from taskgraph import create
 from taskgraph.decision import write_artifact
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.optimize import optimize_task_graph
 from taskgraph.util.taskcluster import get_session, find_task_id, get_artifact, list_tasks
-from taskgraph.util.parameterization import resolve_task_references
 
 logger = logging.getLogger(__name__)
 
 PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1'
 
 
 def find_decision_task(parameters, graph_config):
     """Given the parameters for this action, find the taskId of the decision
@@ -101,26 +100,16 @@ def create_task_from_def(task_id, task_d
     it to this function. No dependencies will be scheduled. You must handle
     this yourself. Seeing how create_tasks handles it might prove helpful."""
     task_def['schedulerId'] = 'gecko-level-{}'.format(level)
     label = task_def['metadata']['name']
     session = get_session()
     create.create_task(session, task_id, label, task_def)
 
 
-def fix_task_dependencies(task_def, label_to_taskid):
-    """fix up the task's dependencies, similar to how optimization would
-    have done in the decision"""
-    dependencies = {name: label_to_taskid[label]
-                    for name, label in task_def.dependencies.iteritems()}
-    new_task_definition = resolve_task_references(task_def.label, task_def.task, dependencies)
-    new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
-    return new_task_definition
-
-
 def update_parent(task, graph):
     task.task.setdefault('extra', {})['parent'] = os.environ.get('TASK_ID', '')
     return task
 
 
 def create_tasks(to_run, full_task_graph, label_to_taskid,
                  params, decision_task_id=None, suffix=''):
     """Create new tasks.  The task definition will have {relative-datestamp':