Bug 1277665 - add --dry-run functionality to mach taskgraph; r?dustin
authorMike Shal <mshal@mozilla.com>
Fri, 24 Jun 2016 13:42:52 -0400
changeset 788595 426ead52ac54eb0a90dbcbb3693a85311a8a4b61
parent 788211 0e3f8401b804702c894eb5fdf7eae3cbdf618668
child 788596 5dfb8ba4874ccefa8004ff09531cbf91013977ec
push id131317
push usermshal@mozilla.com
push dateMon, 27 Jun 2016 21:20:19 +0000
treeherdertry@90eaa53d6345 [default view] [failures only]
reviewersdustin
bugs1277665, 100644
milestone50.0a1
Bug 1277665 - add --dry-run functionality to mach taskgraph; r?dustin From c1027e1c005e871d377d363463d29340fa3b4b10 Mon Sep 17 00:00:00 2001 MozReview-Commit-ID: JByEMoV8OPP --- taskcluster/mach_commands.py | 3 +++ taskcluster/taskgraph/decision.py | 5 ++++ taskcluster/taskgraph/kind/docker_image.py | 15 ++++++++--- taskcluster/taskgraph/kind/dry_run.py | 41 ++++++++++++++++++++++++++++++ taskcluster/taskgraph/kind/legacy.py | 22 ++++++++++------ 5 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 taskcluster/taskgraph/kind/dry_run.py
taskcluster/mach_commands.py
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/kind/docker_image.py
taskcluster/taskgraph/kind/dry_run.py
taskcluster/taskgraph/kind/legacy.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -133,16 +133,19 @@ class MachCommands(MachCommandBase):
                      required=True,
                      default=0)
     @CommandArgument('--owner',
                      required=True,
                      help='email address of who owns this graph')
     @CommandArgument('--level',
                      required=True,
                      help='SCM level of this repository')
+    @CommandArgument('--dry-run',
+                     action='store_true', default=False,
+                     help='Stub out variable data (taskIds, dates, etc) from task definitions.')
     def taskgraph_decision(self, **options):
         """Run the decision task: generate a task graph and submit to
         TaskCluster.  This is only meant to be called within decision tasks,
         and requires a great many arguments.  Commands like `mach taskgraph
         optimized` are better suited to use on the command line, and can take
         the parameters file generated by a decision task.  """
 
         import taskgraph.decision
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -56,16 +56,20 @@ def taskgraph_decision(options):
     # create a TaskGraphGenerator instance
     target_tasks_method = parameters.get('target_tasks_method', 'all_tasks')
     target_tasks_method = get_method(target_tasks_method)
     tgg = TaskGraphGenerator(
         root_dir=options['root'],
         parameters=parameters,
         target_tasks_method=target_tasks_method)
 
+    if options['dry_run']:
+        print(json.dumps(tgg.full_task_graph.to_json(), indent=4, sort_keys=True))
+        return
+
     # write out the parameters used to generate this graph
     write_artifact('parameters.yml', dict(**parameters))
 
     # write out the full graph for reference
     write_artifact('full-task-graph.json', tgg.full_task_graph.to_json())
 
     # write out the target task set to allow reproducing this as input
     write_artifact('target-tasks.json', tgg.target_task_set.tasks.keys())
@@ -91,16 +95,17 @@ def get_decision_parameters(options):
         'head_rev',
         'head_ref',
         'message',
         'project',
         'pushlog_id',
         'owner',
         'level',
         'target_tasks_method',
+        'dry_run',
     ] if n in options}
 
     project = parameters['project']
     try:
         parameters.update(PER_PROJECT_PARAMETERS[project])
     except KeyError:
         logger.warning("using default project parameters; add {} to "
                        "PER_PROJECT_PARAMETERS in {} to customize behavior "
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -13,30 +13,37 @@ import time
 
 from . import base
 from ..types import Task
 from taskgraph.util.docker import (
     docker_image,
     generate_context_hash
 )
 from taskgraph.util.templates import Templates
-from taskgraph.util.time import (
-    json_time_from_now,
-    current_json_time,
-)
 
 logger = logging.getLogger(__name__)
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
 
 
 class DockerImageKind(base.Kind):
 
     def load_tasks(self, params):
+        if params['dry_run']:
+            from .dry_run import (
+                current_json_time,
+                json_time_from_now,
+            )
+        else:
+            from taskgraph.util.time import (
+                current_json_time,
+                json_time_from_now,
+            )
+
         # TODO: make this match the pushdate (get it from a parameter rather than vcs)
         pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
 
         parameters = {
             'pushlog_id': params.get('pushlog_id', 0),
             'pushdate': pushdate,
             'pushtime': pushdate[8:],
             'year': pushdate[0:4],
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/kind/dry_run.py
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+from taskgraph.util.time import value_of
+
+# The current_task counter ensures a unique taskId for each task, but remains
+# consistent between dry runs.
+current_task = 0
+
+
+def json_time_from_now(input_str, now=None):
+    '''
+    :param str input_str: Input string (see value of)
+    :param datetime now: Optionally set the definition of `now`
+    :returns: JSON string representation of time in future.
+    '''
+
+    if now is None:
+        now = datetime.date.fromordinal(1)
+
+    time = now + value_of(input_str)
+
+    # Sorta a big hack but the json schema validator for date does not like the
+    # ISO dates until 'Z' (for timezone) is added...
+    return time.isoformat() + 'Z'
+
+
+def current_json_time():
+    '''
+    :returns: JSON string representation of the current time.
+    '''
+
+    return datetime.date.fromordinal(1).isoformat() + 'Z'
+
+
+def mklabel():
+    global current_task
+    current_task += 1
+    return 'TaskLabel==%04d' % current_task
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -12,20 +12,16 @@ import re
 import time
 from collections import namedtuple
 
 from . import base
 from ..types import Task
 from mozpack.path import match as mozpackmatch
 from slugid import nice as slugid
 from taskgraph.util.legacy_commit_parser import parse_commit
-from taskgraph.util.time import (
-    json_time_from_now,
-    current_json_time,
-)
 from taskgraph.util.templates import Templates
 from taskgraph.util.docker import docker_image
 
 
 ROOT = os.path.dirname(os.path.realpath(__file__))
 GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..'))
 # TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with
 # actual taskIds at the very last minute, in get_task_definition
@@ -43,20 +39,16 @@ TREEHERDER_ROUTES = {
 }
 
 # time after which a try build's results will expire
 TRY_EXPIRATION = "14 days"
 
 logger = logging.getLogger(__name__)
 
 
-def mklabel():
-    return TASKID_PLACEHOLDER.format(slugid())
-
-
 def merge_dicts(*dicts):
     merged_dict = {}
     for dictionary in dicts:
         merged_dict.update(dictionary)
     return merged_dict
 
 
 def gaia_info():
@@ -298,16 +290,30 @@ class LegacyKind(base.Kind):
     `testing/taskcluster/tasks`.  The tasks already have dependency links.
 
     The existing task-graph generation generates slugids for tasks during task
     generation, so this kind labels tasks using those slugids, with a prefix of
     "TaskLabel==".  These labels are unfortunately not stable from run to run.
     """
 
     def load_tasks(self, params):
+        if params['dry_run']:
+            from .dry_run import (
+                mklabel,
+                current_json_time,
+                json_time_from_now,
+            )
+        else:
+            def mklabel():
+                return TASKID_PLACEHOLDER.format(slugid())
+            from taskgraph.util.time import (
+                current_json_time,
+                json_time_from_now,
+            )
+
         root = os.path.abspath(os.path.join(self.path, self.config['legacy_path']))
 
         project = params['project']
         # NOTE: message is ignored here; we always use DEFAULT_TRY, then filter the
         # resulting task graph later
         message = DEFAULT_TRY
 
         templates = Templates(root)