bug 1282923: define a nightly kind in taskgraph draft
authorAnthony Miyaguchi <amiyaguchi@mozilla.com>
Thu, 30 Jun 2016 14:18:16 -0700
changeset 383014 b44ebf941057967b2b842d7057016d18f317d367
parent 383013 3fc0469634a5a315b38144169d9cc8243120805a
child 524359 a6122ac9825b902fb7cfdb9cd68bdfb025c0b9e4
push id21898
push useramiyaguchi@mozilla.com
push dateFri, 01 Jul 2016 00:00:45 +0000
bugs1282923
milestone50.0a1
bug 1282923: define a nightly kind in taskgraph This introduces an entry point into the nightly graph. This is a skeleton for further work. `mach taskgraph nightly` will kick off the nightly process with given respository and revision based on a new nightly kind. This will rope in the legacy builds as dependencies for fennec, which will eventually be transitioned into their own kinds (which I hope will make this look cleaner). MozReview-Commit-ID: 1jHllG9jAr5
taskcluster/ci/nightly/kind.yml
taskcluster/mach_commands.py
taskcluster/taskgraph/kind/nightly.py
taskcluster/taskgraph/nightly.py
taskcluster/taskgraph/target_tasks.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly/kind.yml
@@ -0,0 +1,6 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: 'taskgraph.kind.nightly:NightlyTask'
+nightly_path: '.'
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -148,16 +148,62 @@ class MachCommands(MachCommandBase):
         import taskgraph.decision
         try:
             self.setup_logging()
             return taskgraph.decision.taskgraph_decision(options)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
 
+    @SubCommand('taskgraph', 'nightly',
+                description="Run the nightly task")
+    @CommandArgument('--root', '-r',
+                     default='taskcluster/ci',
+                     help="root of the taskgraph definition relative to topsrcdir")
+    @CommandArgument('--base-repository',
+                     required=True,
+                     help='URL for "base" repository to clone')
+    @CommandArgument('--head-repository',
+                     required=True,
+                     help='URL for "head" repository to fetch revision from')
+    @CommandArgument('--head-ref',
+                     required=True,
+                     help='Reference (this is same as rev usually for hg)')
+    @CommandArgument('--head-rev',
+                     required=True,
+                     help='Commit revision to use from head repository')
+    @CommandArgument('--revision-hash',  # NOTE: not sure how to interact with this
+                     required=False,
+                     help='Treeherder revision hash (long revision id) to attach results to')
+    @CommandArgument('--project',
+                     required=True,
+                     help='Project to use for creating task graph. Example: --project=try')
+    @CommandArgument('--pushlog-id',
+                     dest='pushlog_id',
+                     required=True,
+                     default=0)
+    @CommandArgument('--owner',
+                     required=True,
+                     help='email address of who owns this graph')
+    @CommandArgument('--level',
+                     required=True,
+                     help='SCM level of this repository')
+    @CommandArgument('--dry-run', action="store_true", help="dry run of this command")
+    def taskgraph_decision(self, **options):
+        """Run the nightly task: generate the task graph and submit it to
+        TaskCluster. """
+
+        import taskgraph.nightly
+        try:
+            self.setup_logging()
+            return taskgraph.nightly.taskgraph_nightly(options)
+        except Exception:
+            traceback.print_exc()
+            sys.exit(1)
+
     def setup_logging(self, quiet=False, verbose=True):
         """
         Set up Python logging for all loggers, sending results to stderr (so
         that command output can be redirected easily) and adding the typical
         mach timestamp.
         """
         # remove the old terminal handler
         self.log_manager.replace_terminal_handler(None)
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/kind/nightly.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import os
+
+from . import base
+
+
+logger = logging.getLogger(__name__)
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
+
+
+class NightlyTask(base.Task):
+
+    def __init__(self, *args, **kwargs):
+        super(NightlyTask, self).__init__(*args, **kwargs)
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
+        return []
+
+    def get_dependencies(self, taskgraph):
+        return []
+
+    def optimize(self):
+        # no optimizations
+        return False, None
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/nightly.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+from .generator import TaskGraphGenerator
+from .create import create_tasks
+from .parameters import Parameters
+from .target_tasks import get_method
+from .decision import write_artifact
+
+logger = logging.getLogger(__name__)
+ARTIFACTS_DIR = 'artifacts'
+
+
+def taskgraph_nightly(options):
+    """
+    Run the nightly task.  This function implements `mach taskgraph nightly`,
+    and is responsible for
+
+     * processing nightly task command-line options into parameters
+     * running task-graph generation with the nightly graph in mind
+     * generating a set of artifacts to memorialize the graph
+     * calling TaskCluster APIs to create the graph
+    """
+
+    parameters = get_nightly_parameters(options)
+    logger.info(options['dry_run'])
+
+    # create a TaskGraphGenerator instance
+    target_tasks_method = parameters.get('target_tasks_method', 'nightly')
+    target_tasks_method = get_method(target_tasks_method)
+    tgg = TaskGraphGenerator(
+        root_dir=options['root'],
+        parameters=parameters,
+        target_tasks_method=target_tasks_method)
+
+    # write out the parameters used to generate this graph
+    write_artifact('parameters.yml', dict(**parameters))
+
+    # write out the full graph for reference
+    write_artifact('full-task-graph.json', tgg.full_task_graph.to_json())
+
+    # write out the target task set to allow reproducing this as input
+    write_artifact('target-tasks.json', tgg.target_task_set.tasks.keys())
+
+    # write out the optimized task graph to describe what will actually happen,
+    # and the map of labels to taskids
+    write_artifact('task-graph.json', tgg.optimized_task_graph.to_json())
+    write_artifact('label-to-taskid.json', tgg.label_to_taskid)
+
+    if options['dry_run']:
+        logger.info("This command was a dry run, tasks will not be submitted.")
+        return
+
+    # actually create the graph
+    create_tasks(tgg.optimized_task_graph, tgg.label_to_taskid)
+
+
+def get_nightly_parameters(options):
+    """
+    Load parameters from the command-line options for 'taskgraph nightly'.
+    """
+    parameters = {n: options[n] for n in [
+        'base_repository',
+        'head_repository',
+        'head_rev',
+        'head_ref',
+        'message',
+        'project',
+        'pushlog_id',
+        'owner',
+        'level',
+        'target_tasks_method',
+    ] if n in options}
+
+    parameters.update({
+        'target_tasks_method': 'nightly',
+        'optimize_target_tasks': False
+    })
+
+    return Parameters(parameters)
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -41,8 +41,33 @@ def target_tasks_try_option_syntax(full_
 
 @_target_task('all_builds_and_tests')
 def target_tasks_all_builds_and_tests(full_task_graph, parameters):
     """Trivially target all build and test tasks.  This is used for
     branches where we want to build "everyting", but "everything"
     does not include uninteresting things like docker images"""
     return [t.label for t in full_task_graph.tasks.itervalues()
             if t.attributes.get('kind') == 'legacy']
+
+
+@_target_task('nightly')
+def target_tasks_nightly(full_task_graph, parameters):
+    """Generate all build targets that match the nightly kind, or are necessary
+    to include because of constantly changing taskids (legacy)."""
+    return [t.label for t in full_task_graph.tasks.itervalues()
+            if task_matches_nightly(t.attributes)]
+
+
+def task_matches_nightly(attributes):
+    """Determine if a task belongs in the nightly task set based on its
+    attributes."""
+    target_platforms = ['android-api-15', 'linux64']
+
+    if (attributes.get('kind') == 'legacy' and
+        attributes.get('legacy_kind') == 'build' and
+        attributes.get('build_platform') in target_platforms and
+        attributes.get('build_type') == 'opt'):
+        return True
+
+    if attributes.get('kind') == 'nightly':
+        return True
+
+    return False