Bug 1103136 - dynamically generate graph scope based on tasks r=lightsofapollo
authorGreg Arndt <garndt@mozilla.com>
Wed, 26 Nov 2014 10:11:39 -0800
changeset 225707 d8aaf4bd4a1c7b4db70d90cf0fc3b7ab32278f89
parent 225706 7e55dc8f75e71bc4d59d23b3e61d20eee20bd3b7
child 225708 03cac982fa86003467c523226d64ba1db845dc2f
push id28174
push userryanvm@gmail.com
push dateMon, 26 Jan 2015 21:23:42 +0000
treeherdermozilla-central@c3a90afa2dee [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerslightsofapollo
bugs1103136
milestone38.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1103136 - dynamically generate graph scope based on tasks r=lightsofapollo
testing/taskcluster/mach_commands.py
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -25,22 +25,23 @@ from mach.decorators import (
 from taskcluster_graph.commit_parser import parse_commit
 from taskcluster_graph.slugid import slugid
 from taskcluster_graph.from_now import json_time_from_now, current_json_time
 
 import taskcluster_graph.build_task
 
 ROOT = os.path.dirname(os.path.realpath(__file__))
 DOCKER_ROOT = os.path.join(ROOT, '..', 'docker')
-LOCAL_WORKER_TYPES = ['b2gtest', 'b2gbuild']
 
 # XXX: If/when we have the taskcluster queue use construct url instead
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 REGISTRY = open(os.path.join(DOCKER_ROOT, 'REGISTRY')).read().strip()
 
+DEFINE_TASK = 'queue:define-task:aws-provisioner/{}'
+
 def get_hg_url():
     ''' Determine the url for the mercurial repository'''
     try:
         url = subprocess.check_output(
             ['hg', 'path', 'default'],
             stderr=subprocess.PIPE
         )
     except subprocess.CalledProcessError:
@@ -123,45 +124,21 @@ class TryGraph(object):
             'revision': revision,
             'owner': owner,
             'from_now': json_time_from_now,
             'now': datetime.datetime.now().isoformat()
         }
 
         # Task graph we are generating for taskcluster...
         graph = {
-            'tasks': []
+            'tasks': [],
+            'scopes': []
         }
 
         if ci is False:
-            # XXX: We need to figure out a less ugly way to store these for
-            # local testing.
-            graph['scopes'] = [
-                "docker-worker:cache:sources-mozilla-central",
-                "docker-worker:cache:sources-gaia",
-                "docker-worker:cache:build-b2g-desktop-objects",
-                "docker-worker:cache:build-mulet-linux-objects",
-                "docker-worker:cache:tooltool-cache",
-                "docker-worker:cache:build-emulator-objects"
-            ]
-
-            # XXX: This is a hack figure out how to do this correctly or sanely
-            # at least so we don't need to keep track of all worker types in
-            # existence.
-            for worker_type in LOCAL_WORKER_TYPES:
-                graph['scopes'].append(
-                    'queue:define-task:{}/{}'.format('aws-provisioner',
-                        worker_type)
-                )
-
-                graph['scopes'].append(
-                    'queue:create-task:{}/{}'.format('aws-provisioner',
-                        worker_type)
-                )
-
             graph['metadata'] = {
                 'source': 'http://todo.com/what/goes/here',
                 'owner': owner,
                 # TODO: Add full mach commands to this example?
                 'description': 'Try task graph generated via ./mach trygraph',
                 'name': 'trygraph local'
             }
 
@@ -179,16 +156,21 @@ class TryGraph(object):
                 build_task['task']['extra']['locations']['tests']
             )
 
             build_url = ARTIFACT_URL.format(
                 build_parameters['build_slugid'],
                 build_task['task']['extra']['locations']['build']
             )
 
+            define_task = DEFINE_TASK.format(build_task['task']['workerType'])
+
+            graph['scopes'].append(define_task)
+            graph['scopes'].extend(build_task['task'].get('scopes', []))
+
             for test in build['dependents']:
                 test = test['allowed_build_tasks'][build['task']]
                 test_parameters = copy.copy(build_parameters)
                 test_parameters['build_url'] = build_url
                 test_parameters['tests_url'] = tests_url
                 test_parameters['total_chunks'] = 1
 
                 if 'chunks' in test:
@@ -198,18 +180,28 @@ class TryGraph(object):
                     test_parameters['chunk'] = chunk
                     test_task = import_yaml(test['task'], test_parameters)
                     test_task['taskId'] = slugid()
 
                     if 'requires' not in test_task:
                         test_task['requires'] = []
 
                     test_task['requires'].append(test_parameters['build_slugid'])
+
                     graph['tasks'].append(test_task)
 
+                    define_task = DEFINE_TASK.format(
+                        test_task['task']['workerType']
+                    )
+
+                    graph['scopes'].append(define_task)
+                    graph['scopes'].extend(test_task['task'].get('scopes', []))
+
+        graph['scopes'] = list(set(graph['scopes']))
+
         print(json.dumps(graph, indent=4))
 
 @CommandProvider
 class CIBuild(object):
     @Command('taskcluster-build', category='ci',
         description="Create taskcluster try server build task")
     @CommandArgument('--revision',
         help='revision in gecko to use in sub tasks')