Bug 1258497: Add a try syntax parser draft
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 29 Apr 2016 18:16:00 +0000
changeset 358383 39d0e3d8071e9286bf13446974e9992928f67eb7
parent 358382 e437ab047f993f04e0f596914475bedeaa087daf
child 358384 689b83866aad24d2d5827f0c58cd9ad2d391c0fe
push id16995
push userdmitchell@mozilla.com
push dateMon, 02 May 2016 18:47:33 +0000
bugs1258497
milestone49.0a1
Bug 1258497: Add a try syntax parser This is a brand new parser for try syntax, based on the previous in-tree parser. MozReview-Commit-ID: HDocso8Inl3
taskcluster/mach_commands.py
taskcluster/taskgraph/kind/legacy.py
taskcluster/taskgraph/test/test_try_option_syntax.py
taskcluster/taskgraph/try_option_syntax.py
testing/taskcluster/tasks/branches/base_jobs.yml
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -129,16 +129,19 @@ class MachCommands(MachCommandBase):
     @CommandArgument('--message',
         help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
     @CommandArgument('--revision-hash',
             required=False,
             help='Treeherder revision hash to attach results to')
     @CommandArgument('--project',
         required=True,
         help='Project to use for creating task graph. Example: --project=try')
+    @CommandArgument('--target-tasks-method',
+        required=False,
+        help='Method to use to determine the target task (e.g., `try_option_syntax`)')
     @CommandArgument('--pushlog-id',
         dest='pushlog_id',
         required=False,
         default=0)
     @CommandArgument('--owner',
         required=True,
         help='email address of who owns this graph')
     @CommandArgument('--level',
@@ -219,16 +222,17 @@ class MachCommands(MachCommandBase):
             'head_rev',
             'head_ref',
             'revision_hash',
             'message',
             'project',
             'pushlog_id',
             'owner',
             'level',
+            'target_tasks_method',
         ]
         for n in option_names:
             if options[n]:
                 parameters[n] = options[n]
 
         return parameters
 
     ##
@@ -246,18 +250,21 @@ class MachCommands(MachCommandBase):
             return target_tasks
 
     def target_tasks_from_parameters(self, full_task_graph, parameters):
         """Get the target task set from parameters['target_tasks'].  This is
         useful for re-running a decision task with the same target set as in an
         earlier run, by copying `target_tasks.json` into `parameters.yml`."""
         return parameters['target_tasks']
 
-    def target_tasks_try(self, full_task_graph, parameters):
-        pass
+    def target_tasks_try_option_syntax(self, full_task_graph, parameters):
+        from taskgraph.try_option_syntax import TryOptionSyntax
+        options = TryOptionSyntax(parameters['message'], full_task_graph)
+        return [t.label for t in full_task_graph.tasks.itervalues()
+                if options.task_matches(t.attributes)]
 
     ##
     # Utilities
 
     def get_taskgraph_generator(self, options, parameters):
         import taskgraph.generator
         if options['optimize']:
             optimization_finder = None  # XXX function that searches index
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -33,29 +33,31 @@ from taskcluster_graph.from_now import (
     json_time_from_now,
     current_json_time,
 )
 from taskcluster_graph.templates import Templates
 import taskcluster_graph.build_task
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
-DEFAULT_TRY = 'try: -b do -p all -u all'
+DEFAULT_TRY = 'try: -b do -p all -u all -t all'
 DEFAULT_JOB_PATH = os.path.join(
     'tasks', 'branches', 'base_jobs.yml'
 )
 
 
 class LegacyKind(base.Kind):
 
     def load_tasks(self, params):
         root = os.path.abspath(os.path.join(self.path, self.config['legacy_path']))
 
         project = params['project']
-        message = params.get('message', '') if project == 'try' else DEFAULT_TRY
+        # NOTE: message is ignored here; we always use DEFAULT_TRY, then filter the
+        # resulting task graph later
+        message = DEFAULT_TRY
 
         templates = Templates(root)
 
         job_path = os.path.join(root, 'tasks', 'branches', project, 'job_flags.yml')
         job_path = job_path if os.path.exists(job_path) else \
             os.path.join(root, DEFAULT_JOB_PATH)
 
         jobs = templates.load(job_path, {})
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_try_option_syntax.py
@@ -0,0 +1,213 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import unicode_literals
+
+import sys
+import unittest
+
+from taskgraph.try_option_syntax import TryOptionSyntax
+from taskgraph.graph import Graph
+from taskgraph.types import TaskGraph, Task
+
+from mozunit import main
+
+# an empty graph, for things that don't look at it
+empty_graph = TaskGraph({}, Graph(set(), set()))
+
+def unittest_task(n):
+    return (n, Task('test', n, {'unittest_try_name': n}))
+
+tasks = {k: v for k,v in [
+    unittest_task('mochitest-browser-chrome'),
+    unittest_task('mochitest-browser-chrome-e10s'),
+    unittest_task('mochitest-chrome'),
+    unittest_task('mochitest-webgl'),
+    unittest_task('crashtest-e10s'),
+    unittest_task('gtest'),
+]}
+graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
+
+
+class TestTryOptionSyntax(unittest.TestCase):
+
+    def test_empty_message(self):
+        "Given an empty message, it should return an empty value"
+        tos = TryOptionSyntax('', empty_graph)
+        self.assertEqual(tos.build_types, [])
+        self.assertEqual(tos.jobs, [])
+        self.assertEqual(tos.unittests, [])
+        self.assertEqual(tos.platforms, [])
+
+    def test_message_without_try(self):
+        "Given a non-try message, it should return an empty value"
+        tos = TryOptionSyntax('Bug 1234: frobnicte the foo', empty_graph)
+        self.assertEqual(tos.build_types, [])
+        self.assertEqual(tos.jobs, [])
+        self.assertEqual(tos.unittests, [])
+        self.assertEqual(tos.platforms, [])
+
+    def test_unknown_args(self):
+        "unknown arguments are ignored"
+        tos = TryOptionSyntax('try: --doubledash -z extra', empty_graph)
+        # equilvant to "try:"..
+        self.assertEqual(tos.build_types, [])
+        self.assertEqual(tos.jobs, None)
+
+    def test_b_do(self):
+        "-b do should produce both build_types"
+        tos = TryOptionSyntax('try: -b do', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['debug', 'opt'])
+
+    def test_b_d(self):
+        "-b d should produce build_types=['debug']"
+        tos = TryOptionSyntax('try: -b d', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['debug'])
+
+    def test_b_o(self):
+        "-b o should produce build_types=['opt']"
+        tos = TryOptionSyntax('try: -b o', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['opt'])
+
+    def test_build_o(self):
+        "--build o should produce build_types=['opt']"
+        tos = TryOptionSyntax('try: --build o', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['opt'])
+
+    def test_b_dx(self):
+        "-b dx should produce build_types=['debug'], silently ignoring the x"
+        tos = TryOptionSyntax('try: -b dx', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['debug'])
+
+    def test_j_job(self):
+        "-j somejob sets jobs=['somejob']"
+        tos = TryOptionSyntax('try: -j somejob', empty_graph)
+        self.assertEqual(sorted(tos.jobs), ['somejob'])
+
+    def test_j_jobs(self):
+        "-j job1,job2 sets jobs=['job1', 'job2']"
+        tos = TryOptionSyntax('try: -j job1,job2', empty_graph)
+        self.assertEqual(sorted(tos.jobs), ['job1', 'job2'])
+
+    def test_j_all(self):
+        "-j all sets jobs=None"
+        tos = TryOptionSyntax('try: -j all', empty_graph)
+        self.assertEqual(tos.jobs, None)
+
+    def test_j_twice(self):
+        "-j job1 -j job2 sets jobs=job1, job2"
+        tos = TryOptionSyntax('try: -j job1 -j job2', empty_graph)
+        self.assertEqual(sorted(tos.jobs), sorted(['job1', 'job2']))
+
+    def test_p_all(self):
+        "-p all sets platforms=None"
+        tos = TryOptionSyntax('try: -p all', empty_graph)
+        self.assertEqual(tos.platforms, None)
+
+    def test_p_linux(self):
+        "-p linux sets platforms=['linux']"
+        tos = TryOptionSyntax('try: -p linux', empty_graph)
+        self.assertEqual(tos.platforms, ['linux'])
+
+    def test_p_linux_win32(self):
+        "-p linux,win32 sets platforms=['linux', 'win32']"
+        tos = TryOptionSyntax('try: -p linux,win32', empty_graph)
+        self.assertEqual(sorted(tos.platforms), ['linux', 'win32'])
+
+    def test_p_expands_ridealongs(self):
+        "-p linux,linux64 includes the RIDEALONG_BUILDS"
+        tos = TryOptionSyntax('try: -p linux,linux64', empty_graph)
+        self.assertEqual(sorted(tos.platforms), [
+            'linux',
+            'linux64',
+            'sm-arm-sim',
+            'sm-compacting',
+            'sm-generational',
+            'sm-plain',
+            'sm-rootanalysis',
+            'sm-warnaserr',
+        ])
+
+    def test_u_none(self):
+        "-u none sets unittests=[]"
+        tos = TryOptionSyntax('try: -u none', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), [])
+
+    def test_u_all(self):
+        "-u all sets unittests=[..whole list..]"
+        tos = TryOptionSyntax('try: -u all', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([{'test': t} for t in tasks]))
+
+    def test_u_single(self):
+        "-u mochitest-webgl sets unittests=[mochitest-webgl]"
+        tos = TryOptionSyntax('try: -u mochitest-webgl', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
+
+    def test_u_alias(self):
+        "-u mochitest-gl sets unittests=[mochitest-webgl]"
+        tos = TryOptionSyntax('try: -u mochitest-gl', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
+
+    def test_u_multi_alias(self):
+        "-u e10s sets unittests=[all e10s unittests]"
+        tos = TryOptionSyntax('try: -u e10s', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': t} for t in tasks if 'e10s' in t
+        ]))
+
+    def test_u_commas(self):
+        "-u mochitest-webgl,gtest sets unittests=both"
+        tos = TryOptionSyntax('try: -u mochitest-webgl,gtest', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'mochitest-webgl'},
+            {'test': 'gtest'},
+        ]))
+
+    def test_u_chunks(self):
+        "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest"
+        tos = TryOptionSyntax('try: -u gtest-3,gtest-4', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'only_chunks': set('34')},
+        ]))
+
+    def test_u_platform(self):
+        "-u gtest[linux] selects the linux platform for gtest"
+        tos = TryOptionSyntax('try: -u gtest[linux]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux']},
+        ]))
+
+    def test_u_platforms(self):
+        "-u gtest[linux,win32] selects the linux and win32 platforms for gtest"
+        tos = TryOptionSyntax('try: -u gtest[linux,win32]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux', 'win32']},
+        ]))
+
+    def test_u_chunks_platforms(self):
+        "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest"
+        tos = TryOptionSyntax('try: -u gtest-1[linux,win32]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux', 'win32'], 'only_chunks': set('1')},
+        ]))
+
+    def test_u_chunks_platform_alias(self):
+        "-u e10s-1[linux] selects the first chunk of every e10s test on linux"
+        tos = TryOptionSyntax('try: -u e10s-1[linux]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': t, 'platforms': ['linux'], 'only_chunks': set('1')}
+            for t in tasks if 'e10s' in t
+        ]))
+
+    def test_trigger_tests(self):
+        "--trigger-tests 10 sets trigger_tests"
+        tos = TryOptionSyntax('try: --trigger-tests 10', empty_graph)
+        self.assertEqual(tos.trigger_tests, 10)
+
+    def test_interactive(self):
+        "--interactive sets interactive"
+        tos = TryOptionSyntax('try: --interactive', empty_graph)
+        self.assertEqual(tos.interactive, True)
+
+if __name__ == '__main__':
+    main()
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -0,0 +1,415 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import unicode_literals
+
+import argparse
+import copy
+import re
+import shlex
+
+TRY_DELIMITER = 'try:'
+
+# The build type aliases are very cryptic and only used in try flags these are
+# mappings from the single char alias to a longer more recognizable form.
+BUILD_TYPE_ALIASES = {
+    'o': 'opt',
+    'd': 'debug'
+}
+
+# mapping from shortcut name (usable with -u) to a regexp matching a set of
+# test suites
+UNITTEST_ALIASES = {
+    'cppunit': re.compile(r'^cppunit.*$'),
+    'crashtest': re.compile(r'^crashtest.*$'),
+    'crashtest-e10s': re.compile(r'^crashtest-e10s.*$'),
+    'e10s': re.compile(r'^.*e10s.*$'),
+    'firefox-ui-functional': re.compile(r'^firefox-ui-functional.*$'),
+    'firefox-ui-functional-e10s': re.compile(r'^firefox-ui-functional-e10s.*$'),
+    'gaia-js-integration': re.compile(r'^.*gaia-js-integration.*$'),
+    'gtest': re.compile(r'^gtest.*$'),
+    'jittest': re.compile(r'^jittest.*$'),
+    'jittests': re.compile(r'^jittest.*$'),
+    'jsreftest': re.compile(r'^jsreftest.*$'),
+    'jsreftest-e10s': re.compile(r'^jsreftest-e10s.*$'),
+    'luciddream': re.compile(r'^luciddream.*$'),
+    'marionette': re.compile(r'^marionette.*$'),
+    'marionette-e10s': re.compile(r'^marionette-e10s.*$'),
+    'mochitest': re.compile(r'^mochitest.*$'),
+    'mochitests': re.compile(r'^mochitest.*$'),
+    'mochitest-e10s': re.compile(r'^mochitest-e10s.*$'),
+    'mochitests-e10s': re.compile(r'^mochitest-e10s.*$'),
+    'mochitest-debug': re.compile(r'^mochitest-debug-.*$'),
+    'mochitest-a11y': re.compile(r'^.*mochitest-a11y.*$'),
+    'mochitest-bc': re.compile(r'^mochitest-browser-chrome.*$'),
+    'mochitest-bc-e10s': re.compile(r'^mochitest-browser-chrome-e10s.*$'),
+    'mochitest-browser-chrome': re.compile(r'^mochitest-browser-chrome.*$'),
+    'mochitest-browser-chrome-e10s': re.compile(r'^mochitest-browser-chrome-e10s.*$'),
+    'mochitest-chrome': re.compile(r'^.*mochitest-chrome.*$'),
+    'mochitest-dt': re.compile(r'^mochitest-devtools-chrome.*$'),
+    'mochitest-dt-e10s': re.compile(r'^mochitest-devtools-chrome-e10s.*$'),
+    'mochitest-gl': re.compile(r'^mochitest-webgl.*$'),
+    'mochitest-gl-e10s': re.compile(r'^mochitest-webgl-e10s.*$'),
+    'mochitest-jetpack': re.compile(r'^mochitest-jetpack.*$'),
+    'mochitest-media': re.compile(r'^mochitest-media.*$'),
+    'mochitest-media-e10s': re.compile(r'^mochitest-media-e10s.*$'),
+    'mochitest-vg': re.compile(r'^mochitest-valgrind.*$'),
+    'reftest': re.compile(r'^(plain-)?reftest.*$'),
+    'reftest-no-accel': re.compile(r'^(plain-)?reftest-no-accel.*$'),
+    'reftests': re.compile(r'^(plain-)?reftest.*$'),
+    'reftests-e10s': re.compile(r'^(plain-)?reftest-e10s.*$'),
+    'robocop': re.compile(r'^robocop.*$'),
+    'web-platform-test': re.compile(r'^web-platform-tests.*$'),
+    'web-platform-tests': re.compile(r'^web-platform-tests.*$'),
+    'web-platform-tests-e10s': re.compile(r'^web-platform-tests-e10s.*$'),
+    'web-platform-tests-reftests': re.compile(r'^web-platform-tests-reftests.*$'),
+    'web-platform-tests-reftests-e10s': re.compile(r'^web-platform-tests-reftests-e10s.*$'),
+    'xpcshell': re.compile(r'^xpcshell.*$'),
+}
+
+# We have a few platforms for which we want to do some "extra" builds, or at
+# least build-ish things.  Sort of.  Anyway, these other things are implemented
+# as different "platforms".
+RIDEALONG_BUILDS = {
+    'linux64': [
+        'sm-plain',
+        'sm-arm-sim',
+        'sm-compacting',
+        'sm-generational',
+        'sm-rootanalysis',
+        'sm-warnaserr',
+    ],
+}
+
+TEST_CHUNK_SUFFIX = re.compile('(.*)-([0-9]+)$')
+
+class TryOptionSyntax(object):
+    
+    def __init__(self, message, full_task_graph):
+        """
+        Parse a "try syntax" formatted commit message.  This is the old "-b do -p
+        win32 -u all" format.  Aliases are applied to map short names to full
+        names.
+
+        The resulting object has attributes:
+
+        - build_types: a list containing zero or more of 'opt' and 'debug'
+        - platforms: a list of selected platform names, or None for all
+        - unittests: a list of tests, of the form given below, or None for all
+        - jobs: a list of requested job names, or None for all
+        - trigger_tests: the number of times tests should be triggered
+        - interactive; true if --interactive
+
+        Note that -t is currently completely ignored.
+
+        The unittests and talos lists contain dictionaries of the form:
+
+        { 
+            'test': '<suite name>',
+            'platforms': [..platform names..], # to limit to only certain platforms
+            'only_chunks': set([..chunk numbers..]), # to limit only to certain chunks
+        }
+        """
+        # shlex used to ensure we split correctly when giving values to argparse.
+        parts = shlex.split(self.escape_whitespace_in_brackets(message))
+        try_idx = None
+        for idx, part in enumerate(parts):
+            if part == TRY_DELIMITER:
+                try_idx = idx
+                break
+
+        if try_idx is None:
+            self.make_empty()
+            return
+
+        # Argument parser based on try flag flags
+        parser = argparse.ArgumentParser()
+        parser.add_argument('-b', '--build', dest='build_types')
+        parser.add_argument('-p', '--platform', nargs='?', dest='platforms', const='all', default='all')
+        parser.add_argument('-u', '--unittests', nargs='?', dest='unittests', const='all', default='all')
+        parser.add_argument('-i', '--interactive', dest='interactive', action='store_true', default=False)
+        parser.add_argument('-j', '--job', dest='jobs', action='append')
+        # In order to run test jobs multiple times
+        parser.add_argument('--trigger-tests', dest='trigger_tests', type=int, default=1)
+        args, _ = parser.parse_known_args(parts[try_idx:])
+
+        self.jobs = self.parse_jobs(args.jobs)
+        self.build_types = self.parse_build_types(args.build_types)
+        self.platforms = self.parse_platforms(args.platforms)
+        self.unittests = self.parse_unittests(args.unittests, full_task_graph)
+        self.trigger_tests = args.trigger_tests
+        self.interactive = args.interactive
+
+    def make_empty(self):
+        self.jobs = []
+        self.build_types = []
+        self.platforms = []
+        self.unittests = []
+        self.trigger_tests = 0
+        self.interactive = False
+
+    def parse_jobs(self, jobs_arg):
+        if not jobs_arg or jobs_arg == ['all']:
+            return None
+        expanded = []
+        for job in jobs_arg:
+            expanded.extend(j.strip() for j in job.split(','))
+        return expanded
+
+    def parse_build_types(self, build_types_arg):
+        if build_types_arg is None:
+            build_types_arg = []
+        build_types = filter(None, [ BUILD_TYPE_ALIASES.get(build_type) for
+                build_type in build_types_arg ])
+        return build_types
+
+    def parse_platforms(self, platform_arg):
+        if platform_arg == 'all':
+            return None
+
+        results = []
+        for build in platform_arg.split(','):
+            results.append(build)
+            if build in RIDEALONG_BUILDS:
+                results.extend(RIDEALONG_BUILDS[build])
+
+        return results
+
+    def parse_unittests(self, unittest_arg, full_task_graph):
+        '''
+        Parse a unittest (-u) option, in the context of a full task graph containing
+        available `unittest_try_name` attributes.  There are three cases:
+
+            - unittest_arg is == 'none' (meaning an empty list)
+            - unittest_arg is == 'all' (meaning use the list of jobs for that job type)
+            - unittest_arg is comma string which needs to be parsed
+        '''
+
+        # Empty job list case...
+        if unittest_arg is None or unittest_arg == 'none':
+            return []
+
+        tests = self.parse_test_opts(unittest_arg)
+
+        if not tests:
+            return []
+
+        all_tests = set([t.attributes['unittest_try_name']
+                         for t in full_task_graph.tasks.itervalues()
+                         if 'unittest_try_name' in t.attributes])
+
+        # Special case where tests is 'all' and must be expanded
+        if tests[0]['test'] == 'all':
+            results = []
+            all_entry = tests[0]
+            for test in all_tests:
+                entry = { 'test': test }
+                # If there are platform restrictions copy them across the list.
+                if 'platforms' in all_entry:
+                    entry['platforms'] = list(all_entry['platforms'])
+                results.append(entry)
+            return self.parse_test_chunks(all_tests, results)
+        else:
+            return self.parse_test_chunks(all_tests, tests)
+
+    def parse_test_opts(self, input_str):
+        '''
+        Test argument parsing is surprisingly complicated with the "restrictions"
+        logic this function is responsible for parsing this out into a easier to
+        work with structure like { test: '..', platforms: ['..'] }
+        '''
+
+        # Final results which we will return.
+        tests = []
+
+        cur_test = {}
+        token = ''
+        in_platforms = False
+
+        def add_test(value):
+            cur_test['test'] = value.strip()
+            tests.insert(0, cur_test)
+
+        def add_platform(value):
+            # Ensure platforms exists...
+            cur_test['platforms'] = cur_test.get('platforms', [])
+            cur_test['platforms'].insert(0, value.strip())
+
+        # This might be somewhat confusing but we parse the string _backwards_ so
+        # there is no ambiguity over what state we are in.
+        for char in reversed(input_str):
+
+            # , indicates exiting a state
+            if char == ',':
+
+                # Exit a particular platform.
+                if in_platforms:
+                    add_platform(token)
+
+                # Exit a particular test.
+                else:
+                    add_test(token)
+                    cur_test = {}
+
+                # Token must always be reset after we exit a state
+                token = ''
+            elif char == '[':
+                # Exiting platform state entering test state.
+                add_platform(token)
+                token = ''
+                in_platforms = False
+            elif char == ']':
+                # Entering platform state.
+                in_platforms = True
+            else:
+                # Accumulator.
+                token = char + token
+
+        # Handle any left over tokens.
+        if token:
+            add_test(token)
+
+        return tests
+
+    def handle_alias(self, test, all_tests):
+        '''
+        Expand a test if its name refers to an alias, returning a list of test
+        dictionaries cloned from the first (to maintain any metadata).
+        '''
+        if test['test'] not in UNITTEST_ALIASES:
+            return [test]
+
+        alias = UNITTEST_ALIASES[test['test']]
+        def mktest(name):
+            newtest = copy.deepcopy(test)
+            newtest['test'] = name
+            return newtest
+
+        def exprmatch(alias):
+            return [t for t in all_tests if alias.match(t)]
+
+        return [mktest(t) for t in exprmatch(alias)]
+
+
+    def parse_test_chunks(self, all_tests, tests):
+        '''
+        Test flags may include parameters to narrow down the number of chunks in a
+        given push. We don't model 1 chunk = 1 job in taskcluster so we must check
+        each test flag to see if it is actually specifying a chunk.
+        '''
+        results = []
+        seen_chunks = {}
+        for test in tests:
+            matches = TEST_CHUNK_SUFFIX.match(test['test'])
+
+            if not matches:
+                results.extend(self.handle_alias(test, all_tests))
+                continue
+
+            name = matches.group(1)
+            chunk = matches.group(2)
+            test['test'] = name
+
+            for test in self.handle_alias(test, all_tests):
+                name = test['test']
+                if name in seen_chunks:
+                    seen_chunks[name].add(chunk)
+                else:
+                    seen_chunks[name] = set([chunk])
+                    test['test'] = name
+                    test['only_chunks'] = seen_chunks[name]
+                    results.append(test)
+
+        # uniquify the results over the test names
+        results = {test['test']: test for test in results}.values()
+        return results
+
+    def find_all_attribute_suffixes(self, graph, prefix):
+        rv = set()
+        for t in graph.tasks.itervalues():
+            for a in t.attributes:
+                if a.startswith(prefix):
+                    rv.add(a[len(prefix):])
+        return sorted(rv)
+
+    def escape_whitespace_in_brackets(self, input_str):
+        '''
+        In tests you may restrict them by platform [] inside of the brackets
+        whitespace may occur this is typically invalid shell syntax so we escape it
+        with backslash sequences    .
+        '''
+        result = ""
+        in_brackets = False
+        for char in input_str:
+            if char == '[':
+                in_brackets = True
+                result += char
+                continue
+
+            if char == ']':
+                in_brackets = False
+                result += char
+                continue
+
+            if char == ' ' and in_brackets:
+                result += '\ '
+                continue
+
+            result += char
+
+        return result
+
+    def task_matches(self, attributes):
+        attr = attributes.get
+        if attr('kind') == 'legacy':
+            if attr('legacy_kind') in ('build', 'post_build'):
+                if attr('build_type') not in self.build_types:
+                    return False
+                if self.platforms is not None:
+                    if attr('build_platform') not in self.platforms:
+                        return False
+                return True
+            elif attr('legacy_kind') == 'job':
+                if self.jobs is not None:
+                    if attr('job') not in self.jobs:
+                        return False
+                return True
+            elif attr('legacy_kind') == 'unittest':
+                if attr('build_type') not in self.build_types:
+                    return False
+                if self.platforms is not None:
+                    if attr('build_platform') not in self.platforms:
+                        return False
+                if self.unittests is not None:
+                    # TODO: optimize this search a bit
+                    for ut in self.unittests:
+                        if attr('unittest_try_name') == ut['test']:
+                            break
+                    else:
+                        return False
+                    if 'platforms' in ut and attr('test_platform') not in ut['platforms']:
+                        return False
+                    if 'only_chunks' in ut and attr('test_chunk') not in ut['only_chunks']:
+                        return False
+                    return True
+                return True
+            return False
+        else:
+            # TODO: match other kinds
+            return False
+
+    def __str__(self):
+        def none_for_all(list):
+            if list is None:
+                return '<all>'
+            return ', '.join(str (e) for e in list)
+
+        return "\n".join([
+            "build_types: " + ", ".join(self.build_types),
+            "platforms: " + none_for_all(self.platforms),
+            "unittests: " + none_for_all(self.unittests),
+            "jobs: " + none_for_all(self.jobs),
+            "trigger_tests: " + str(self.trigger_tests),
+            "interactive: " + str(self.interactive),
+        ])
+
--- a/testing/taskcluster/tasks/branches/base_jobs.yml
+++ b/testing/taskcluster/tasks/branches/base_jobs.yml
@@ -42,17 +42,17 @@ builds:
     types:
       opt:
         task: tasks/builds/opt_linux32.yml
       debug:
         task: tasks/builds/dbg_linux32.yml
   linux64:
     platforms:
       - Linux64
-    extra-builds:
+    extra-builds:  # see RIDEALONG_BUILDS in `mach taskgraph`
       - sm-plain
       - sm-arm-sim
       - sm-compacting
       - sm-generational
       - sm-rootanalysis
       - sm-warnaserr
     types:
       opt: