Bug 1269340: add support for try -t option; r=jmaher
authorDustin J. Mitchell <dustin@mozilla.com>
Wed, 18 May 2016 17:55:33 +0000
changeset 340219 2555e6b571f37b4a2315339858af46ac0c0713a2
parent 340218 dd558e06d595766b9e273482af05baee33dcbdc6
child 340220 7092af4d5f2dd5424dd42ce0b23ee3a71c2bd3fd
push id1183
push userraliiev@mozilla.com
push dateMon, 05 Sep 2016 20:01:49 +0000
treeherdermozilla-release@3148731bed45 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1269340
milestone49.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1269340: add support for try -t option; r=jmaher This adds support for the `-t`/`--talos` option, matching such jobs against `talos_try_name`. There are no such tasks just yet. MozReview-Commit-ID: FTEx7Nyyi9Z
taskcluster/docs/attributes.rst
taskcluster/taskgraph/test/test_try_option_syntax.py
taskcluster/taskgraph/try_option_syntax.py
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -58,28 +58,33 @@ sometimes set to match the suite).  Exam
 ``mochitest-devtools-chrome-chunked`` or ``a11y``.
 
 unittest_try_name
 =================
 
 (deprecated) This is the name used to refer to a unit test via try syntax.  It
 may not match either of ``unittest_suite`` or ``unittest_flavor``.
 
+talos_try_name
+==============
+
+(deprecated) This is the name used to refer to a talos job via try syntax.
+
 test_chunk
 ==========
 
 This is the chunk number of a chunked test suite (talos or unittest).  Note
 that this is a string!
 
 legacy_kind
 ===========
 
 (deprecated) The kind of task as created by the legacy kind.  This is valid
-only for the ``legacy`` kind.  One of ``build``, ``unittest,`` ``post_build``,
-or ``job``.
+only for the ``legacy`` kind.  One of ``build``, ``unittest,``, ``talos``,
+``post_build``, or ``job``.
 
 job
 ===
 
 (deprecated) The name of the job (corresponding to a ``-j`` option or the name
 of a post-build job).  This is valid only for the ``legacy`` kind.
 
 post_build
--- a/taskcluster/taskgraph/test/test_try_option_syntax.py
+++ b/taskcluster/taskgraph/test/test_try_option_syntax.py
@@ -15,43 +15,56 @@ from mozunit import main
 empty_graph = TaskGraph({}, Graph(set(), set()))
 
 def unittest_task(n, tp):
     return (n, Task('test', n, {
         'unittest_try_name': n,
         'test_platform': tp,
     }))
 
+def talos_task(n, tp):
+    return (n, Task('test', n, {
+        'talos_try_name': n,
+        'test_platform': tp,
+    }))
+
 tasks = {k: v for k,v in [
     unittest_task('mochitest-browser-chrome', 'linux'),
     unittest_task('mochitest-browser-chrome-e10s', 'linux64'),
     unittest_task('mochitest-chrome', 'linux'),
     unittest_task('mochitest-webgl', 'linux'),
     unittest_task('crashtest-e10s', 'linux'),
     unittest_task('gtest', 'linux64'),
+    talos_task('dromaeojs', 'linux64'),
 ]}
+unittest_tasks = {k: v for k,v in tasks.iteritems()
+                  if 'unittest_try_name' in v.attributes}
+talos_tasks = {k: v for k,v in tasks.iteritems()
+               if 'talos_try_name' in v.attributes}
 graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
 
 
 class TestTryOptionSyntax(unittest.TestCase):
 
     def test_empty_message(self):
         "Given an empty message, it should return an empty value"
         tos = TryOptionSyntax('', empty_graph)
         self.assertEqual(tos.build_types, [])
         self.assertEqual(tos.jobs, [])
         self.assertEqual(tos.unittests, [])
+        self.assertEqual(tos.talos, [])
         self.assertEqual(tos.platforms, [])
 
     def test_message_without_try(self):
         "Given a non-try message, it should return an empty value"
         tos = TryOptionSyntax('Bug 1234: frobnicte the foo', empty_graph)
         self.assertEqual(tos.build_types, [])
         self.assertEqual(tos.jobs, [])
         self.assertEqual(tos.unittests, [])
+        self.assertEqual(tos.talos, [])
         self.assertEqual(tos.platforms, [])
 
     def test_unknown_args(self):
         "unknown arguments are ignored"
         tos = TryOptionSyntax('try: --doubledash -z extra', empty_graph)
         # equilvant to "try:"..
         self.assertEqual(tos.build_types, [])
         self.assertEqual(tos.jobs, None)
@@ -132,33 +145,33 @@ class TestTryOptionSyntax(unittest.TestC
     def test_u_none(self):
         "-u none sets unittests=[]"
         tos = TryOptionSyntax('try: -u none', graph_with_jobs)
         self.assertEqual(sorted(tos.unittests), [])
 
     def test_u_all(self):
         "-u all sets unittests=[..whole list..]"
         tos = TryOptionSyntax('try: -u all', graph_with_jobs)
-        self.assertEqual(sorted(tos.unittests), sorted([{'test': t} for t in tasks]))
+        self.assertEqual(sorted(tos.unittests), sorted([{'test': t} for t in unittest_tasks]))
 
     def test_u_single(self):
         "-u mochitest-webgl sets unittests=[mochitest-webgl]"
         tos = TryOptionSyntax('try: -u mochitest-webgl', graph_with_jobs)
         self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
 
     def test_u_alias(self):
         "-u mochitest-gl sets unittests=[mochitest-webgl]"
         tos = TryOptionSyntax('try: -u mochitest-gl', graph_with_jobs)
         self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
 
     def test_u_multi_alias(self):
         "-u e10s sets unittests=[all e10s unittests]"
         tos = TryOptionSyntax('try: -u e10s', graph_with_jobs)
         self.assertEqual(sorted(tos.unittests), sorted([
-            {'test': t} for t in tasks if 'e10s' in t
+            {'test': t} for t in unittest_tasks if 'e10s' in t
         ]))
 
     def test_u_commas(self):
         "-u mochitest-webgl,gtest sets unittests=both"
         tos = TryOptionSyntax('try: -u mochitest-webgl,gtest', graph_with_jobs)
         self.assertEqual(sorted(tos.unittests), sorted([
             {'test': 'mochitest-webgl'},
             {'test': 'gtest'},
@@ -213,19 +226,36 @@ class TestTryOptionSyntax(unittest.TestC
             {'test': 'gtest', 'platforms': ['linux', 'win32'], 'only_chunks': set('1')},
         ]))
 
     def test_u_chunks_platform_alias(self):
         "-u e10s-1[linux] selects the first chunk of every e10s test on linux"
         tos = TryOptionSyntax('try: -u e10s-1[linux]', graph_with_jobs)
         self.assertEqual(sorted(tos.unittests), sorted([
             {'test': t, 'platforms': ['linux'], 'only_chunks': set('1')}
-            for t in tasks if 'e10s' in t
+            for t in unittest_tasks if 'e10s' in t
         ]))
 
+    def test_t_none(self):
+        "-t none sets talos=[]"
+        tos = TryOptionSyntax('try: -t none', graph_with_jobs)
+        self.assertEqual(sorted(tos.talos), [])
+
+    def test_t_all(self):
+        "-t all sets talos=[..whole list..]"
+        tos = TryOptionSyntax('try: -t all', graph_with_jobs)
+        self.assertEqual(sorted(tos.talos), sorted([{'test': t} for t in talos_tasks]))
+
+    def test_t_single(self):
+        "-t mochitest-webgl sets talos=[mochitest-webgl]"
+        tos = TryOptionSyntax('try: -t mochitest-webgl', graph_with_jobs)
+        self.assertEqual(sorted(tos.talos), sorted([{'test': 'mochitest-webgl'}]))
+
+    # -t shares an implementation with -u, so it's not tested heavily
+
     def test_trigger_tests(self):
         "--trigger-tests 10 sets trigger_tests"
         tos = TryOptionSyntax('try: --trigger-tests 10', empty_graph)
         self.assertEqual(tos.trigger_tests, 10)
 
     def test_interactive(self):
         "--interactive sets interactive"
         tos = TryOptionSyntax('try: --interactive', empty_graph)
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -143,16 +143,17 @@ class TryOptionSyntax(object):
             'platforms': [..platform names..], # to limit to only certain platforms
             'only_chunks': set([..chunk numbers..]), # to limit only to certain chunks
         }
         """
         self.jobs = []
         self.build_types = []
         self.platforms = []
         self.unittests = []
+        self.talos = []
         self.trigger_tests = 0
         self.interactive = False
 
         # shlex used to ensure we split correctly when giving values to argparse.
         parts = shlex.split(self.escape_whitespace_in_brackets(message))
         try_idx = None
         for idx, part in enumerate(parts):
             if part == TRY_DELIMITER:
@@ -162,26 +163,28 @@ class TryOptionSyntax(object):
         if try_idx is None:
             return
 
         # Argument parser based on try flag flags
         parser = argparse.ArgumentParser()
         parser.add_argument('-b', '--build', dest='build_types')
         parser.add_argument('-p', '--platform', nargs='?', dest='platforms', const='all', default='all')
         parser.add_argument('-u', '--unittests', nargs='?', dest='unittests', const='all', default='all')
+        parser.add_argument('-t', '--talos', nargs='?', dest='talos', const='all', default='all')
         parser.add_argument('-i', '--interactive', dest='interactive', action='store_true', default=False)
         parser.add_argument('-j', '--job', dest='jobs', action='append')
         # In order to run test jobs multiple times
         parser.add_argument('--trigger-tests', dest='trigger_tests', type=int, default=1)
         args, _ = parser.parse_known_args(parts[try_idx:])
 
         self.jobs = self.parse_jobs(args.jobs)
         self.build_types = self.parse_build_types(args.build_types)
         self.platforms = self.parse_platforms(args.platforms)
-        self.unittests = self.parse_unittests(args.unittests, full_task_graph)
+        self.unittests = self.parse_test_option("unittest_try_name", args.unittests, full_task_graph)
+        self.talos = self.parse_test_option("talos_try_name", args.talos, full_task_graph)
         self.trigger_tests = args.trigger_tests
         self.interactive = args.interactive
 
     def parse_jobs(self, jobs_arg):
         if not jobs_arg or jobs_arg == ['all']:
             return None
         expanded = []
         for job in jobs_arg:
@@ -202,42 +205,44 @@ class TryOptionSyntax(object):
         results = []
         for build in platform_arg.split(','):
             results.append(build)
             if build in RIDEALONG_BUILDS:
                 results.extend(RIDEALONG_BUILDS[build])
 
         return results
 
-    def parse_unittests(self, unittest_arg, full_task_graph):
+    def parse_test_option(self, attr_name, test_arg, full_task_graph):
         '''
-        Parse a unittest (-u) option, in the context of a full task graph containing
-        available `unittest_try_name` attributes.  There are three cases:
 
-            - unittest_arg is == 'none' (meaning an empty list)
-            - unittest_arg is == 'all' (meaning use the list of jobs for that job type)
-            - unittest_arg is comma string which needs to be parsed
+        Parse a unittest (-u) or talos (-t) option, in the context of a full
+        task graph containing available `unittest_try_name` or `talos_try_name`
+        attributes.  There are three cases:
+
+            - test_arg is == 'none' (meaning an empty list)
+            - test_arg is == 'all' (meaning use the list of jobs for that job type)
+            - test_arg is comma string which needs to be parsed
         '''
 
         # Empty job list case...
-        if unittest_arg is None or unittest_arg == 'none':
+        if test_arg is None or test_arg == 'none':
             return []
 
         all_platforms = set(t.attributes['test_platform']
                             for t in full_task_graph.tasks.itervalues()
                             if 'test_platform' in t.attributes)
 
-        tests = self.parse_test_opts(unittest_arg, all_platforms)
+        tests = self.parse_test_opts(test_arg, all_platforms)
 
         if not tests:
             return []
 
-        all_tests = set(t.attributes['unittest_try_name']
+        all_tests = set(t.attributes[attr_name]
                         for t in full_task_graph.tasks.itervalues()
-                        if 'unittest_try_name' in t.attributes)
+                        if attr_name in t.attributes)
 
         # Special case where tests is 'all' and must be expanded
         if tests[0]['test'] == 'all':
             results = []
             all_entry = tests[0]
             for test in all_tests:
                 entry = {'test': test}
                 # If there are platform restrictions copy them across the list.
@@ -424,48 +429,54 @@ class TryOptionSyntax(object):
                 continue
 
             result += char
 
         return result
 
     def task_matches(self, attributes):
         attr = attributes.get
+
+        def match_test(try_spec, attr_name):
+            if attr('build_type') not in self.build_types:
+                return False
+            if self.platforms is not None:
+                if attr('build_platform') not in self.platforms:
+                    return False
+            if try_spec is not None:
+                # TODO: optimize this search a bit
+                for test in try_spec:
+                    if attr(attr_name) == test['test']:
+                        break
+                else:
+                    return False
+                if 'platforms' in test and attr('test_platform') not in test['platforms']:
+                    return False
+                if 'only_chunks' in test and attr('test_chunk') not in test['only_chunks']:
+                    return False
+                return True
+            return True
+
         if attr('kind') == 'legacy':
             if attr('legacy_kind') in ('build', 'post_build'):
                 if attr('build_type') not in self.build_types:
                     return False
                 if self.platforms is not None:
                     if attr('build_platform') not in self.platforms:
                         return False
                 return True
             elif attr('legacy_kind') == 'job':
                 if self.jobs is not None:
                     if attr('job') not in self.jobs:
                         return False
                 return True
             elif attr('legacy_kind') == 'unittest':
-                if attr('build_type') not in self.build_types:
-                    return False
-                if self.platforms is not None:
-                    if attr('build_platform') not in self.platforms:
-                        return False
-                if self.unittests is not None:
-                    # TODO: optimize this search a bit
-                    for ut in self.unittests:
-                        if attr('unittest_try_name') == ut['test']:
-                            break
-                    else:
-                        return False
-                    if 'platforms' in ut and attr('test_platform') not in ut['platforms']:
-                        return False
-                    if 'only_chunks' in ut and attr('test_chunk') not in ut['only_chunks']:
-                        return False
-                    return True
-                return True
+                return match_test(self.unittests, 'unittest_try_name')
+            elif attr('legacy_kind') == 'talos':
+                return match_test(self.talos, 'talos_try_name')
             return False
         else:
             # TODO: match other kinds
             return False
 
     def __str__(self):
         def none_for_all(list):
             if list is None: