Bug 1131450 - Implement chunk specific scheduling for tests r=garndt
authorjlal@mozilla.com
Tue, 10 Feb 2015 10:53:02 -0800
changeset 228488 c0d09f8100132bc57b1039616a4ddda25e33a1ca
parent 228487 afcef65f8399055c25043b5eeb61918e25ac5b6b
child 228489 d532034ae6996e9ece8dd513f5640960cb98dc9d
push id28263
push usercbook@mozilla.com
push dateWed, 11 Feb 2015 13:51:51 +0000
treeherdermozilla-central@117e52087be3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgarndt
bugs1131450
milestone38.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1131450 - Implement chunk specific scheduling for tests r=garndt
testing/taskcluster/mach_commands.py
testing/taskcluster/taskcluster_graph/commit_parser.py
testing/taskcluster/tests/test_commit_parser.py
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -310,16 +310,20 @@ class Graph(object):
                 test_parameters['build_url'] = build_url
                 test_parameters['tests_url'] = tests_url
                 test_parameters['total_chunks'] = 1
 
                 if 'chunks' in test:
                     test_parameters['total_chunks'] = test['chunks']
 
                 for chunk in range(1, test_parameters['total_chunks'] + 1):
+                    if 'only_chunks' in test and \
+                        chunk not in test['only_chunks']:
+                        continue;
+
                     test_parameters['chunk'] = chunk
                     test_task = templates.load(test['task'], test_parameters)
                     test_task['taskId'] = slugid()
 
                     if 'requires' not in test_task:
                         test_task['requires'] = []
 
                     test_task['requires'].append(test_parameters['build_slugid'])
--- a/testing/taskcluster/taskcluster_graph/commit_parser.py
+++ b/testing/taskcluster/taskcluster_graph/commit_parser.py
@@ -1,20 +1,22 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import argparse
+import copy
+import functools
+import re
 import shlex
-import argparse
-import functools
-import copy
 from try_test_parser import parse_test_opts
 
-TRY_DELIMITER='try:'
+TRY_DELIMITER = 'try:'
+TEST_CHUNK_SUFFIX = re.compile('(.*)-([0-9]+)$')
 
 # The build type aliases are very cryptic and only used in try flags these are
 # mappings from the single char alias to a longer more recognizable form.
 BUILD_TYPE_ALIASES = {
     'o': 'opt',
     'd': 'debug'
 }
 
@@ -54,19 +56,51 @@ def normalize_test_list(all_tests, job_l
         results = []
         all_entry = tests[0]
         for test in all_tests:
             entry = { 'test': test }
             # If there are platform restrictions copy them across the list.
             if 'platforms' in all_entry:
                 entry['platforms'] = list(all_entry['platforms'])
             results.append(entry)
-        return results
+        return parse_test_chunks(results)
     else:
-        return tests
+        return parse_test_chunks(tests)
+
+def parse_test_chunks(tests):
+    '''
+    Test flags may include parameters to narrow down the number of chunks in a
+    given push. We don't model 1 chunk = 1 job in taskcluster so we must check
+    each test flag to see if it is actually specifying a chunk.
+
+    :param list tests: Result from normalize_test_list
+    :returns: List of jobs
+    '''
+
+    results = []
+    seen_chunks = {}
+    for test in tests:
+        matches = TEST_CHUNK_SUFFIX.match(test['test'])
+
+        if not matches:
+            results.append(test)
+            continue
+
+        name = matches.group(1)
+        chunk = int(matches.group(2))
+
+        if name in seen_chunks:
+            seen_chunks[name].add(chunk)
+        else:
+            seen_chunks[name] = set([chunk])
+            test['test'] = name
+            test['only_chunks'] = seen_chunks[name]
+            results.append(test)
+
+    return results;
 
 def extract_tests_from_platform(test_jobs, build_platform, build_task, tests):
     '''
     Build the list of tests from the current build.
 
     :param dict test_jobs: Entire list of tests (from job_flags.yml).
     :param dict build_platform: Current build platform.
     :param str build_task: Build task path.
@@ -99,17 +133,28 @@ def extract_tests_from_platform(test_job
             # then we must skip this set.
             common_platforms = set(test_entry['platforms']) & set(build_platform['platforms'])
             if not common_platforms:
                 # Tests should not run on this platform...
                 continue
 
         # Add the job to the list and ensure to copy it so we don't accidentally
         # mutate the state of the test job in the future...
-        results.append(copy.deepcopy(test_job))
+        specific_test_job = copy.deepcopy(test_job)
+
+        # Update the task configuration for all tests in the matrix...
+        for build_name in specific_test_job:
+            for test_task_name in specific_test_job[build_name]:
+                test_task = specific_test_job[build_name][test_task_name]
+                # Copy over the chunk restrictions if given...
+                if 'only_chunks' in test_entry:
+                    test_task['only_chunks'] = \
+                            copy.copy(test_entry['only_chunks'])
+
+        results.append(specific_test_job)
 
     return results
 
 '''
 This module exists to deal with parsing the options flags that try uses. We do
 not try to build a graph or anything here but match up build flags to tasks via
 the "jobs" datastructure (see job_flags.yml)
 '''
--- a/testing/taskcluster/tests/test_commit_parser.py
+++ b/testing/taskcluster/tests/test_commit_parser.py
@@ -338,16 +338,72 @@ class TestCommitParser(unittest.TestCase
                 ],
                 'additional-parameters': {}
             }
         ]
 
         result = parse_commit(commit, jobs)
         self.assertEqual(expected, result)
 
+    def test_specific_chunks(self):
+        '''
+        This test covers specifying specific chunks for a given test suite.
+        '''
+        commit = 'try: -b o -p linux -u mochitest-1,mochitest-2 -t none'
+        jobs = {
+            'flags': {
+                'builds': ['linux'],
+                'tests': ['mochitest'],
+            },
+            'builds': {
+                'linux': {
+                    'types': {
+                        'opt': {
+                            'task': 'task/linux',
+                         },
+                        'debug': {
+                            'task': 'task/linux-debug'
+                        }
+                    }
+                },
+            },
+            'tests': {
+                'mochitest': {
+                    'allowed_build_tasks': {
+                        'task/linux': {
+                            'task': 'task/mochitest',
+                            'chunks': 5
+                        },
+                    }
+                }
+            }
+        }
+
+        expected = [
+            {
+                'task': 'task/linux',
+                'dependents': [
+                    {
+                        'allowed_build_tasks': {
+                            'task/linux': {
+                                'task': 'task/mochitest',
+                                'chunks': 5,
+                                'only_chunks': set([1, 2])
+                            },
+                        }
+                    }
+                ],
+                'additional-parameters': {}
+            }
+        ]
+        result = parse_commit(commit, jobs)
+        self.assertEqual(expected, result)
+
+
+
     def test_commit_with_builds_and_tests(self):
         '''
         This test covers the broad case of a commit which has both builds and
         tests without any exclusions or other fancy logic.
         '''
         commit = 'try: -b od -p linux,linux64 -u web-platform-tests -t none'
         jobs = {
             'flags': {