Bug 1453056 - test-verify should have the ability to run in chunks depending on the incoming tests. r=gbrown
authorJoel Maher <jmaher@mozilla.com>
Tue, 24 Apr 2018 10:20:11 -0400
changeset 415300 c26bbed87ab0a3977bff7b9f5da0a6f537db7362
parent 415299 0c5a4939300c48255f926c49c56bfb66ed17184a
child 415301 88b650567a7f57d782abea7cc2936550ea362e89
push id102539
push userjmaher@mozilla.com
push dateTue, 24 Apr 2018 14:24:24 +0000
treeherdermozilla-inbound@88b650567a7f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1453056
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1453056 - test-verify should have the ability to run in chunks depending on the incoming tests. r=gbrown
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/testing/per_test_base.py
testing/mozharness/scripts/android_emulator_unittest.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/web_platform_tests.py
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -18,16 +18,18 @@ for example - use `all_tests.py` instead
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import resolve_keyed_by, OptimizationSchema
 from taskgraph.util.treeherder import split_symbol, join_symbol
 from taskgraph.util.platforms import platform_family
+from taskgraph import files_changed
+from mozpack.path import match as mozpackmatch
 from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
     Schema,
 )
 from taskgraph.util.taskcluster import get_artifact_path
 from mozbuild.schedules import INCLUSIVE_COMPONENTS
 
@@ -35,16 +37,17 @@ from voluptuous import (
     Any,
     Optional,
     Required,
     Exclusive,
 )
 
 import copy
 import logging
+import math
 
 # default worker types keyed by instance-size
 LINUX_WORKER_TYPES = {
     'large': 'aws-provisioner-v1/gecko-t-linux-large',
     'xlarge': 'aws-provisioner-v1/gecko-t-linux-xlarge',
     'default': 'aws-provisioner-v1/gecko-t-linux-large',
 }
 
@@ -668,17 +671,17 @@ def handle_suite_category(config, tests)
             suite = flavor = test['suite']
 
         test.setdefault('attributes', {})
         test['attributes']['unittest_suite'] = suite
         test['attributes']['unittest_flavor'] = flavor
 
         script = test['mozharness']['script']
         category_arg = None
-        if suite == 'test-verify' or suite == 'test-coverage':
+        if suite.startswith('test-verify') or suite.startswith('test-coverage'):
             pass
         elif script == 'android_emulator_unittest.py':
             category_arg = '--test-suite'
         elif script == 'desktop_unittest.py':
             category_arg = '--{}-suite'.format(suite)
 
         if category_arg:
             test['mozharness'].setdefault('extra-options', [])
@@ -768,16 +771,28 @@ def split_e10s(config, tests):
 
 
 @transforms.add
 def split_chunks(config, tests):
     """Based on the 'chunks' key, split tests up into chunks by duplicating
     them and assigning 'this-chunk' appropriately and updating the treeherder
     symbol."""
     for test in tests:
+        if test['suite'].startswith('test-verify'):
+            test['chunks'] = perfile_number_of_chunks(config, test['test-name'])
+            if test['chunks'] == 0:
+                continue
+            # limit the number of chunks we run for test-verify mode because
+            # test-verify is comprehensive and takes a lot of time, if we have
+            # >30 tests changed, this is probably an import of external tests,
+            # or a patch renaming/moving files in bulk
+            maximum_number_verify_chunks = 3
+            if test['chunks'] > maximum_number_verify_chunks:
+                test['chunks'] = maximum_number_verify_chunks
+
         if test['chunks'] == 1:
             test['this-chunk'] = 1
             yield test
             continue
 
         for this_chunk in range(1, test['chunks'] + 1):
             # copy the test and update with the chunk number
             chunked = copy.deepcopy(test)
@@ -786,16 +801,41 @@ def split_chunks(config, tests):
             # add the chunk number to the TH symbol
             group, symbol = split_symbol(chunked['treeherder-symbol'])
             symbol += str(this_chunk)
             chunked['treeherder-symbol'] = join_symbol(group, symbol)
 
             yield chunked
 
 
+def perfile_number_of_chunks(config, type):
+    # A rough estimate of how many chunks we need based on simple rules
+    # for determining what a test file is.
+
+    # TODO: Make this flexible based on coverage vs verify || test type
+    tests_per_chunk = 10.0
+
+    if type.startswith('test-verify-wpt'):
+        file_patterns = ['testing/web-platform/tests/**']
+    elif type.startswith('test-verify'):
+        file_patterns = ['**/test_*', '**/browser_*', '**/crashtest*/**',
+                          'js/src/test/test/', 'js/src/test/non262/', 'js/src/test/test262/']
+
+    changed_files = files_changed.get_changed_files(config.params.get('head_repository'),
+                                                    config.params.get('head_rev'))
+    test_count = 0
+    for pattern in file_patterns:
+        for path in changed_files:
+            if mozpackmatch(path, pattern):
+                test_count += 1
+
+    chunks = test_count/tests_per_chunk
+    return int(math.ceil(chunks))
+
+
 @transforms.add
 def allow_software_gl_layers(config, tests):
     """
     Handle the "allow-software-gl-layers" property for platforms where it
     applies.
     """
     for test in tests:
         if test.get('allow-software-gl-layers'):
--- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
+++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
@@ -1,16 +1,17 @@
 #!/usr/bin/env python
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import argparse
+import math
 import os
 import posixpath
 import re
 import sys
 import mozinfo
 from manifestparser import TestManifest
 from mozharness.base.script import PostScriptAction
 
@@ -184,16 +185,42 @@ class SingleTestMixin(object):
                 desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
             changed_files |= set(c['files'])
 
         if self.config.get('per_test_category') == "web-platform":
             self._find_wpt_tests(dirs, changed_files)
         else:
             self._find_misc_tests(dirs, changed_files)
 
+        # per test mode run specific tests from any given test suite
+        # _find_*_tests organizes tests to run into suites so we can
+        # run each suite at a time
+
+        # chunk files
+        total_tests = sum([len(self.suites[x]) for x in self.suites])
+
+        files_per_chunk = total_tests / float(self.config.get('total_chunks', 1))
+        files_per_chunk = int(math.ceil(files_per_chunk))
+
+        chunk_number = int(self.config.get('this_chunk', 1))
+        suites = {}
+        start = (chunk_number - 1) * files_per_chunk
+        end = (chunk_number * files_per_chunk)
+        current = -1
+        for suite in self.suites:
+            for test in self.suites[suite]:
+                current += 1
+                if current >= start and current < end:
+                    if suite not in suites:
+                        suites[suite] = []
+                    suites[suite].append(test)
+            if current >= end:
+                break
+
+        self.suites = suites
         self.tests_downloaded = True
 
     def query_args(self, suite):
         """
            For the specified suite, return an array of command line arguments to
            be passed to test harnesses when running in per-test mode.
 
            Each array element is an array of command line arguments for a modified
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -460,17 +460,17 @@ class AndroidEmulatorTest(TestingMixin, 
             if '%(app)' in option:
                 # only query package name if requested
                 cmd.extend([option % {'app': self._query_package_name()}])
             else:
                 cmd.extend([option % str_format_values])
 
         if user_paths:
             cmd.extend(user_paths.split(':'))
-        else:
+        elif not self.verify_enabled:
             if self.this_chunk is not None:
                 cmd.extend(['--this-chunk', self.this_chunk])
             if self.total_chunks is not None:
                 cmd.extend(['--total-chunks', self.total_chunks])
 
         try_options, try_tests = self.try_args(self.test_suite)
         cmd.extend(try_options)
         if not self.verify_enabled and not self.per_test_coverage:
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -395,17 +395,17 @@ class DesktopUnittest(TestingMixin, Merc
                 if suite_category in SUITE_DEFAULT_E10S and not c['e10s']:
                     base_cmd.append('--disable-e10s')
                 elif suite_category not in SUITE_DEFAULT_E10S and c['e10s']:
                     base_cmd.append('--e10s')
 
             # Ignore chunking if we have user specified test paths
             if os.environ.get('MOZHARNESS_TEST_PATHS'):
                 base_cmd.extend(os.environ['MOZHARNESS_TEST_PATHS'].split(':'))
-            elif c.get('total_chunks') and c.get('this_chunk'):
+            elif c.get('total_chunks') and c.get('this_chunk') and not self.verify_enabled:
                 base_cmd.extend(['--total-chunks', c['total_chunks'],
                                  '--this-chunk', c['this_chunk']])
 
             if c['no_random']:
                 if suite_category == "mochitest":
                     base_cmd.append('--bisect-chunk=default')
                 else:
                     self.warning("--no-random does not currently work with suites other than "
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -214,17 +214,17 @@ class WebPlatformTest(TestingMixin, Merc
             cmd.append("--stylo-threads=4")
 
         if os.environ.get('MOZHARNESS_TEST_PATHS'):
             prefix = 'testing/web-platform'
             paths = os.environ['MOZHARNESS_TEST_PATHS'].split(':')
             paths = [os.path.join(dirs["abs_wpttest_dir"], os.path.relpath(p, prefix))
                      for p in paths if p.startswith(prefix)]
             cmd.extend(paths)
-        else:
+        elif not self.verify_enabled:
             for opt in ["total_chunks", "this_chunk"]:
                 val = c.get(opt)
                 if val:
                     cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
 
         if "wdspec" in test_types:
             geckodriver_path = self._query_geckodriver()
             if not geckodriver_path or not os.path.isfile(geckodriver_path):