Bug 1467837 - adjust some code for test-coverage gpu/chunks and minor cleanup in per test code. r=gbrown
authorJoel Maher <jmaher@mozilla.com>
Fri, 08 Jun 2018 16:21:14 -0400
changeset 422033 37faedd4429185358558a9f1aa046e1110ec3522
parent 422032 da935e9b476d83b3652ee3425aa94a28d02397f1
child 422034 68fd5d83cc0787e8240cca4851b6920fe27af43f
push id34114
push userbtara@mozilla.com
push dateSat, 09 Jun 2018 15:31:58 +0000
treeherdermozilla-central@e02a5155d815 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1467837
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1467837 - adjust some code for test-coverage gpu/chunks and minor cleanup in per test code. r=gbrown
taskcluster/ci/test/misc.yml
taskcluster/taskgraph/util/perfile.py
testing/mozharness/mozharness/mozilla/testing/per_test_base.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/web_platform_tests.py
--- a/taskcluster/ci/test/misc.yml
+++ b/taskcluster/ci/test/misc.yml
@@ -200,8 +200,47 @@ test-coverage:
                     - unittests/linux_unittest.py
                     - remove_executables.py
                 macosx.*:
                     - unittests/mac_unittest.py
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
         extra-options:
             - --per-test-coverage
+
+test-coverage-gpu:
+    description: "Per-test coverage for tests that require gpu instances"
+    suite: test-coverage
+    treeherder-symbol: TCg
+    loopback-video: true
+    virtualization: virtual-with-gpu
+    instance-size: default
+    max-run-time: 10800
+    allow-software-gl-layers: false
+    run-on-projects:
+        by-test-platform:
+            # only run on mozilla-central and try.
+            .*-ccov/.*: ['mozilla-central', 'try']
+            default: []
+    tier:
+        by-test-platform:
+            windows10-64-asan.*: 3
+            default: 2
+    mozharness:
+        script:
+            by-test-platform:
+                android.*: android_emulator_unittest.py
+                default: desktop_unittest.py
+        config:
+            by-test-platform:
+                android.*:
+                    - android/android_common.py
+                    - android/androidarm_4_3.py
+                linux.*:
+                    - unittests/linux_unittest.py
+                    - remove_executables.py
+                macosx.*:
+                    - unittests/mac_unittest.py
+                windows.*:
+                    - unittests/win_taskcluster_unittest.py
+        extra-options:
+            - --per-test-coverage
+            - --gpu-required
--- a/taskcluster/taskgraph/util/perfile.py
+++ b/taskcluster/taskgraph/util/perfile.py
@@ -14,18 +14,19 @@ from subprocess import CalledProcessErro
 from taskgraph import files_changed
 from .. import GECKO
 
 logger = logging.getLogger(__name__)
 
 
 @memoize
 def perfile_number_of_chunks(try_task_config, head_repository, head_rev, type):
-    # TODO: Make this flexible based on coverage vs verify || test type
     tests_per_chunk = 10.0
+    if type.startswith('test-coverage'):
+        tests_per_chunk = 30.0
 
     if type.startswith('test-verify-wpt') or type.startswith('test-coverage-wpt'):
         file_patterns = ['testing/web-platform/tests/**',
                          'testing/web-platform/mozilla/tests/**']
     elif type.startswith('test-verify-gpu') or type.startswith('test-coverage-gpu'):
         file_patterns = ['**/*webgl*/**/test_*',
                          '**/dom/canvas/**/test_*',
                          '**/gfx/tests/**/test_*',
--- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
+++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
@@ -87,45 +87,47 @@ class SingleTestMixin(object):
             self.info("Per-test run updated with manifest %s" % path)
 
         # for each changed file, determine if it is a test file, and what suite it is in
         for file in changed_files:
             # manifest paths use os.sep (like backslash on Windows) but
             # automation-relevance uses posixpath.sep
             file = file.replace(posixpath.sep, os.sep)
             entry = tests_by_path.get(file)
-            if entry:
-                if gpu and entry[1] not in ['gpu', 'webgl']:
-                    continue
-                elif not gpu and entry[1] in ['gpu', 'webgl']:
-                    continue
+            if not entry:
+                continue
+
+            if gpu and entry[1] not in ['gpu', 'webgl']:
+                continue
+            elif not gpu and entry[1] in ['gpu', 'webgl']:
+                continue
 
-                self.info("Per-test run found test %s" % file)
-                subsuite_mapping = {
-                    ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard',
-                    ('chrome', 'clipboard'): 'chrome-clipboard',
-                    ('plain', 'clipboard'): 'plain-clipboard',
-                    ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome',
-                    ('browser-chrome', 'screenshots'): 'browser-chrome-screenshots',
-                    ('plain', 'media'): 'mochitest-media',
-                    # below should be on test-verify-gpu job
-                    ('browser-chrome', 'gpu'): 'browser-chrome-gpu',
-                    ('chrome', 'gpu'): 'chrome-gpu',
-                    ('plain', 'gpu'): 'plain-gpu',
-                    ('plain', 'webgl'): 'mochitest-gl',
-                }
-                if entry in subsuite_mapping:
-                    suite = subsuite_mapping[entry]
-                else:
-                    suite = entry[0]
-                suite_files = self.suites.get(suite)
-                if not suite_files:
-                    suite_files = []
-                suite_files.append(file)
-                self.suites[suite] = suite_files
+            self.info("Per-test run found test %s (%s)" % (file, entry[0]))
+            subsuite_mapping = {
+                ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard',
+                ('chrome', 'clipboard'): 'chrome-clipboard',
+                ('plain', 'clipboard'): 'plain-clipboard',
+                ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome',
+                ('browser-chrome', 'screenshots'): 'browser-chrome-screenshots',
+                ('plain', 'media'): 'mochitest-media',
+                # below should be on test-verify-gpu job
+                ('browser-chrome', 'gpu'): 'browser-chrome-gpu',
+                ('chrome', 'gpu'): 'chrome-gpu',
+                ('plain', 'gpu'): 'plain-gpu',
+                ('plain', 'webgl'): 'mochitest-gl',
+            }
+            if entry in subsuite_mapping:
+                suite = subsuite_mapping[entry]
+            else:
+                suite = entry[0]
+            suite_files = self.suites.get(suite)
+            if not suite_files:
+                suite_files = []
+            suite_files.append(file)
+            self.suites[suite] = suite_files
 
     def _find_wpt_tests(self, dirs, changed_files):
         # Setup sys.path to include all the dependencies required to import
         # the web-platform-tests manifest parser. web-platform-tests provides
         # the localpaths.py to do the path manipulation, which we load,
         # providing the __file__ variable so it can resolve the relative
         # paths correctly.
         paths_file = os.path.join(dirs['abs_wpttest_dir'],
@@ -194,17 +196,17 @@ class SingleTestMixin(object):
             for c in contents['changesets']:
                 self.info(" {cset} {desc}".format(
                     cset=c['node'][0:12],
                     desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
                 changed_files |= set(c['files'])
 
         if self.config.get('per_test_category') == "web-platform":
             self._find_wpt_tests(dirs, changed_files)
-        elif self.config.get('gpu_required'):
+        elif self.config.get('gpu_required', 'False') != 'False':
             self._find_misc_tests(dirs, changed_files, gpu=True)
         else:
             self._find_misc_tests(dirs, changed_files)
 
         # per test mode run specific tests from any given test suite
         # _find_*_tests organizes tests to run into suites so we can
         # run each suite at a time
 
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -764,16 +764,18 @@ class DesktopUnittest(TestingMixin, Merc
         """run suite(s) to a specific category"""
         dirs = self.query_abs_dirs()
         suites = self._query_specified_suites(suite_category)
         abs_app_dir = self.query_abs_app_dir()
         abs_res_dir = self.query_abs_res_dir()
 
         max_per_test_time = timedelta(minutes=60)
         max_per_test_tests = 10
+        if self.per_test_coverage:
+            max_per_test_tests = 30
         executed_tests = 0
         executed_too_many_tests = False
 
         if suites:
             self.info('#### Running %s suites' % suite_category)
             for suite in suites:
                 if executed_too_many_tests and not self.per_test_coverage:
                     return False
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -313,16 +313,18 @@ class WebPlatformTest(TestingMixin, Merc
         else:
             env['STYLO_THREADS'] = '4'
 
         env = self.query_env(partial_env=env, log_level=INFO)
 
         start_time = datetime.now()
         max_per_test_time = timedelta(minutes=60)
         max_per_test_tests = 10
+        if self.per_test_coverage:
+            max_per_test_tests = 30
         executed_tests = 0
         executed_too_many_tests = False
 
         if self.per_test_coverage or self.verify_enabled:
             suites = self.query_per_test_category_suites(None, None)
             if "wdspec" in suites:
                 # geckodriver is required for wdspec, but not always available
                 geckodriver_path = self._query_geckodriver()