Bug 1425929 - test-verify jobs should pick a virtualization appropriate to the test. r=gbrown
authorJoel Maher <jmaher@mozilla.com>
Tue, 24 Apr 2018 10:20:19 -0400
changeset 787352 88b650567a7f57d782abea7cc2936550ea362e89
parent 787351 c26bbed87ab0a3977bff7b9f5da0a6f537db7362
child 787353 b2a75639bd93809b0fb7d1be9393d18e7c85fd92
push id107709
push userbmo:james@hoppipolla.co.uk
push dateTue, 24 Apr 2018 18:14:59 +0000
reviewersgbrown
bugs1425929
milestone61.0a1
Bug 1425929 - test-verify jobs should pick a virtualization appropriate to the test. r=gbrown
moz.build
python/mozbuild/mozbuild/schedules.py
taskcluster/ci/test/misc.yml
taskcluster/ci/test/test-sets.yml
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/testing/per_test_base.py
testing/mozharness/scripts/desktop_unittest.py
--- a/moz.build
+++ b/moz.build
@@ -54,35 +54,35 @@ with Files('README.txt'):
 with Files("nsprpub/**"):
     BUG_COMPONENT = ("NSPR", "NSPR")
 
 with Files('**/Makefile.in'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
     FINAL = True
 
 with Files("**/*.js"):
-    SCHEDULES.inclusive += ['test-verify', 'docs']
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu', 'docs']
 
 with Files("**/*.jsm"):
     SCHEDULES.inclusive += ['docs']
 
 with Files("**/*.rst"):
     SCHEDULES.inclusive += ['docs']
 
 with Files("**/*.md"):
     SCHEDULES.inclusive += ['docs']
 
 with Files("**/*.html"):
-    SCHEDULES.inclusive += ['test-verify']
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
 
 with Files("**/*.xhtml"):
-    SCHEDULES.inclusive += ['test-verify']
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
 
 with Files("**/*.xul"):
-    SCHEDULES.inclusive += ['test-verify']
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
 
 CONFIGURE_SUBST_FILES += [
     'config/autoconf.mk',
     'config/emptyvars.mk',
 ]
 
 if CONFIG['ENABLE_CLANG_PLUGIN']:
     DIRS += ['build/clang-plugin']
--- a/python/mozbuild/mozbuild/schedules.py
+++ b/python/mozbuild/mozbuild/schedules.py
@@ -18,16 +18,17 @@ from __future__ import absolute_import, 
 INCLUSIVE_COMPONENTS = [
     'docs',
     'py-lint',
     'js-lint',
     'yaml-lint',
     # inclusive test suites -- these *only* run when certain files have changed
     'jittest',
     'test-verify',
+    'test-verify-gpu',
     'test-verify-wpt',
     'test-coverage',
     'test-coverage-wpt',
     'jsreftest',
 ]
 INCLUSIVE_COMPONENTS = sorted(INCLUSIVE_COMPONENTS)
 
 # Exclusive components are those which are scheduled by default, but for which
--- a/taskcluster/ci/test/misc.yml
+++ b/taskcluster/ci/test/misc.yml
@@ -93,16 +93,61 @@ test-verify:
                 macosx.*:
                     - unittests/mac_unittest.py
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
         no-read-buildbot-config: true
         extra-options:
             - --verify
 
+test-verify-gpu:
+    description: "Extra verification of tests modified on this push on gpu instances"
+    suite: test-verify
+    treeherder-symbol: TVg
+    loopback-video: true
+    virtualization: virtual-with-gpu
+    instance-size:
+        by-test-platform:
+            android.*: xlarge
+            default: default
+    max-run-time: 10800
+    allow-software-gl-layers: false
+    run-on-projects:
+        by-test-platform:
+            # do not run on ccov; see also the enable_code_coverage transform
+            linux64-ccov/.*: []
+            windows10-64-ccov/debug: []
+            # do not run on beta or release: usually just confirms earlier results
+            default: ['trunk', 'try']
+    tier:
+        by-test-platform:
+            windows10-64-asan.*: 3
+            default: 2
+    mozharness:
+        script:
+            by-test-platform:
+                android.*: android_emulator_unittest.py
+                default: desktop_unittest.py
+        config:
+            by-test-platform:
+                android.*:
+                    - android/android_common.py
+                    - android/androidarm_4_3.py
+                linux.*:
+                    - unittests/linux_unittest.py
+                    - remove_executables.py
+                macosx.*:
+                    - unittests/mac_unittest.py
+                windows.*:
+                    - unittests/win_taskcluster_unittest.py
+        no-read-buildbot-config: true
+        extra-options:
+            - --verify
+            - --gpu-required
+
 test-coverage:
     description: "Per-test coverage"
     suite: test-coverage
     treeherder-symbol: TC
     loopback-video: true
     instance-size: default
     max-run-time: 10800
     allow-software-gl-layers: false
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -35,16 +35,17 @@ common-tests:
     - mochitest-media
     - mochitest-webgl
     - reftest
     - reftest-no-accel
     - telemetry-tests-client
     - test-coverage
     - test-coverage-wpt
     - test-verify
+    - test-verify-gpu
     - test-verify-wpt
     - xpcshell
 
 web-platform-tests:
     - web-platform-tests
     - web-platform-tests-reftests
     - web-platform-tests-wdspec
 
@@ -185,16 +186,17 @@ windows-tests:
     - mochitest-gpu
     - mochitest-media
     - mochitest-webgl
     - reftest
     - reftest-no-accel
     - test-coverage
     - test-coverage-wpt
     - test-verify
+    - test-verify-gpu
     - test-verify-wpt
     - web-platform-tests
     - web-platform-tests-reftests
     - xpcshell
 
 windows-talos:
     - talos-chrome
     - talos-damp
@@ -248,16 +250,17 @@ macosx64-tests:
     - mochitest-chrome
     - mochitest-clipboard
     - mochitest-devtools-chrome
     - mochitest-gpu
     - mochitest-media
     - mochitest-webgl
     - reftest
     - test-verify
+    - test-verify-gpu
     - test-verify-wpt
     - web-platform-tests
     - web-platform-tests-reftests
     - xpcshell
 
 macosx64-talos:
     - talos-chrome
     - talos-damp
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -810,27 +810,47 @@ def perfile_number_of_chunks(config, typ
     # A rough estimate of how many chunks we need based on simple rules
     # for determining what a test file is.
 
     # TODO: Make this flexible based on coverage vs verify || test type
     tests_per_chunk = 10.0
 
     if type.startswith('test-verify-wpt'):
         file_patterns = ['testing/web-platform/tests/**']
+    elif type.startswith('test-verify-gpu'):
+        file_patterns = ['**/*webgl*/**/test_*',
+                         '**/dom/canvas/**/test_*',
+                         '**/gfx/tests/**/test_*',
+                         '**/devtools/canvasdebugger/**/browser_*',
+                         '**/reftest*/**']
     elif type.startswith('test-verify'):
-        file_patterns = ['**/test_*', '**/browser_*', '**/crashtest*/**',
-                          'js/src/test/test/', 'js/src/test/non262/', 'js/src/test/test262/']
+        file_patterns = ['**/test_*',
+                         '**/browser_*',
+                         '**/crashtest*/**',
+                         'js/src/test/test/',
+                         'js/src/test/non262/',
+                         'js/src/test/test262/']
 
     changed_files = files_changed.get_changed_files(config.params.get('head_repository'),
                                                     config.params.get('head_rev'))
     test_count = 0
     for pattern in file_patterns:
         for path in changed_files:
             if mozpackmatch(path, pattern):
-                test_count += 1
+                gpu = False
+                if type == 'test-verify-e10s':
+                    # file_patterns for test-verify will pick up some gpu tests, lets ignore
+                    # in the case of reftest, we will not have any in the regular case
+                    gpu_dirs = ['dom/canvas', 'gfx/tests', 'devtools/canvasdebugger', 'webgl']
+                    for gdir in gpu_dirs:
+                        if len(path.split(gdir)) > 1:
+                            gpu = True
+
+                if not gpu:
+                    test_count += 1
 
     chunks = test_count/tests_per_chunk
     return int(math.ceil(chunks))
 
 
 @transforms.add
 def allow_software_gl_layers(config, tests):
     """
--- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
+++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
@@ -20,17 +20,17 @@ class SingleTestMixin(object):
     """Utility functions for per-test testing like test verification and per-test coverage."""
 
     def __init__(self):
         self.suites = {}
         self.tests_downloaded = False
         self.reftest_test_dir = None
         self.jsreftest_test_dir = None
 
-    def _find_misc_tests(self, dirs, changed_files):
+    def _find_misc_tests(self, dirs, changed_files, gpu=False):
         manifests = [
             (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'),
             (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'),
             (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'),
             (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'),
             (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'),
         ]
         tests_by_path = {}
@@ -45,27 +45,27 @@ class SingleTestMixin(object):
                 disabled = [t['relpath'] for t in active if 'disabled' in t]
                 new_by_path = {t['relpath']:(suite,t.get('subsuite')) \
                                for t in active if 'disabled' not in t and \
                                t['relpath'] not in disabled}
                 tests_by_path.update(new_by_path)
                 self.info("Per-test run updated with manifest %s" % path)
 
         ref_manifests = [
-            (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest'),
-            (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest'),
+            (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest', 'gpu'), #gpu
+            (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest', None),
         ]
         sys.path.append(dirs['abs_reftest_dir'])
         import manifest
         self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests')
-        for (path, suite) in ref_manifests:
+        for (path, suite, subsuite) in ref_manifests:
             if os.path.exists(path):
                 man = manifest.ReftestManifest()
                 man.load(path)
-                tests_by_path.update({os.path.relpath(t,self.reftest_test_dir):(suite,None) for t in man.files})
+                tests_by_path.update({os.path.relpath(t,self.reftest_test_dir):(suite,subsuite) for t in man.files})
                 self.info("Per-test run updated with manifest %s" % path)
 
         suite = 'jsreftest'
         self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests')
         path = os.path.join(self.jsreftest_test_dir, 'jstests.list')
         if os.path.exists(path):
             man = manifest.ReftestManifest()
             man.load(path)
@@ -85,27 +85,33 @@ class SingleTestMixin(object):
 
         # for each changed file, determine if it is a test file, and what suite it is in
         for file in changed_files:
             # manifest paths use os.sep (like backslash on Windows) but
             # automation-relevance uses posixpath.sep
             file = file.replace(posixpath.sep, os.sep)
             entry = tests_by_path.get(file)
             if entry:
+                if gpu and entry[1] not in ['gpu', 'webgl']:
+                    continue
+                elif not gpu and entry[1] in ['gpu', 'webgl']:
+                    continue
+
                 self.info("Per-test run found test %s" % file)
                 subsuite_mapping = {
                     ('browser-chrome', 'clipboard') : 'browser-chrome-clipboard',
                     ('chrome', 'clipboard') : 'chrome-clipboard',
                     ('plain', 'clipboard') : 'plain-clipboard',
                     ('browser-chrome', 'devtools') : 'mochitest-devtools-chrome',
+                    ('browser-chrome', 'screenshots') : 'browser-chrome-screenshots',
+                    ('plain', 'media') : 'mochitest-media',
+                     # below should be on test-verify-gpu job
                     ('browser-chrome', 'gpu') : 'browser-chrome-gpu',
-                    ('browser-chrome', 'screenshots') : 'browser-chrome-screenshots',
                     ('chrome', 'gpu') : 'chrome-gpu',
                     ('plain', 'gpu') : 'plain-gpu',
-                    ('plain', 'media') : 'mochitest-media',
                     ('plain', 'webgl') : 'mochitest-gl',
                 }
                 if entry in subsuite_mapping:
                     suite = subsuite_mapping[entry]
                 else:
                     suite = entry[0]
                 suite_files = self.suites.get(suite)
                 if not suite_files:
@@ -182,16 +188,18 @@ class SingleTestMixin(object):
         for c in contents['changesets']:
             self.info(" {cset} {desc}".format(
                 cset=c['node'][0:12],
                 desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
             changed_files |= set(c['files'])
 
         if self.config.get('per_test_category') == "web-platform":
             self._find_wpt_tests(dirs, changed_files)
+        elif self.config.get('gpu_required') == True:
+            self._find_misc_tests(dirs, changed_files, gpu=True)
         else:
             self._find_misc_tests(dirs, changed_files)
 
         # per test mode run specific tests from any given test suite
         # _find_*_tests organizes tests to run into suites so we can
         # run each suite at a time
 
         # chunk files
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -164,16 +164,22 @@ class DesktopUnittest(TestingMixin, Merc
             "help": "Forcibly enable single thread traversal in Stylo with STYLO_THREADS=1"}
          ],
         [["--enable-webrender"], {
             "action": "store_true",
             "dest": "enable_webrender",
             "default": False,
             "help": "Tries to enable the WebRender compositor."}
          ],
+        [["--gpu-required"], {
+            "action": "store_true",
+            "dest": "gpu_required",
+            "default": "False",
+            "help": "Run additional verification on modified tests using gpu instances."}
+         ],
     ] + copy.deepcopy(testing_config_options) + \
         copy.deepcopy(blobupload_config_options) + \
         copy.deepcopy(code_coverage_config_options)
 
     def __init__(self, require_config_file=True):
         # abs_dirs defined already in BaseScript but is here to make pylint happy
         self.abs_dirs = None
         super(DesktopUnittest, self).__init__(