Bug 1425929 - test-verify jobs should pick a virtualization appropriate to the test. r=gbrown
authorJoel Maher <jmaher@mozilla.com>
Tue, 24 Apr 2018 10:20:19 -0400
changeset 415301 88b650567a7f57d782abea7cc2936550ea362e89
parent 415300 c26bbed87ab0a3977bff7b9f5da0a6f537db7362
child 415302 b2a75639bd93809b0fb7d1be9393d18e7c85fd92
push id102539
push userjmaher@mozilla.com
push dateTue, 24 Apr 2018 14:24:24 +0000
treeherdermozilla-inbound@88b650567a7f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1425929
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1425929 - test-verify jobs should pick a virtualization appropriate to the test. r=gbrown
moz.build
python/mozbuild/mozbuild/schedules.py
taskcluster/ci/test/misc.yml
taskcluster/ci/test/test-sets.yml
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/testing/per_test_base.py
testing/mozharness/scripts/desktop_unittest.py
--- a/moz.build
+++ b/moz.build
@@ -54,35 +54,35 @@ with Files('README.txt'):
 with Files("nsprpub/**"):
     BUG_COMPONENT = ("NSPR", "NSPR")
 
 with Files('**/Makefile.in'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
     FINAL = True
 
 with Files("**/*.js"):
-    SCHEDULES.inclusive += ['test-verify', 'docs']
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu', 'docs']
 
 with Files("**/*.jsm"):
     SCHEDULES.inclusive += ['docs']
 
 with Files("**/*.rst"):
     SCHEDULES.inclusive += ['docs']
 
 with Files("**/*.md"):
     SCHEDULES.inclusive += ['docs']
 
 with Files("**/*.html"):
-    SCHEDULES.inclusive += ['test-verify']
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
 
 with Files("**/*.xhtml"):
-    SCHEDULES.inclusive += ['test-verify']
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
 
 with Files("**/*.xul"):
-    SCHEDULES.inclusive += ['test-verify']
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
 
 CONFIGURE_SUBST_FILES += [
     'config/autoconf.mk',
     'config/emptyvars.mk',
 ]
 
 if CONFIG['ENABLE_CLANG_PLUGIN']:
     DIRS += ['build/clang-plugin']
--- a/python/mozbuild/mozbuild/schedules.py
+++ b/python/mozbuild/mozbuild/schedules.py
@@ -18,16 +18,17 @@ from __future__ import absolute_import, 
 INCLUSIVE_COMPONENTS = [
     'docs',
     'py-lint',
     'js-lint',
     'yaml-lint',
     # inclusive test suites -- these *only* run when certain files have changed
     'jittest',
     'test-verify',
+    'test-verify-gpu',
     'test-verify-wpt',
     'test-coverage',
     'test-coverage-wpt',
     'jsreftest',
 ]
 INCLUSIVE_COMPONENTS = sorted(INCLUSIVE_COMPONENTS)
 
 # Exclusive components are those which are scheduled by default, but for which
--- a/taskcluster/ci/test/misc.yml
+++ b/taskcluster/ci/test/misc.yml
@@ -93,16 +93,61 @@ test-verify:
                 macosx.*:
                     - unittests/mac_unittest.py
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
         no-read-buildbot-config: true
         extra-options:
             - --verify
 
+test-verify-gpu:
+    description: "Extra verification of tests modified on this push on gpu instances"
+    suite: test-verify
+    treeherder-symbol: TVg
+    loopback-video: true
+    virtualization: virtual-with-gpu
+    instance-size:
+        by-test-platform:
+            android.*: xlarge
+            default: default
+    max-run-time: 10800
+    allow-software-gl-layers: false
+    run-on-projects:
+        by-test-platform:
+            # do not run on ccov; see also the enable_code_coverage transform
+            linux64-ccov/.*: []
+            windows10-64-ccov/debug: []
+            # do not run on beta or release: usually just confirms earlier results
+            default: ['trunk', 'try']
+    tier:
+        by-test-platform:
+            windows10-64-asan.*: 3
+            default: 2
+    mozharness:
+        script:
+            by-test-platform:
+                android.*: android_emulator_unittest.py
+                default: desktop_unittest.py
+        config:
+            by-test-platform:
+                android.*:
+                    - android/android_common.py
+                    - android/androidarm_4_3.py
+                linux.*:
+                    - unittests/linux_unittest.py
+                    - remove_executables.py
+                macosx.*:
+                    - unittests/mac_unittest.py
+                windows.*:
+                    - unittests/win_taskcluster_unittest.py
+        no-read-buildbot-config: true
+        extra-options:
+            - --verify
+            - --gpu-required
+
 test-coverage:
     description: "Per-test coverage"
     suite: test-coverage
     treeherder-symbol: TC
     loopback-video: true
     instance-size: default
     max-run-time: 10800
     allow-software-gl-layers: false
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -35,16 +35,17 @@ common-tests:
     - mochitest-media
     - mochitest-webgl
     - reftest
     - reftest-no-accel
     - telemetry-tests-client
     - test-coverage
     - test-coverage-wpt
     - test-verify
+    - test-verify-gpu
     - test-verify-wpt
     - xpcshell
 
 web-platform-tests:
     - web-platform-tests
     - web-platform-tests-reftests
     - web-platform-tests-wdspec
 
@@ -185,16 +186,17 @@ windows-tests:
     - mochitest-gpu
     - mochitest-media
     - mochitest-webgl
     - reftest
     - reftest-no-accel
     - test-coverage
     - test-coverage-wpt
     - test-verify
+    - test-verify-gpu
     - test-verify-wpt
     - web-platform-tests
     - web-platform-tests-reftests
     - xpcshell
 
 windows-talos:
     - talos-chrome
     - talos-damp
@@ -248,16 +250,17 @@ macosx64-tests:
     - mochitest-chrome
     - mochitest-clipboard
     - mochitest-devtools-chrome
     - mochitest-gpu
     - mochitest-media
     - mochitest-webgl
     - reftest
     - test-verify
+    - test-verify-gpu
     - test-verify-wpt
     - web-platform-tests
     - web-platform-tests-reftests
     - xpcshell
 
 macosx64-talos:
     - talos-chrome
     - talos-damp
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -810,27 +810,47 @@ def perfile_number_of_chunks(config, typ
     # A rough estimate of how many chunks we need based on simple rules
     # for determining what a test file is.
 
     # TODO: Make this flexible based on coverage vs verify || test type
     tests_per_chunk = 10.0
 
     if type.startswith('test-verify-wpt'):
         file_patterns = ['testing/web-platform/tests/**']
+    elif type.startswith('test-verify-gpu'):
+        file_patterns = ['**/*webgl*/**/test_*',
+                         '**/dom/canvas/**/test_*',
+                         '**/gfx/tests/**/test_*',
+                         '**/devtools/canvasdebugger/**/browser_*',
+                         '**/reftest*/**']
     elif type.startswith('test-verify'):
-        file_patterns = ['**/test_*', '**/browser_*', '**/crashtest*/**',
-                          'js/src/test/test/', 'js/src/test/non262/', 'js/src/test/test262/']
+        file_patterns = ['**/test_*',
+                         '**/browser_*',
+                         '**/crashtest*/**',
+                         'js/src/test/test/',
+                         'js/src/test/non262/',
+                         'js/src/test/test262/']
 
     changed_files = files_changed.get_changed_files(config.params.get('head_repository'),
                                                     config.params.get('head_rev'))
     test_count = 0
     for pattern in file_patterns:
         for path in changed_files:
             if mozpackmatch(path, pattern):
-                test_count += 1
+                gpu = False
+                if type == 'test-verify-e10s':
+                    # file_patterns for test-verify will pick up some gpu tests, lets ignore
+                    # in the case of reftest, we will not have any in the regular case
+                    gpu_dirs = ['dom/canvas', 'gfx/tests', 'devtools/canvasdebugger', 'webgl']
+                    for gdir in gpu_dirs:
+                        if len(path.split(gdir)) > 1:
+                            gpu = True
+
+                if not gpu:
+                    test_count += 1
 
     chunks = test_count/tests_per_chunk
     return int(math.ceil(chunks))
 
 
 @transforms.add
 def allow_software_gl_layers(config, tests):
     """
--- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
+++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
@@ -20,17 +20,17 @@ class SingleTestMixin(object):
     """Utility functions for per-test testing like test verification and per-test coverage."""
 
     def __init__(self):
         self.suites = {}
         self.tests_downloaded = False
         self.reftest_test_dir = None
         self.jsreftest_test_dir = None
 
-    def _find_misc_tests(self, dirs, changed_files):
+    def _find_misc_tests(self, dirs, changed_files, gpu=False):
         manifests = [
             (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'),
             (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'),
             (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'),
             (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'),
             (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'),
         ]
         tests_by_path = {}
@@ -45,27 +45,27 @@ class SingleTestMixin(object):
                 disabled = [t['relpath'] for t in active if 'disabled' in t]
                 new_by_path = {t['relpath']:(suite,t.get('subsuite')) \
                                for t in active if 'disabled' not in t and \
                                t['relpath'] not in disabled}
                 tests_by_path.update(new_by_path)
                 self.info("Per-test run updated with manifest %s" % path)
 
         ref_manifests = [
-            (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest'),
-            (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest'),
+            (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest', 'gpu'), #gpu
+            (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest', None),
         ]
         sys.path.append(dirs['abs_reftest_dir'])
         import manifest
         self.reftest_test_dir = os.path.join(dirs['abs_reftest_dir'], 'tests')
-        for (path, suite) in ref_manifests:
+        for (path, suite, subsuite) in ref_manifests:
             if os.path.exists(path):
                 man = manifest.ReftestManifest()
                 man.load(path)
-                tests_by_path.update({os.path.relpath(t,self.reftest_test_dir):(suite,None) for t in man.files})
+                tests_by_path.update({os.path.relpath(t,self.reftest_test_dir):(suite,subsuite) for t in man.files})
                 self.info("Per-test run updated with manifest %s" % path)
 
         suite = 'jsreftest'
         self.jsreftest_test_dir = os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests')
         path = os.path.join(self.jsreftest_test_dir, 'jstests.list')
         if os.path.exists(path):
             man = manifest.ReftestManifest()
             man.load(path)
@@ -85,27 +85,33 @@ class SingleTestMixin(object):
 
         # for each changed file, determine if it is a test file, and what suite it is in
         for file in changed_files:
             # manifest paths use os.sep (like backslash on Windows) but
             # automation-relevance uses posixpath.sep
             file = file.replace(posixpath.sep, os.sep)
             entry = tests_by_path.get(file)
             if entry:
+                if gpu and entry[1] not in ['gpu', 'webgl']:
+                    continue
+                elif not gpu and entry[1] in ['gpu', 'webgl']:
+                    continue
+
                 self.info("Per-test run found test %s" % file)
                 subsuite_mapping = {
                     ('browser-chrome', 'clipboard') : 'browser-chrome-clipboard',
                     ('chrome', 'clipboard') : 'chrome-clipboard',
                     ('plain', 'clipboard') : 'plain-clipboard',
                     ('browser-chrome', 'devtools') : 'mochitest-devtools-chrome',
+                    ('browser-chrome', 'screenshots') : 'browser-chrome-screenshots',
+                    ('plain', 'media') : 'mochitest-media',
+                     # below should be on test-verify-gpu job
                     ('browser-chrome', 'gpu') : 'browser-chrome-gpu',
-                    ('browser-chrome', 'screenshots') : 'browser-chrome-screenshots',
                     ('chrome', 'gpu') : 'chrome-gpu',
                     ('plain', 'gpu') : 'plain-gpu',
-                    ('plain', 'media') : 'mochitest-media',
                     ('plain', 'webgl') : 'mochitest-gl',
                 }
                 if entry in subsuite_mapping:
                     suite = subsuite_mapping[entry]
                 else:
                     suite = entry[0]
                 suite_files = self.suites.get(suite)
                 if not suite_files:
@@ -182,16 +188,18 @@ class SingleTestMixin(object):
         for c in contents['changesets']:
             self.info(" {cset} {desc}".format(
                 cset=c['node'][0:12],
                 desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
             changed_files |= set(c['files'])
 
         if self.config.get('per_test_category') == "web-platform":
             self._find_wpt_tests(dirs, changed_files)
+        elif self.config.get('gpu_required') == True:
+            self._find_misc_tests(dirs, changed_files, gpu=True)
         else:
             self._find_misc_tests(dirs, changed_files)
 
         # per test mode run specific tests from any given test suite
         # _find_*_tests organizes tests to run into suites so we can
         # run each suite at a time
 
         # chunk files
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -164,16 +164,22 @@ class DesktopUnittest(TestingMixin, Merc
             "help": "Forcibly enable single thread traversal in Stylo with STYLO_THREADS=1"}
          ],
         [["--enable-webrender"], {
             "action": "store_true",
             "dest": "enable_webrender",
             "default": False,
             "help": "Tries to enable the WebRender compositor."}
          ],
+        [["--gpu-required"], {
+            "action": "store_true",
+            "dest": "gpu_required",
+            "default": "False",
+            "help": "Run additional verification on modified tests using gpu instances."}
+         ],
     ] + copy.deepcopy(testing_config_options) + \
         copy.deepcopy(blobupload_config_options) + \
         copy.deepcopy(code_coverage_config_options)
 
     def __init__(self, require_config_file=True):
         # abs_dirs defined already in BaseScript but is here to make pylint happy
         self.abs_dirs = None
         super(DesktopUnittest, self).__init__(