Bug 1523303 - Align mozharness suite names with the ones in 'moztest.resolve', r=gbrown
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 22 Apr 2019 22:32:34 +0000
changeset 470488 60c512bab3e986b609d7fda44d827c21d90612d2
parent 470487 00c2b2449639493d03dbed26e5765838c017db7d
child 470489 66fe93fc552c537916e311e8dfc61bb90fbb8132
push id35906
push useraciure@mozilla.com
push dateTue, 23 Apr 2019 22:14:56 +0000
treeherdermozilla-central@0ce3633f8b80 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1523303
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1523303 - Align mozharness suite names with the ones in 'moztest.resolve', r=gbrown This officially makes 'moztest.resolve' the source of truth when it comes to suite names. It aligns that file with the names used in both the desktop_unittest and android_emulator_unittest scripts. Differential Revision: https://phabricator.services.mozilla.com/D27555
taskcluster/ci/test/mochitest.yml
taskcluster/taskgraph/target_tasks.py
testing/mach_commands.py
testing/mozbase/moztest/moztest/resolve.py
testing/mozbase/moztest/tests/test_resolve.py
testing/mozharness/configs/android/android_common.py
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/configs/unittests/mac_unittest.py
testing/mozharness/configs/unittests/win_unittest.py
testing/mozharness/mozharness/mozilla/testing/errors.py
testing/mozharness/mozharness/mozilla/testing/per_test_base.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
testing/mozharness/scripts/android_emulator_unittest.py
testing/mozharness/scripts/android_hardware_unittest.py
tools/tryselect/tasks.py
tools/tryselect/test/test_mozharness_integration.py
tools/tryselect/test/test_tasks.py
--- a/taskcluster/ci/test/mochitest.yml
+++ b/taskcluster/ci/test/mochitest.yml
@@ -37,17 +37,17 @@ job-defaults:
                 macosx.*:
                     - unittests/mac_unittest.py
                 windows.*:
                     - unittests/win_unittest.py
 
 mochitest:
     description: "Mochitest plain run"
     suite:
-        name: plain-chunked
+        name: mochitest-plain-chunked
     treeherder-symbol: M()
     loopback-video: true
     tier:
         by-test-platform:
             android-em-7.*/opt: 2
             windows10-aarch64.*: 2
             default: default
     instance-size:
@@ -78,42 +78,40 @@ mochitest:
             android-em.*: 7200
             default: 5400
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
         extra-options:
             by-test-platform:
                 android-em.*:
-                    - --test-suite=mochitest
+                    - --test-suite=mochitest-plain
                 default: []
         chunked:
             by-test-platform:
                 android-em.*: false
                 default: true
 
 mochitest-a11y:
     description: "Mochitest a11y run"
-    suite:
-        name: a11y
     treeherder-symbol: M(a11y)
     loopback-video: true
     tier:
         by-test-platform:
             windows10-aarch64.*: 2
             default: default
     e10s: false
     run-on-projects: built-projects
     mozharness:
         mochitest-flavor: a11y
 
 mochitest-browser-chrome:
     description: "Mochitest browser-chrome run"
     suite:
-        name: browser-chrome-chunked
+        name: mochitest-browser-chrome-chunked
     treeherder-symbol: M(bc)
     loopback-video: true
     tier:
         by-test-platform:
             windows10-aarch64.*: 2
             default: default
     chunks:
         by-test-platform:
@@ -134,33 +132,33 @@ mochitest-browser-chrome:
         chunked: true
     # Bug 1281241: migrating to m3.large instances
     instance-size: default
     allow-software-gl-layers: false
 
 browser-instrumentation:
     description: "Extra instrumentation for a browser-chrome run (XUL, XBL, etc)"
     suite:
-        name: browser-chrome-instrumentation
+        name: mochitest-browser-chrome-instrumentation
     treeherder-symbol: M(inst)
     loopback-video: true
     tier: 3
     run-on-projects:
         by-test-platform:
             windows.*-(?:nightly|shippable)/.*: ["mozilla-central"]
             default: []
     max-run-time: 14400
     mozharness:
         mochitest-flavor: browser
     allow-software-gl-layers: false
 
 browser-screenshots:
     description: "Browser Screenshots"
     suite:
-        name: browser-chrome-screenshots
+        name: mochitest-browser-chrome-screenshots
     treeherder-symbol: M(ss)
     loopback-video: true
     run-on-projects:
         by-test-platform:
             windows7-32(?:-pgo|-shippable)(?:-qr)?/opt: ['mozilla-central', 'integration']
             windows10-64(?:-pgo|-shippable)(?:-qr)?/opt: ['mozilla-central', 'integration']
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central']
             linux64-(?:pgo|shippable)(?:-qr)?/opt: ['mozilla-central', 'integration']
@@ -168,18 +166,16 @@ browser-screenshots:
             default: []
     max-run-time: 3600
     mozharness:
         mochitest-flavor: browser
     allow-software-gl-layers: false
 
 mochitest-chrome:
     description: "Mochitest chrome run"
-    suite:
-        name: chrome
     treeherder-symbol: M(c)
     loopback-video: true
     tier:
         by-test-platform:
             windows10-aarch64.*: 2
             default: default
     instance-size:
         by-test-platform:
@@ -229,17 +225,17 @@ mochitest-clipboard:
     mozharness:
         mochitest-flavor: plain
         extra-options:
             by-test-platform:
                 android-em.*:
                     # note that Android runs fewer suites than other platforms
                     - --test-suite=mochitest-plain-clipboard
                 default:
-                    - --mochitest-suite=plain-clipboard,chrome-clipboard,browser-chrome-clipboard
+                    - --mochitest-suite=mochitest-plain-clipboard,mochitest-chrome-clipboard,mochitest-browser-chrome-clipboard
 
 mochitest-devtools-chrome:
     description: "Mochitest devtools-chrome run"
     suite:
         name: mochitest-devtools-chrome-chunked
     treeherder-symbol: M(dt)
     loopback-video: true
     tier:
@@ -282,17 +278,17 @@ mochitest-devtools-webreplay:
         by-test-platform:
             macosx64/opt: ['mozilla-central', 'try']
             macosx64.*/opt: ['trunk', 'try']
             default: []
 
 mochitest-gpu:
     description: "Mochitest GPU run"
     suite:
-        name: plain-gpu
+        name: mochitest-plain-gpu
     treeherder-symbol: M(gpu)
     loopback-video: true
     tier:
         by-test-platform:
             windows10-aarch64.*: 2
             default: default
     instance-size:
         by-test-platform:
@@ -306,17 +302,17 @@ mochitest-gpu:
     mozharness:
         mochitest-flavor: plain
         extra-options:
             by-test-platform:
                 android-em.*:
                     # note that Android runs fewer suites than other platforms
                     - --test-suite=mochitest-plain-gpu
                 default:
-                    - --mochitest-suite=plain-gpu,chrome-gpu,browser-chrome-gpu
+                    - --mochitest-suite=mochitest-plain-gpu,mochitest-chrome-gpu,mochitest-browser-chrome-gpu
 
 mochitest-media:
     description: "Mochitest media run"
     treeherder-symbol: M(mda)
     max-run-time:
         by-test-platform:
             windows10-64-ccov/debug: 7200
             macosx64-ccov/debug: 7200
@@ -359,17 +355,17 @@ mochitest-media:
         by-test-platform:
             android-hw.*: 2
             windows10-aarch64.*: 2
             default: default
 
 mochitest-plain-headless:
     description: "Mochitest plain headless run"
     suite:
-        name: plain-chunked
+        name: mochitest-plain-chunked
     treeherder-symbol: M(h)
     loopback-video: true
     chunks:
         by-test-platform:
             linux.*/debug: 16
             default: 5
     e10s: true
     max-run-time: 5400
@@ -382,17 +378,17 @@ mochitest-plain-headless:
         mochitest-flavor: plain
         chunked: true
         extra-options:
             - --headless
 
 mochitest-valgrind:
     description: "Mochitest plain Valgrind run"
     suite:
-        name: valgrind-plain
+        name: mochitest-valgrind-plain
     treeherder-symbol: M-V()
     run-on-projects: []
     tier: 3
     loopback-video: true
     chunks: 40
     max-run-time: 14400
     # We could re-enable e10s later.
     # There's no intrinsic reason not to use it.
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -231,18 +231,17 @@ def target_tasks_graphics(full_task_grap
 @_target_task('mochitest_valgrind')
 def target_tasks_valgrind(full_task_graph, parameters, graph_config):
     """Target tasks for mochitest valgrind jobs."""
     def filter(task):
         platform = task.attributes.get('test_platform', '').split('/')[0]
         if platform not in ['linux64']:
             return False
 
-        if task.attributes.get('unittest_suite', '').startswith('mochitest') and \
-           task.attributes.get('unittest_flavor', '').startswith('valgrind-plain'):
+        if task.attributes.get('unittest_suite', '').startswith('mochitest-valgrind-plain'):
             return True
         return False
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('mozilla_beta_tasks')
 def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config):
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -345,17 +345,17 @@ class Test(MachCommandBase):
                     status = res
 
         buckets = {}
         for test in run_tests:
             key = (test['flavor'], test.get('subsuite', ''))
             buckets.setdefault(key, []).append(test)
 
         for (flavor, subsuite), tests in sorted(buckets.items()):
-            m = get_suite_definition(flavor, subsuite)
+            _, m = get_suite_definition(flavor, subsuite)
             if 'mach_command' not in m:
                 substr = '-{}'.format(subsuite) if subsuite else ''
                 print(UNKNOWN_FLAVOR % (flavor, substr))
                 status = 1
                 continue
 
             kwargs = dict(m['kwargs'])
             kwargs['log'] = log
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -25,34 +25,31 @@ MOCHITEST_TOTAL_CHUNKS = 5
 
 def WebglSuite(name):
     return {
         'aliases': (name,),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'plain', 'subsuite': name, 'test_paths': None},
         'task_regex': ['mochitest-' + name + '($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'mochitest-' + name
     }
 
 
 TEST_SUITES = {
     'cppunittest': {
         'aliases': ('cpp',),
         'mach_command': 'cppunittest',
         'kwargs': {'test_file': None},
-        'mozharness_name': 'cppunittest',
     },
     'crashtest': {
         'aliases': ('c', 'rc'),
         'mach_command': 'crashtest',
         'kwargs': {'test_file': None},
         'task_regex': ['crashtest($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'crashtest',
     },
     'firefox-ui-functional': {
         'aliases': ('fxfn',),
         'mach_command': 'firefox-ui-functional',
         'kwargs': {},
     },
     'firefox-ui-update': {
         'aliases': ('fxup',),
@@ -77,121 +74,108 @@ TEST_SUITES = {
         'task_regex': ['marionette($|.*(-1|[^0-9])$)'],
     },
     'mochitest-a11y': {
         'aliases': ('a11y', 'ally'),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'a11y', 'test_paths': None},
         'task_regex': ['mochitest-a11y($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'a11y',
     },
-    'mochitest-browser': {
-        'aliases': ('bc', 'browser-chrome'),
+    'mochitest-browser-chrome': {
+        'aliases': ('bc', 'browser'),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'browser-chrome', 'test_paths': None},
         'task_regex': ['mochitest-browser-chrome($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'browser-chrome',
     },
-    'mochitest-browser-clipboard': {
+    'mochitest-browser-chrome-clipboard': {
         'aliases': ('cl', 'clipboard',),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'browser-chrome', 'subsuite': 'clipboard', 'test_paths': None},
         'task_regex': ['mochitest-clipboard($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'browser-chrome-clipboard',
+    },
+    'mochitest-browser-chrome-screenshots': {
+        'aliases': ('ss', 'screenshots-chrome'),
+        'mach_command': 'mochitest',
+        'kwargs': {'flavor': 'browser-chrome', 'subsuite': 'screenshots', 'test_paths': None},
+        'task_regex': ['browser-screenshots($|.*(-1|[^0-9])$)'],
     },
     'mochitest-chrome': {
         'aliases': ('mc',),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'chrome', 'test_paths': None},
         'task_regex': ['mochitest-chrome($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'chrome',
     },
     'mochitest-chrome-clipboard': {
         'aliases': ('cl', 'clipboard',),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'chrome', 'subsuite': 'clipboard', 'test_paths': None},
         'task_regex': ['mochitest-clipboard($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'chrome-clipboard',
     },
     'mochitest-chrome-gpu': {
         'aliases': ('gpu',),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'chrome', 'subsuite': 'gpu', 'test_paths': None},
         'task_regex': ['mochitest-gpu($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'chrome-gpu',
     },
-    'mochitest-devtools': {
-        'aliases': ('dt', 'devtools-chrome'),
+    'mochitest-devtools-chrome': {
+        'aliases': ('dt', 'devtools'),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'browser-chrome', 'subsuite': 'devtools', 'test_paths': None},
         'task_regex': ['mochitest-devtools-chrome($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'mochitest-devtools-chrome',
     },
     'mochitest-media': {
         'aliases': ('mpm', 'plain-media'),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'plain', 'subsuite': 'media', 'test_paths': None},
         'task_regex': ['mochitest-media($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'mochitest-media',
     },
     'mochitest-plain': {
         'aliases': ('mp', 'plain',),
         'mach_command': 'mochitest',
         'kwargs': {'flavor': 'plain', 'test_paths': None},
         'task_regex': ['mochitest(?!-a11y|-browser|-chrome|-clip|-devtools|-gpu|-harness|-media|-screen|-webgl)($|.*(-1|[^0-9])$)',  # noqa
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'plain',
     },
     'mochitest-plain-clipboard': {
         'aliases': ('cl', 'clipboard',),
         'mach_command': 'mochitest',
         'kwargs': {'subsuite': 'clipboard', 'test_paths': None},
         'task_regex': ['mochitest-clipboard($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'plain-clipboard',
     },
     'mochitest-plain-gpu': {
         'aliases': ('gpu',),
         'mach_command': 'mochitest',
         'kwargs': {'subsuite': 'gpu', 'test_paths': None},
         'task_regex': ['mochitest-gpu($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'plain-gpu',
-    },
-    'mochitest-screenshots': {
-        'aliases': ('ss', 'screenshots-chrome'),
-        'mach_command': 'mochitest',
-        'kwargs': {'flavor': 'browser-chrome', 'subsuite': 'screenshots', 'test_paths': None},
-        'task_regex': ['browser-screenshots($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'browser-chrome-screenshots',
     },
     'mochitest-webgl1-core': WebglSuite('webgl1-core'),
     'mochitest-webgl1-ext': WebglSuite('webgl1-ext'),
     'mochitest-webgl2-core': WebglSuite('webgl2-core'),
     'mochitest-webgl2-ext': WebglSuite('webgl2-ext'),
     'mochitest-webgl2-deqp': WebglSuite('webgl2-deqp'),
     'python': {
         'mach_command': 'python-test',
         'kwargs': {'tests': None},
     },
     'reftest': {
         'aliases': ('rr',),
         'mach_command': 'reftest',
         'kwargs': {'tests': None},
         'task_regex': ['(opt|debug)-reftest($|.*(-1|[^0-9])$)',
                        'test-verify-gpu($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'reftest',
     },
     'robocop': {
         'mach_command': 'robocop',
         'kwargs': {'test_paths': None},
         'task_regex': ['robocop($|.*(-1|[^0-9])$)'],
     },
     'web-platform-tests': {
         'aliases': ('wpt',),
@@ -227,17 +211,16 @@ TEST_SUITES = {
         'kwargs': {},
     },
     'xpcshell': {
         'aliases': ('x',),
         'mach_command': 'xpcshell-test',
         'kwargs': {'test_file': 'all'},
         'task_regex': ['xpcshell($|.*(-1|[^0-9])$)',
                        'test-verify($|.*(-1|[^0-9])$)'],
-        'mozharness_name': 'xpcshell',
     },
 }
 
 for i in range(1, MOCHITEST_TOTAL_CHUNKS + 1):
     TEST_SUITES['mochitest-%d' % i] = {
         'aliases': ('m%d' % i,),
         'mach_command': 'mochitest',
         'kwargs': {
@@ -247,35 +230,35 @@ for i in range(1, MOCHITEST_TOTAL_CHUNKS
             'total_chunks': MOCHITEST_TOTAL_CHUNKS,
             'this_chunk': i,
             'test_paths': None,
         },
     }
 
 _test_flavors = {
     'a11y': 'mochitest-a11y',
-    'browser-chrome': 'mochitest-browser',
+    'browser-chrome': 'mochitest-browser-chrome',
     'chrome': 'mochitest-chrome',
     'crashtest': 'crashtest',
     'firefox-ui-functional': 'firefox-ui-functional',
     'firefox-ui-update': 'firefox-ui-update',
     'marionette': 'marionette',
     'mochitest': 'mochitest-plain',
     'python': 'python',
     'reftest': 'reftest',
     'steeplechase': '',
     'web-platform-tests': 'web-platform-tests',
     'xpcshell': 'xpcshell',
 }
 
 _test_subsuites = {
-    ('browser-chrome', 'clipboard'): 'mochitest-browser-clipboard',
-    ('browser-chrome', 'devtools'): 'mochitest-devtools',
-    ('browser-chrome', 'gpu'): 'mochitest-browser-gpu',
-    ('browser-chrome', 'screenshots'): 'mochitest-screenshots',
+    ('browser-chrome', 'clipboard'): 'mochitest-browser-chrome-clipboard',
+    ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome',
+    ('browser-chrome', 'gpu'): 'mochitest-browser-chrome-gpu',
+    ('browser-chrome', 'screenshots'): 'mochitest-browser-chrome-screenshots',
     ('chrome', 'clipboard'): 'mochitest-chrome-clipboard',
     ('chrome', 'gpu'): 'mochitest-chrome-gpu',
     ('mochitest', 'clipboard'): 'mochitest-plain-clipboard',
     ('mochitest', 'gpu'): 'mochitest-plain-gpu',
     ('mochitest', 'media'): 'mochitest-media',
     ('mochitest', 'robocop'): 'robocop',
     ('mochitest', 'webgl1-core'): 'mochitest-webgl1-core',
     ('mochitest', 'webgl1-ext'): 'mochitest-webgl1-ext',
@@ -295,30 +278,30 @@ def get_suite_definition(flavor, subsuit
     Otherwise, the entry for 'flavor' will be returned with the 'subsuite'
     keyword arg set.
 
     With or without strict mode, an empty dict will be returned if no
     matching suite definition was found.
     """
     if not subsuite:
         suite_name = _test_flavors.get(flavor)
-        return TEST_SUITES.get(suite_name, {}).copy()
+        return suite_name, TEST_SUITES.get(suite_name, {}).copy()
 
     suite_name = _test_subsuites.get((flavor, subsuite))
     if suite_name or strict:
-        return TEST_SUITES.get(suite_name, {}).copy()
+        return suite_name, TEST_SUITES.get(suite_name, {}).copy()
 
     suite_name = _test_flavors.get(flavor)
     if suite_name not in TEST_SUITES:
-        return {}
+        return suite_name, {}
 
     suite = TEST_SUITES[suite_name].copy()
     suite.setdefault('kwargs', {})
     suite['kwargs']['subsuite'] = subsuite
-    return suite
+    return suite_name, suite
 
 
 def rewrite_test_base(test, new_base, honor_install_to_subdir=False):
     """Rewrite paths in a test to be under a new base path.
 
     This is useful for running tests from a separate location from where they
     were defined.
 
--- a/testing/mozbase/moztest/tests/test_resolve.py
+++ b/testing/mozbase/moztest/tests/test_resolve.py
@@ -367,55 +367,55 @@ class TestTestResolver(Base):
             path = t['file_relpath']
             self.assertTrue(path.startswith('accessible') or path.endswith('.js'))
 
     def test_resolve_metadata(self):
         """Test finding metadata from outgoing files."""
         r = self._get_resolver()
 
         suites, tests = r.resolve_metadata(['bc'])
-        assert suites == {'mochitest-browser'}
+        assert suites == {'mochitest-browser-chrome'}
         assert tests == []
 
-        suites, tests = r.resolve_metadata(['mochitest-a11y', 'browser', 'xpcshell'])
+        suites, tests = r.resolve_metadata(['mochitest-a11y', '/browser', 'xpcshell'])
         assert suites == {'mochitest-a11y', 'xpcshell'}
         assert sorted(t['file_relpath'] for t in tests) == [
             'devtools/client/markupview/test/browser_markupview_copy_image_data.js',
             'image/test/browser/browser_bug666317.js',
             'mobile/android/tests/browser/junit3/src/TestDistribution.java',
         ]
 
     def test_task_regexes(self):
         """Test the task_regexes defined in TEST_SUITES."""
 
         test_cases = {
-            'mochitest-browser': [
+            'mochitest-browser-chrome': [
                 'test-linux64/opt-mochitest-browser-chrome',
                 'test-linux64/opt-mochitest-browser-chrome-e10s',
             ],
             'mochitest-chrome': [
                 'test-linux64/opt-mochitest-chrome',
             ],
-            'mochitest-devtools': [
+            'mochitest-devtools-chrome': [
                 'test-linux64/opt-mochitest-devtools-chrome',
             ],
             'mochitest-media': [
                 'test-linux64/opt-mochitest-media-e10s-1',
             ],
             'mochitest-plain': [
                 'test-linux64/opt-mochitest',
                 'test-linux64/debug-mochitest-e10s',
                 # this isn't a real task but the regex would match it if it were
                 'test-linux64/opt-mochitest-plain',
             ],
             'mochitest-plain-gpu': [
                 'test-linux64/opt-mochitest-gpu',
                 'test-linux64/opt-mochitest-gpu-e10s',
             ],
-            'mochitest-screenshots': [
+            'mochitest-browser-chrome-screenshots': [
                 'test-linux64/opt-browser-screenshots-1',
                 'test-linux64/opt-browser-screenshots-e10s-1',
             ],
             'reftest': [
                 'test-linux64/opt-reftest',
                 'test-linux64/debug-reftest-e10s-1',
             ],
             'robocop': [
--- a/testing/mozharness/configs/android/android_common.py
+++ b/testing/mozharness/configs/android/android_common.py
@@ -53,17 +53,17 @@ config = {
     "marionette_address": "localhost:2828",
     "marionette_test_manifest": "unit-tests.ini",
     # To take device screenshots at timed intervals (each time in seconds, relative
     # to the start of the run-tests step) specify screenshot_times. For example, to
     # take 4 screenshots at one minute intervals you could specify:
     # "screenshot_times": [60, 120, 180, 240],
 
     "suite_definitions": {
-        "mochitest": {
+        "mochitest-plain": {
             "run_filename": "runtestsremote.py",
             "testsdir": "mochitest",
             "options": [
                 "--app=%(app)s",
                 "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s",
                 "--utility-path=%(utility_path)s",
                 "--http-port=%(http_port)s",
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -146,45 +146,45 @@ config = {
                 "--utility-path=tests/bin",
                 "%(binary_path)s",
             ],
             "run_filename": "rungtests.py",
         },
     },
     # local mochi suites
     "all_mochitest_suites": {
-        "valgrind-plain": ["--valgrind=/usr/bin/valgrind",
+        "mochitest-valgrind-plain": ["--valgrind=/usr/bin/valgrind",
                            "--valgrind-supp-files=" + VALGRIND_SUPP_ARCH +
                                "," + VALGRIND_SUPP_CROSS_ARCH,
                            "--timeout=900", "--max-timeouts=50"],
-        "plain": [],
-        "plain-gpu": ["--subsuite=gpu"],
-        "plain-clipboard": ["--subsuite=clipboard"],
-        "plain-chunked": ["--chunk-by-dir=4"],
-        "plain-chunked-coverage": ["--chunk-by-dir=4", "--timeout=1200"],
+        "mochitest-plain": [],
+        "mochitest-plain-gpu": ["--subsuite=gpu"],
+        "mochitest-plain-clipboard": ["--subsuite=clipboard"],
+        "mochitest-plain-chunked": ["--chunk-by-dir=4"],
+        "mochitest-plain-chunked-coverage": ["--chunk-by-dir=4", "--timeout=1200"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--flavor=chrome"],
-        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--flavor=browser"],
-        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
-        "browser-chrome-coverage": ["--flavor=browser", "--chunk-by-runtime", "--timeout=1200"],
-        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
-        "browser-chrome-instrumentation": ["--flavor=browser"],
+        "mochitest-chrome": ["--flavor=chrome"],
+        "mochitest-chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "mochitest-chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "mochitest-chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "mochitest-browser-chrome": ["--flavor=browser"],
+        "mochitest-browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "mochitest-browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "mochitest-browser-chrome-coverage": ["--flavor=browser", "--chunk-by-runtime", "--timeout=1200"],
+        "mochitest-browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
+        "mochitest-browser-chrome-instrumentation": ["--flavor=browser"],
         "mochitest-webgl1-core": ["--subsuite=webgl1-core"],
         "mochitest-webgl1-ext": ["--subsuite=webgl1-ext"],
         "mochitest-webgl2-core": ["--subsuite=webgl2-core"],
         "mochitest-webgl2-ext": ["--subsuite=webgl2-ext"],
         "mochitest-webgl2-deqp": ["--subsuite=webgl2-deqp"],
         "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
         "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
         "mochitest-devtools-chrome-coverage": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime", "--timeout=1200"],
-        "a11y": ["--flavor=a11y"],
+        "mochitest-a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "crashtest": {
             "options": ["--suite=crashtest"],
             "tests": ["tests/reftest/tests/testing/crashtest/crashtests.list"]
         },
         "jsreftest": {
--- a/testing/mozharness/configs/unittests/mac_unittest.py
+++ b/testing/mozharness/configs/unittests/mac_unittest.py
@@ -116,39 +116,39 @@ config = {
                 "--utility-path=tests/bin",
                 "%(binary_path)s",
             ],
             "run_filename": "rungtests.py",
         },
     },
     # local mochi suites
     "all_mochitest_suites": {
-        "plain": [],
-        "plain-gpu": ["--subsuite=gpu"],
-        "plain-clipboard": ["--subsuite=clipboard"],
-        "plain-chunked": ["--chunk-by-dir=4"],
+        "mochitest-plain": [],
+        "mochitest-plain-gpu": ["--subsuite=gpu"],
+        "mochitest-plain-clipboard": ["--subsuite=clipboard"],
+        "mochitest-plain-chunked": ["--chunk-by-dir=4"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--flavor=chrome"],
-        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--flavor=browser"],
-        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
-        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
-        "browser-chrome-instrumentation": ["--flavor=browser"],
+        "mochitest-chrome": ["--flavor=chrome"],
+        "mochitest-chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "mochitest-chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "mochitest-chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "mochitest-browser-chrome": ["--flavor=browser"],
+        "mochitest-browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "mochitest-browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "mochitest-browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
+        "mochitest-browser-chrome-instrumentation": ["--flavor=browser"],
         "mochitest-webgl1-core": ["--subsuite=webgl1-core"],
         "mochitest-webgl1-ext": ["--subsuite=webgl1-ext"],
         "mochitest-webgl2-core": ["--subsuite=webgl2-core"],
         "mochitest-webgl2-ext": ["--subsuite=webgl2-ext"],
         "mochitest-webgl2-deqp": ["--subsuite=webgl2-deqp"],
         "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
         "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
-        "mochitest-devtools-webreplay": ["--flavor=browser", "--subsuite=devtools-webreplay"],
-        "a11y": ["--flavor=a11y"],
+        "mochitest-devtools-chrome-webreplay": ["--flavor=browser", "--subsuite=devtools-webreplay"],
+        "mochitest-a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "crashtest": {
             'options': ["--suite=crashtest"],
             'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
         },
         "jsreftest": {
--- a/testing/mozharness/configs/unittests/win_unittest.py
+++ b/testing/mozharness/configs/unittests/win_unittest.py
@@ -139,38 +139,38 @@ config = {
                 "%(binary_path)s",
             ],
             "run_filename": "rungtests.py",
         },
     },
     # local mochi suites
     "all_mochitest_suites":
     {
-        "plain": [],
-        "plain-gpu": ["--subsuite=gpu"],
-        "plain-clipboard": ["--subsuite=clipboard"],
-        "plain-chunked": ["--chunk-by-dir=4"],
+        "mochitest-plain": [],
+        "mochitest-plain-gpu": ["--subsuite=gpu"],
+        "mochitest-plain-clipboard": ["--subsuite=clipboard"],
+        "mochitest-plain-chunked": ["--chunk-by-dir=4"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--flavor=chrome"],
-        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--flavor=browser"],
-        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
-        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
-        "browser-chrome-instrumentation": ["--flavor=browser"],
+        "mochitest-chrome": ["--flavor=chrome"],
+        "mochitest-chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "mochitest-chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "mochitest-chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "mochitest-browser-chrome": ["--flavor=browser"],
+        "mochitest-browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "mochitest-browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "mochitest-browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
+        "mochitest-browser-chrome-instrumentation": ["--flavor=browser"],
         "mochitest-webgl1-core": ["--subsuite=webgl1-core"],
         "mochitest-webgl1-ext": ["--subsuite=webgl1-ext"],
         "mochitest-webgl2-core": ["--subsuite=webgl2-core"],
         "mochitest-webgl2-ext": ["--subsuite=webgl2-ext"],
         "mochitest-webgl2-deqp": ["--subsuite=webgl2-deqp"],
         "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
         "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
-        "a11y": ["--flavor=a11y"],
+        "mochitest-a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "crashtest": {
             'options': ["--suite=crashtest"],
             'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
         },
         "jsreftest": {
--- a/testing/mozharness/mozharness/mozilla/testing/errors.py
+++ b/testing/mozharness/mozharness/mozilla/testing/errors.py
@@ -21,24 +21,24 @@ from mozharness.base.log import INFO, WA
 _mochitest_summary = {
     'regex': re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))'''),  # NOQA: E501
     'pass_group': "Passed",
     'fail_group': "Failed",
     'known_fail_group': "Todo",
 }
 
 TinderBoxPrintRe = {
-    "mochitest_summary": _mochitest_summary,
     "mochitest-chrome_summary": _mochitest_summary,
     "mochitest-webgl1-core_summary": _mochitest_summary,
     "mochitest-webgl1-ext_summary": _mochitest_summary,
     "mochitest-webgl2-core_summary": _mochitest_summary,
     "mochitest-webgl2-ext_summary": _mochitest_summary,
     "mochitest-webgl2-deqp_summary": _mochitest_summary,
     "mochitest-media_summary": _mochitest_summary,
+    "mochitest-plain_summary": _mochitest_summary,
     "mochitest-plain-clipboard_summary": _mochitest_summary,
     "mochitest-plain-gpu_summary": _mochitest_summary,
     "marionette_summary": {
         'regex': re.compile(r'''(passed|failed|todo):\ +(\d+)'''),
         'pass_group': "passed",
         'fail_group': "failed",
         'known_fail_group': "todo",
     },
--- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
+++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
@@ -39,21 +39,21 @@ class SingleTestMixin(FetchesMixin):
 
     def _is_gpu_suite(self, suite):
         if suite and (suite == 'gpu' or suite.startswith('webgl')):
             return True
         return False
 
     def _find_misc_tests(self, dirs, changed_files, gpu=False):
         manifests = [
-            (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'),
-            (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'),
+            (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'mochitest-plain'),
+            (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'mochitest-chrome'),
             (os.path.join(dirs['abs_mochitest_dir'], 'browser',
-                          'browser-chrome.ini'), 'browser-chrome'),
-            (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'),
+                          'browser-chrome.ini'), 'mochitest-browser-chrome'),
+            (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'mochitest-a11y'),
             (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'),
         ]
         tests_by_path = {}
         all_disabled = []
         for (path, suite) in manifests:
             if os.path.exists(path):
                 man = TestManifest([path], strict=False)
                 active = man.active_tests(exists=False, disabled=True, filters=[], **mozinfo.info)
@@ -134,30 +134,30 @@ class SingleTestMixin(FetchesMixin):
                 continue
 
             self.info("Per-test run found test %s (%s/%s)" % (file, entry[0], entry[1]))
             subsuite_mapping = {
                 # Map (<suite>, <subsuite>): <full-suite>
                 #   <suite> is associated with a manifest, explicitly in code above
                 #   <subsuite> comes from "subsuite" tags in some manifest entries
                 #   <full-suite> is a unique id for the suite, matching desktop mozharness configs
-                ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard',
-                ('chrome', 'clipboard'): 'chrome-clipboard',
-                ('plain', 'clipboard'): 'plain-clipboard',
-                ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome',
-                ('browser-chrome', 'screenshots'): 'browser-chrome-screenshots',
-                ('plain', 'media'): 'mochitest-media',
+                ('mochitest-browser-chrome', 'clipboard'): 'mochitest-browser-chrome-clipboard',
+                ('mochitest-chrome', 'clipboard'): 'mochitest-chrome-clipboard',
+                ('mochitest-plain', 'clipboard'): 'mochitest-plain-clipboard',
+                ('mochitest-browser-chrome', 'devtools'): 'mochitest-devtools-chrome',
+                ('mochitest-browser-chrome', 'screenshots'): 'mochitest-browser-chrome-screenshots',  # noqa
+                ('mochitest-plain', 'media'): 'mochitest-media',
                 # below should be on test-verify-gpu job
-                ('chrome', 'gpu'): 'chrome-gpu',
-                ('plain', 'gpu'): 'plain-gpu',
-                ('plain', 'webgl1-core'): 'mochitest-webgl1-core',
-                ('plain', 'webgl1-ext'): 'mochitest-webgl1-ext',
-                ('plain', 'webgl2-core'): 'mochitest-webgl2-core',
-                ('plain', 'webgl2-ext'): 'mochitest-webgl2-ext',
-                ('plain', 'webgl2-deqp'): 'mochitest-webgl2-deqp',
+                ('mochitest-chrome', 'gpu'): 'mochitest-chrome-gpu',
+                ('mochitest-plain', 'gpu'): 'mochitest-plain-gpu',
+                ('mochitest-plain', 'webgl1-core'): 'mochitest-webgl1-core',
+                ('mochitest-plain', 'webgl1-ext'): 'mochitest-webgl1-ext',
+                ('mochitest-plain', 'webgl2-core'): 'mochitest-webgl2-core',
+                ('mochitest-plain', 'webgl2-ext'): 'mochitest-webgl2-ext',
+                ('mochitest-plain', 'webgl2-deqp'): 'mochitest-webgl2-deqp',
             }
             if entry in subsuite_mapping:
                 suite = subsuite_mapping[entry]
             else:
                 suite = entry[0]
             suite_files = self.suites.get(suite)
             if not suite_files:
                 suite_files = []
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -309,16 +309,17 @@ You can set this by specifying --test-ur
     def _download_test_packages(self, suite_categories, extract_dirs):
         # Some platforms define more suite categories/names than others.
         # This is a difference in the convention of the configs more than
         # to how these tests are run, so we pave over these differences here.
         aliases = {
             'robocop': 'mochitest',
             'mochitest-chrome': 'mochitest',
             'mochitest-media': 'mochitest',
+            'mochitest-plain': 'mochitest',
             'mochitest-plain-clipboard': 'mochitest',
             'mochitest-plain-gpu': 'mochitest',
             'mochitest-webgl1-core': 'mochitest',
             'mochitest-webgl1-ext': 'mochitest',
             'mochitest-webgl2-core': 'mochitest',
             'mochitest-webgl2-ext': 'mochitest',
             'mochitest-webgl2-deqp': 'mochitest',
             'geckoview': 'mochitest',
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -4,17 +4,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import copy
 import datetime
 import json
 import os
-import re
 import sys
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.log import FATAL
 from mozharness.base.script import BaseScript, PreScriptAction
@@ -131,23 +130,16 @@ class AndroidEmulatorTest(TestingMixin, 
         self.installer_path = c.get('installer_path')
         self.test_url = c.get('test_url')
         self.test_packages_url = c.get('test_packages_url')
         self.test_manifest = c.get('test_manifest')
         self.robocop_path = os.path.join(abs_dirs['abs_work_dir'], "robocop.apk")
         self.test_suite = c.get('test_suite')
         self.this_chunk = c.get('this_chunk')
         self.total_chunks = c.get('total_chunks')
-        if self.test_suite and self.test_suite not in self.config["suite_definitions"]:
-            # accept old-style test suite name like "mochitest-3"
-            m = re.match("(.*)-(\d*)", self.test_suite)
-            if m:
-                self.test_suite = m.group(1)
-                if self.this_chunk is None:
-                    self.this_chunk = m.group(2)
         self.xre_path = None
         self.device_serial = 'emulator-5554'
         self.log_raw_level = c.get('log_raw_level')
         self.log_tbpl_level = c.get('log_tbpl_level')
         self.e10s = c.get('e10s')
         self.enable_webrender = c.get('enable_webrender')
         self.disable_webrender = c.get('disable_webrender')
         if self.enable_webrender:
@@ -315,21 +307,21 @@ class AndroidEmulatorTest(TestingMixin, 
 
     def _query_suites(self):
         if self.test_suite:
             return [(self.test_suite, self.test_suite)]
         # per-test mode: determine test suites to run
 
         # For each test category, provide a list of supported sub-suites and a mapping
         # between the per_test_base suite name and the android suite name.
-        all = [('mochitest', {'plain': 'mochitest',
-                              'chrome': 'mochitest-chrome',
+        all = [('mochitest', {'mochitest-plain': 'mochitest-plain',
+                              'mochitest-chrome': 'mochitest-chrome',
                               'mochitest-media': 'mochitest-media',
-                              'plain-clipboard': 'mochitest-plain-clipboard',
-                              'plain-gpu': 'mochitest-plain-gpu'}),
+                              'mochitest-plain-clipboard': 'mochitest-plain-clipboard',
+                              'mochitest-plain-gpu': 'mochitest-plain-gpu'}),
                ('reftest', {'reftest': 'reftest',
                             'crashtest': 'crashtest',
                             'jsreftest': 'jsreftest'}),
                ('xpcshell', {'xpcshell': 'xpcshell'})]
         suites = []
         for (category, all_suites) in all:
             cat_suites = self.query_per_test_category_suites(category, all_suites)
             for k in cat_suites.keys():
--- a/testing/mozharness/scripts/android_hardware_unittest.py
+++ b/testing/mozharness/scripts/android_hardware_unittest.py
@@ -4,17 +4,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import copy
 import datetime
 import json
 import os
-import re
 import sys
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.log import FATAL
 from mozharness.base.script import BaseScript, PreScriptAction
@@ -107,23 +106,16 @@ class AndroidHardwareTest(TestingMixin, 
         self.installer_path = c.get('installer_path')
         self.test_url = c.get('test_url')
         self.test_packages_url = c.get('test_packages_url')
         self.test_manifest = c.get('test_manifest')
         self.robocop_path = os.path.join(abs_dirs['abs_work_dir'], "robocop.apk")
         self.test_suite = c.get('test_suite')
         self.this_chunk = c.get('this_chunk')
         self.total_chunks = c.get('total_chunks')
-        if self.test_suite and self.test_suite not in self.config["suite_definitions"]:
-            # accept old-style test suite name like "mochitest-3"
-            m = re.match("(.*)-(\d*)", self.test_suite)
-            if m:
-                self.test_suite = m.group(1)
-                if self.this_chunk is None:
-                    self.this_chunk = m.group(2)
         self.xre_path = None
         self.log_raw_level = c.get('log_raw_level')
         self.log_tbpl_level = c.get('log_tbpl_level')
         self.e10s = c.get('e10s')
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
@@ -261,20 +253,20 @@ class AndroidHardwareTest(TestingMixin, 
                 try_tests))
 
         return cmd
 
     def _query_suites(self):
         if self.test_suite:
             return [(self.test_suite, self.test_suite)]
         # per-test mode: determine test suites to run
-        all = [('mochitest', {'plain': 'mochitest',
-                              'chrome': 'mochitest-chrome',
-                              'plain-clipboard': 'mochitest-plain-clipboard',
-                              'plain-gpu': 'mochitest-plain-gpu'}),
+        all = [('mochitest', {'mochitest-plain': 'mochitest-plain',
+                              'mochitest-chrome': 'mochitest-chrome',
+                              'mochitest-plain-clipboard': 'mochitest-plain-clipboard',
+                              'mochitest-plain-gpu': 'mochitest-plain-gpu'}),
                ('reftest', {'reftest': 'reftest', 'crashtest': 'crashtest'}),
                ('xpcshell', {'xpcshell': 'xpcshell'})]
         suites = []
         for (category, all_suites) in all:
             cat_suites = self.query_per_test_category_suites(category, all_suites)
             for k in cat_suites.keys():
                 suites.append((k, cat_suites[k]))
         return suites
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -100,17 +100,17 @@ def generate_tasks(params, full, root):
 
 def filter_tasks_by_paths(tasks, paths):
     resolver = TestResolver.from_environment(cwd=here)
     run_suites, run_tests = resolver.resolve_metadata(paths)
     flavors = set([(t['flavor'], t.get('subsuite')) for t in run_tests])
 
     task_regexes = set()
     for flavor, subsuite in flavors:
-        suite = get_suite_definition(flavor, subsuite, strict=True)
+        _, suite = get_suite_definition(flavor, subsuite, strict=True)
         if 'task_regex' not in suite:
             print("warning: no tasks could be resolved from flavor '{}'{}".format(
                     flavor, " and subsuite '{}'".format(subsuite) if subsuite else ""))
             continue
 
         task_regexes.update(suite['task_regex'])
 
     def match_task(task):
@@ -120,22 +120,12 @@ def filter_tasks_by_paths(tasks, paths):
 
 
 def resolve_tests_by_suite(paths):
     resolver = TestResolver.from_environment(cwd=here)
     _, run_tests = resolver.resolve_metadata(paths)
 
     suite_to_tests = defaultdict(list)
     for test in run_tests:
-        suite = get_suite_definition(test['flavor'], test.get('subsuite'), strict=True)
-
-        if 'mozharness_name' in suite:
-            key = suite['mozharness_name']
-        else:
-            key = test['flavor']
-            subsuite = test.get('subsuite')
-
-            if subsuite:
-                key += '-' + subsuite
-
+        key, _ = get_suite_definition(test['flavor'], test.get('subsuite'), strict=True)
         suite_to_tests[key].append(test['srcdir_relpath'])
 
     return suite_to_tests
--- a/tools/tryselect/test/test_mozharness_integration.py
+++ b/tools/tryselect/test/test_mozharness_integration.py
@@ -20,44 +20,39 @@ MOZHARNESS_SCRIPTS = {
             'android/android_common.py',
         ],
         'xfail': [
             'cppunittest',
             'gtest',
             'geckoview-junit',
             'jittest',
             'jsreftest',
-            'mochitest',
-            'mochitest-chrome',
-            'mochitest-plain-clipboard',
-            'mochitest-plain-gpu',
-            'robocop',
         ],
     },
     'desktop_unittest': {
         'class_name': 'DesktopUnittest',
         'configs': [
             'unittests/linux_unittest.py',
             'unittests/mac_unittest.py',
             'unittests/win_unittest.py',
         ],
         'xfail': [
-            'browser-chrome-instrumentation',
             'cppunittest',
             'gtest',
             'jittest',
             'jittest-chunked',
             'jittest1',
             'jittest2',
             'jsreftest',
-            'mochitest-devtools-webreplay',
+            'mochitest-browser-chrome-instrumentation',
+            'mochitest-devtools-chrome-webreplay',
+            'mochitest-valgrind-plain',
             'reftest-gpu',
             'reftest-no-accel',
             'reftest-qr',
-            'valgrind-plain',
         ],
     },
 }
 """A suite being listed in a script's `xfail` list  means it won't work
 properly with MOZHARNESS_TEST_PATHS (the mechanism |mach try fuzzy <path>|
 uses).
 """
 
--- a/tools/tryselect/test/test_tasks.py
+++ b/tools/tryselect/test/test_tasks.py
@@ -26,14 +26,14 @@ def test_resolve_tests_by_suite(patch_re
     }
 
     patch_resolver([], [
         {'flavor': 'xpcshell', 'srcdir_relpath': 'xpcshell.js'},
         {'flavor': 'mochitest', 'srcdir_relpath': 'mochitest.js'},
     ])
     assert resolve_tests_by_suite(['xpcshell.js', 'mochitest.js']) == {
         'xpcshell': ['xpcshell.js'],
-        'plain': ['mochitest.js'],
+        'mochitest-plain': ['mochitest.js'],
     }
 
 
 if __name__ == '__main__':
     mozunit.main()