Bug 1489100 - Specify tests per suite in MOZHARNESS_TEST_PATHS. r=ahal
authorMarco Castelluccio <mcastelluccio@mozilla.com>
Thu, 15 Nov 2018 17:54:13 +0100
changeset 504854 920c0a874e738a781a087faba7e170aaed2f1fa3
parent 504853 5f2c6354443e6d41f9ef41386ff2aaa033180b41
child 504855 c0caa96ca64a9f9c5538d88b748208ca75233d60
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1489100
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1489100 - Specify tests per suite in MOZHARNESS_TEST_PATHS. r=ahal
python/mozbuild/mozbuild/backend/test_manifest.py
taskcluster/taskgraph/actions/backfill.py
taskcluster/taskgraph/util/perfile.py
testing/mozbase/moztest/moztest/resolve.py
testing/mozharness/mozharness/mozilla/testing/per_test_base.py
testing/mozharness/scripts/android_emulator_unittest.py
testing/mozharness/scripts/android_hardware_unittest.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/marionette.py
testing/mozharness/scripts/web_platform_tests.py
tools/tryselect/docs/selectors/fuzzy.rst
tools/tryselect/selectors/coverage.py
tools/tryselect/tasks.py
tools/tryselect/templates.py
tools/tryselect/test/conftest.py
tools/tryselect/test/test_tasks.py
tools/tryselect/test/test_templates.py
--- a/python/mozbuild/mozbuild/backend/test_manifest.py
+++ b/python/mozbuild/mozbuild/backend/test_manifest.py
@@ -73,16 +73,17 @@ class TestManifestBackend(PartialBackend
         t['flavor'] = flavor
 
         path = mozpath.normpath(t['path'])
         assert mozpath.basedir(path, [topsrcdir])
 
         key = path[len(topsrcdir)+1:]
         t['file_relpath'] = key
         t['dir_relpath'] = mozpath.dirname(key)
+        t['srcdir_relpath'] = key
 
         self.tests_by_path[key].append(t)
 
     def add_defaults(self, manifest):
         if not hasattr(manifest, 'manifest_defaults'):
             return
         for sub_manifest, defaults in manifest.manifest_defaults.items():
             self.manifest_defaults[sub_manifest] = defaults
--- a/taskcluster/taskgraph/actions/backfill.py
+++ b/taskcluster/taskgraph/actions/backfill.py
@@ -1,16 +1,17 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import json
 import logging
 
 import requests
 from requests.exceptions import HTTPError
 
 from .registry import register_callback_action
 from .util import find_decision_task, create_tasks, combine_task_graph_files
 from taskgraph.util.taskcluster import get_artifact_from_index
@@ -135,17 +136,20 @@ def backfill_action(parameters, graph_co
                                    '--this-chunk=1']
                     if is_android:
                         # no --e10s; todo, what about future geckoView?
                         verify_args.remove('--e10s')
 
                     if gpu_required:
                         verify_args.append('--gpu-required')
 
-                    task.task['payload']['env']['MOZHARNESS_TEST_PATHS'] = input.get('testPath')
+                    if 'testPath' in input:
+                        task.task['payload']['env']['MOZHARNESS_TEST_PATHS'] = json.dums({
+                            task.task['extra']['suite']['flavor']: input['testPath']
+                        })
 
                     cmd_parts = task.task['payload']['command']
                     keep_args = ['--installer-url', '--download-symbols', '--test-packages-url']
                     cmd_parts = remove_args_from_command(cmd_parts, preamble_length, keep_args)
                     cmd_parts = add_args_to_command(cmd_parts, verify_args)
                     task.task['payload']['command'] = cmd_parts
 
                     # morph the task label to a test-verify job
--- a/taskcluster/taskgraph/util/perfile.py
+++ b/taskcluster/taskgraph/util/perfile.py
@@ -1,14 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import itertools
+import json
 import logging
 import math
 
 from mozbuild.util import memoize
 from mozpack.path import match as mozpackmatch
 from taskgraph import files_changed
 import taskgraph
 from .. import GECKO
@@ -42,27 +44,27 @@ def perfile_number_of_chunks(is_try, try
                          'js/src/tests/test/**',
                          'js/src/tests/non262/**',
                          'js/src/tests/test262/**']
     else:
         # Returning 0 means no tests to run, this captures non test-verify tasks
         return 1
 
     changed_files = set()
-    specified_files = []
     if try_task_config:
-        specified_files = try_task_config.split(":")
+        suite_to_paths = json.loads(try_task_config)
+        specified_files = itertools.chain.from_iterable(suite_to_paths.values())
+        changed_files.update(specified_files)
 
     if is_try:
         changed_files.update(files_changed.get_locally_changed_files(GECKO))
     else:
         changed_files.update(files_changed.get_changed_files(head_repository,
                                                              head_rev))
 
-    changed_files.update(specified_files)
     test_count = 0
     for pattern in file_patterns:
         for path in changed_files:
             # TODO: consider running tests if a manifest changes
             if path.endswith('.list') or path.endswith('.ini'):
                 continue
             if path.endswith('^headers^'):
                 continue
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -482,16 +482,17 @@ class TestMetadata(object):
                         "here": os.path.dirname(path),
                         "manifest": data["manifest_path"],
                         "name": test.id,
                         "file_relpath": path,
                         "head": "",
                         "support-files": "",
                         "subsuite": test_type,
                         "dir_relpath": os.path.dirname(src_path),
+                        "srcdir_relpath": src_path,
                         })
 
         self._wpt_loaded = True
 
 
 class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
--- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
+++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
@@ -1,15 +1,17 @@
 #!/usr/bin/env python
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
+import itertools
+import json
 import math
 import os
 import posixpath
 import re
 import sys
 import mozinfo
 from manifestparser import TestManifest
 
@@ -233,17 +235,18 @@ class SingleTestMixin(FetchesMixin):
         mozinfo.update({"headless": headless})
         # FIXME(emilio): Need to update test expectations.
         mozinfo.update({'stylo': True})
         mozinfo.update({'verify': True})
         self.info("Per-test run using mozinfo: %s" % str(mozinfo.info))
 
         changed_files = set()
         if os.environ.get('MOZHARNESS_TEST_PATHS', None) is not None:
-            changed_files |= set(os.environ['MOZHARNESS_TEST_PATHS'].split(':'))
+            suite_to_paths = json.loads(os.environ['MOZHARNESS_TEST_PATHS'])
+            changed_files |= itertools.chain.from_iterable(suite_to_paths.values())
             self.info("Per-test run found explicit request in MOZHARNESS_TEST_PATHS:")
             self.info(str(changed_files))
         else:
             # determine which files were changed on this push
             url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
             contents = self.retry(get_automationrelevance, attempts=2, sleeptime=10)
             for c in contents['changesets']:
                 self.info(" {cset} {desc}".format(
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -2,16 +2,17 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import copy
 import datetime
+import json
 import os
 import re
 import sys
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
@@ -220,17 +221,18 @@ class AndroidEmulatorTest(TestingMixin, 
             'marionette_extra': c.get('marionette_extra', ''),
             'xpcshell_extra': c.get('xpcshell_extra', ''),
             'test_manifest': os.path.join(
                 dirs['abs_marionette_tests_dir'],
                 self.config.get('marionette_test_manifest', '')
             ),
         }
 
-        user_paths = os.environ.get('MOZHARNESS_TEST_PATHS')
+        user_paths = json.loads(os.environ.get('MOZHARNESS_TEST_PATHS', '""'))
+
         for option in self.config["suite_definitions"][self.test_suite]["options"]:
             opt = option.split('=')[0]
             # override configured chunk options with script args, if specified
             if opt in ('--this-chunk', '--total-chunks'):
                 if user_paths or getattr(self, opt.replace('-', '_').strip('_'), None) is not None:
                     continue
 
             if '%(app)' in option:
@@ -250,17 +252,18 @@ class AndroidEmulatorTest(TestingMixin, 
         if category not in SUITE_NO_E10S:
             if category in SUITE_DEFAULT_E10S and not self.e10s:
                 cmd.extend(['--disable-e10s'])
             elif category not in SUITE_DEFAULT_E10S and self.e10s:
                 cmd.extend(['--e10s'])
 
         if not (self.verify_enabled or self.per_test_coverage):
             if user_paths:
-                cmd.extend(user_paths.split(':'))
+                if self.test_suite in user_paths:
+                    cmd.extend(user_paths[self.test_suite])
             elif not (self.verify_enabled or self.per_test_coverage):
                 if self.this_chunk is not None:
                     cmd.extend(['--this-chunk', self.this_chunk])
                 if self.total_chunks is not None:
                     cmd.extend(['--total-chunks', self.total_chunks])
 
         try_options, try_tests = self.try_args(self.test_suite)
         cmd.extend(try_options)
--- a/testing/mozharness/scripts/android_hardware_unittest.py
+++ b/testing/mozharness/scripts/android_hardware_unittest.py
@@ -2,16 +2,17 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import copy
 import datetime
+import json
 import os
 import re
 import sys
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
@@ -207,34 +208,36 @@ class AndroidHardwareTest(TestingMixin, 
             'marionette_extra': c.get('marionette_extra', ''),
             'xpcshell_extra': c.get('xpcshell_extra', ''),
             'test_manifest': os.path.join(
                 dirs['abs_marionette_tests_dir'],
                 self.config.get('marionette_test_manifest', '')
             ),
         }
 
-        user_paths = os.environ.get('MOZHARNESS_TEST_PATHS')
+        user_paths = json.loads(os.environ.get('MOZHARNESS_TEST_PATHS', '""'))
+
         for option in self.config["suite_definitions"][self.test_suite]["options"]:
             opt = option.split('=')[0]
             # override configured chunk options with script args, if specified
             if opt in ('--this-chunk', '--total-chunks'):
                 if user_paths or getattr(self, opt.replace('-', '_').strip('_'), None) is not None:
                     continue
 
             if '%(app)' in option:
                 # only query package name if requested
                 cmd.extend([option % {'app': self.query_package_name()}])
             else:
                 option = option % str_format_values
                 if option:
                     cmd.extend([option])
 
         if user_paths:
-            cmd.extend(user_paths.split(':'))
+            if self.test_suite in user_paths:
+                cmd.extend(user_paths[self.test_suite])
         elif not self.verify_enabled:
             if self.this_chunk is not None:
                 cmd.extend(['--this-chunk', self.this_chunk])
             if self.total_chunks is not None:
                 cmd.extend(['--total-chunks', self.total_chunks])
 
         if 'mochitest' in self.test_suite:
             category = 'mochitest'
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -5,16 +5,17 @@
 
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """desktop_unittest.py
 
 author: Jordan Lund
 """
 
+import json
 import os
 import re
 import sys
 import copy
 import shutil
 import glob
 import imp
 
@@ -353,16 +354,31 @@ class DesktopUnittest(TestingMixin, Merc
                 self.fatal("self.installer_url was found but symbols_url could \
                         not be determined")
         else:
             self.fatal("self.installer_url was not found in self.config")
         self.info("setting symbols_url as %s" % (symbols_url))
         self.symbols_url = symbols_url
         return self.symbols_url
 
+    def _get_mozharness_test_paths(self, suite_category, suite):
+        test_paths = json.loads(os.environ.get('MOZHARNESS_TEST_PATHS', '""'))
+
+        if not test_paths or suite not in test_paths:
+            return None
+
+        suite_test_paths = test_paths[suite]
+
+        if suite_category == 'reftest':
+            dirs = self.query_abs_dirs()
+            suite_test_paths = [os.path.join(dirs['abs_reftest_dir'], 'tests', p)
+                                for p in suite_test_paths]
+
+        return suite_test_paths
+
     def _query_abs_base_cmd(self, suite_category, suite):
         if self.binary_path:
             c = self.config
             dirs = self.query_abs_dirs()
             run_file = c['run_file_names'][suite_category]
             base_cmd = [self.query_python_path('python'), '-u']
             base_cmd.append(os.path.join(dirs["abs_%s_dir" % suite_category], run_file))
             abs_app_dir = self.query_abs_app_dir()
@@ -393,21 +409,18 @@ class DesktopUnittest(TestingMixin, Merc
             if suite_category not in SUITE_NO_E10S:
                 if suite_category in SUITE_DEFAULT_E10S and not c['e10s']:
                     base_cmd.append('--disable-e10s')
                 elif suite_category not in SUITE_DEFAULT_E10S and c['e10s']:
                     base_cmd.append('--e10s')
 
             # Ignore chunking if we have user specified test paths
             if not (self.verify_enabled or self.per_test_coverage):
-                if os.environ.get('MOZHARNESS_TEST_PATHS'):
-                    test_paths = os.environ['MOZHARNESS_TEST_PATHS'].split(':')
-                    if suite_category == 'reftest':
-                        test_paths = [os.path.join(dirs['abs_reftest_dir'], 'tests', p)
-                                      for p in test_paths]
+                test_paths = self._get_mozharness_test_paths(suite_category, suite)
+                if test_paths:
                     base_cmd.extend(test_paths)
                 elif c.get('total_chunks') and c.get('this_chunk'):
                     base_cmd.extend(['--total-chunks', c['total_chunks'],
                                      '--this-chunk', c['this_chunk']])
 
             if c['no_random']:
                 if suite_category == "mochitest":
                     base_cmd.append('--bisect-chunk=default')
@@ -795,17 +808,17 @@ class DesktopUnittest(TestingMixin, Merc
                 }
                 options_list = []
                 env = {
                     'TEST_SUITE': suite
                 }
                 if isinstance(suites[suite], dict):
                     options_list = suites[suite].get('options', [])
                     if (self.verify_enabled or self.per_test_coverage or
-                        os.environ.get('MOZHARNESS_TEST_PATHS')):
+                        self._get_mozharness_test_paths(suite_category, suite)):
                         # Ignore tests list in modes where we are running specific tests.
                         tests_list = []
                     else:
                         tests_list = suites[suite].get('tests', [])
                     env = copy.deepcopy(suites[suite].get('env', {}))
                 else:
                     options_list = suites[suite]
                     tests_list = []
--- a/testing/mozharness/scripts/marionette.py
+++ b/testing/mozharness/scripts/marionette.py
@@ -1,16 +1,17 @@
 #!/usr/bin/env python
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import copy
+import json
 import os
 import sys
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.errors import BaseErrorList, TarErrorList
 from mozharness.base.log import INFO
@@ -312,19 +313,21 @@ class MarionetteTest(TestingMixin, Mercu
 
         for arg in self.config["suite_definitions"][self.test_suite]["options"]:
             cmd.append(arg % config_fmt_args)
 
         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
             # Make sure that the logging directory exists
             self.fatal("Could not create blobber upload directory")
 
-        if os.environ.get('MOZHARNESS_TEST_PATHS'):
+        test_paths = json.loads(os.environ.get('MOZHARNESS_TEST_PATHS', '""'))
+
+        if test_paths and 'marionette' in test_paths:
             paths = [os.path.join(dirs['abs_test_install_dir'], 'marionette', 'tests', p)
-                     for p in os.environ['MOZHARNESS_TEST_PATHS'].split(':')]
+                     for p in test_paths['marionette']]
             cmd.extend(paths)
         else:
             cmd.append(manifest)
 
         try_options, try_tests = self.try_args("marionette")
         cmd.extend(self.query_tests_args(try_tests,
                                          str_format_values=config_fmt_args))
 
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -1,15 +1,16 @@
 #!/usr/bin/env python
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 import copy
+import json
 import os
 import sys
 
 from datetime import datetime, timedelta
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
@@ -246,21 +247,22 @@ class WebPlatformTest(TestingMixin, Merc
             cmd.append("--disable-e10s")
 
         if c["single_stylo_traversal"]:
             cmd.append("--stylo-threads=1")
         else:
             cmd.append("--stylo-threads=4")
 
         if not (self.verify_enabled or self.per_test_coverage):
-            if os.environ.get('MOZHARNESS_TEST_PATHS'):
-                prefix = 'testing/web-platform'
-                paths = os.environ['MOZHARNESS_TEST_PATHS'].split(':')
-                paths = [os.path.join(dirs["abs_wpttest_dir"], os.path.relpath(p, prefix))
-                         for p in paths if p.startswith(prefix)]
+            test_paths = json.loads(os.environ.get('MOZHARNESS_TEST_PATHS', '""'))
+            if test_paths and 'web-platform-tests' in test_paths:
+                relpaths = [os.path.relpath(p, 'testing/web-platform')
+                            for p in test_paths['web-platform-tests']]
+                paths = [os.path.join(dirs["abs_wpttest_dir"], relpath)
+                         for relpath in relpaths]
                 cmd.extend(paths)
             else:
                 for opt in ["total_chunks", "this_chunk"]:
                     val = c.get(opt)
                     if val:
                         cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
 
         options = list(c.get("options", []))
--- a/tools/tryselect/docs/selectors/fuzzy.rst
+++ b/tools/tryselect/docs/selectors/fuzzy.rst
@@ -95,17 +95,17 @@ For example, running:
 
 Would produce the following ``try_task_config.json``:
 
 .. code-block:: json
 
     {
       "templates":{
         "env":{
-          "MOZHARNESS_TEST_PATHS":"layout/reftests/reftest-sanity"
+          "MOZHARNESS_TEST_PATHS":"{\"reftest\":\"layout/reftests/reftest-sanity\"}"
         }
       },
       "tasks":[
         "test-linux64-qr/debug-reftest-e10s-1",
         "test-linux64-qr/opt-reftest-e10s-1",
         "test-linux64/debug-reftest-e10s-1",
         "test-linux64/debug-reftest-no-accel-e10s-1",
         "test-linux64/opt-reftest-e10s-1",
--- a/tools/tryselect/selectors/coverage.py
+++ b/tools/tryselect/selectors/coverage.py
@@ -17,17 +17,17 @@ import datetime
 
 from mozboot.util import get_state_dir
 from mozbuild.base import MozbuildObject
 from mozpack.files import FileFinder
 from moztest.resolve import TestResolver
 from mozversioncontrol import get_repository_object
 
 from ..cli import BaseTryParser
-from ..tasks import generate_tasks, filter_tasks_by_paths
+from ..tasks import generate_tasks, filter_tasks_by_paths, resolve_tests_by_suite
 from ..push import push_to_try
 
 here = os.path.abspath(os.path.dirname(__file__))
 build = MozbuildObject.from_environment(cwd=here)
 vcs = get_repository_object(build.topsrcdir)
 
 root_hash = hashlib.sha256(os.path.abspath(build.topsrcdir)).hexdigest()
 cache_dir = os.path.join(get_state_dir()[0], 'cache', root_hash, 'chunk_mapping')
@@ -370,15 +370,15 @@ def run_coverage_try(templates={}, full=
         test_count=len(test_files),
         test_plural='' if len(test_files) == 1 else 's',
         test_singular='s' if len(test_files) == 1 else '',
         task_count=len(tasks),
         task_plural='' if len(tasks) == 1 else 's')
     print('Found ' + test_count_message)
 
     # Set the test paths to be run by setting MOZHARNESS_TEST_PATHS.
-    path_env = {'MOZHARNESS_TEST_PATHS': ':'.join(test_files)}
+    path_env = {'MOZHARNESS_TEST_PATHS': json.dumps(resolve_tests_by_suite(test_files))}
     templates.setdefault('env', {}).update(path_env)
 
     # Build commit message.
     msg = 'try coverage - ' + test_count_message
     return push_to_try('coverage', message.format(msg=msg), tasks, templates, push=push,
                        closed_tree=kwargs['closed_tree'])
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -6,16 +6,17 @@ from __future__ import absolute_import, 
 
 import glob
 import hashlib
 import json
 import os
 import re
 import shutil
 import sys
+from collections import defaultdict
 
 from mozboot.util import get_state_dir
 from mozbuild.base import MozbuildObject
 from mozpack.files import FileFinder
 from moztest.resolve import TestResolver, get_suite_definition
 
 import taskgraph
 from taskgraph.generator import TaskGraphGenerator
@@ -115,8 +116,23 @@ def filter_tasks_by_paths(tasks, paths):
             continue
 
         task_regexes.update(suite['task_regex'])
 
     def match_task(task):
         return any(re.search(pattern, task) for pattern in task_regexes)
 
     return filter(match_task, tasks)
+
+
+def resolve_tests_by_suite(paths):
+    resolver = TestResolver.from_environment(cwd=here)
+    _, run_tests = resolver.resolve_metadata(paths)
+
+    suite_to_tests = defaultdict(list)
+    for test in run_tests:
+        key = test['flavor']
+        subsuite = test.get('subsuite')
+        if subsuite:
+            key += '-' + subsuite
+        suite_to_tests[key].append(test['srcdir_relpath'])
+
+    return suite_to_tests
--- a/tools/tryselect/templates.py
+++ b/tools/tryselect/templates.py
@@ -4,23 +4,25 @@
 
 """
 Templates provide a way of modifying the task definition of selected
 tasks. They live under taskcluster/taskgraph/templates.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import json
 import os
 import sys
 from abc import ABCMeta, abstractmethod
 from argparse import Action, SUPPRESS
 
 import mozpack.path as mozpath
 from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+from .tasks import resolve_tests_by_suite
 
 here = os.path.abspath(os.path.dirname(__file__))
 build = MozbuildObject.from_environment(cwd=here)
 
 
 class Template(object):
     __metaclass__ = ABCMeta
 
@@ -74,18 +76,17 @@ class Path(Template):
         for p in paths:
             if not os.path.exists(p):
                 print("error: '{}' is not a valid path.".format(p), file=sys.stderr)
                 sys.exit(1)
 
         paths = [mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir) for p in paths]
         return {
             'env': {
-                # can't use os.pathsep as machine splitting could be a different platform
-                'MOZHARNESS_TEST_PATHS': ':'.join(paths),
+                'MOZHARNESS_TEST_PATHS': json.dumps(resolve_tests_by_suite(paths)),
             }
         }
 
 
 class Environment(Template):
 
     def add_arguments(self, parser):
         parser.add_argument('--env', action='append', default=None,
--- a/tools/tryselect/test/conftest.py
+++ b/tools/tryselect/test/conftest.py
@@ -1,18 +1,28 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+from moztest.resolve import TestResolver
 from mozversioncontrol import HgRepository, GitRepository
 import pytest
 
 
+@pytest.fixture
+def patch_resolver(monkeypatch):
+    def inner(suites, tests):
+        def fake_test_metadata(*args, **kwargs):
+            return suites, tests
+        monkeypatch.setattr(TestResolver, 'resolve_metadata', fake_test_metadata)
+    return inner
+
+
 @pytest.fixture(autouse=True)
 def patch_vcs(monkeypatch, tmpdir):
     # Make sure we don't accidentally push to try
     def fake_push_to_try(*args, **kwargs):
         pass
 
     def fake_working_directory_clean(*args, **kwargs):
         return True
--- a/tools/tryselect/test/test_tasks.py
+++ b/tools/tryselect/test/test_tasks.py
@@ -1,34 +1,39 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import mozunit
-import pytest
-from moztest.resolve import TestResolver
 
-from tryselect.tasks import filter_tasks_by_paths
-
-
-@pytest.fixture
-def patch_resolver(monkeypatch):
-    def inner(suites, tests):
-        def fake_test_metadata(*args, **kwargs):
-            return suites, tests
-        monkeypatch.setattr(TestResolver, 'resolve_metadata', fake_test_metadata)
-    return inner
+from tryselect.tasks import filter_tasks_by_paths, resolve_tests_by_suite
 
 
 def test_filter_tasks_by_paths(patch_resolver):
     tasks = ['foobar/xpcshell-1', 'foobar/mochitest', 'foobar/xpcshell']
 
     patch_resolver(['xpcshell'], {})
     assert filter_tasks_by_paths(tasks, 'dummy') == []
 
     patch_resolver([], [{'flavor': 'xpcshell'}])
     assert filter_tasks_by_paths(tasks, 'dummy') == ['foobar/xpcshell-1', 'foobar/xpcshell']
 
 
+def test_resolve_tests_by_suite(patch_resolver):
+    patch_resolver([], [{'flavor': 'xpcshell', 'srcdir_relpath': 'xpcshell.js'}])
+    assert resolve_tests_by_suite(['xpcshell.js']) == {
+        'xpcshell': ['xpcshell.js'],
+    }
+
+    patch_resolver([], [
+        {'flavor': 'xpcshell', 'srcdir_relpath': 'xpcshell.js'},
+        {'flavor': 'mochitest', 'srcdir_relpath': 'mochitest.js'},
+    ])
+    assert resolve_tests_by_suite(['xpcshell.js', 'mochitest.js']) == {
+        'xpcshell': ['xpcshell.js'],
+        'mochitest': ['mochitest.js'],
+    }
+
+
 if __name__ == '__main__':
     mozunit.main()
--- a/tools/tryselect/test/test_templates.py
+++ b/tools/tryselect/test/test_templates.py
@@ -24,19 +24,19 @@ TEMPLATE_TESTS = {
         (['--chemspill-prio'], {'chemspill-prio': {}}),
     ],
     'env': [
         ([], None),
         (['--env', 'foo=bar', '--env', 'num=10'], {'env': {'foo': 'bar', 'num': '10'}}),
     ],
     'path': [
         ([], None),
-        (['dom/indexedDB'], {'env': {'MOZHARNESS_TEST_PATHS': 'dom/indexedDB'}}),
+        (['dom/indexedDB'], {'env': {'MOZHARNESS_TEST_PATHS': '{"xpcshell": ["dom/indexedDB"]}'}}),
         (['dom/indexedDB', 'testing'],
-         {'env': {'MOZHARNESS_TEST_PATHS': 'dom/indexedDB:testing'}}),
+         {'env': {'MOZHARNESS_TEST_PATHS': '{"xpcshell": ["dom/indexedDB", "testing"]}'}}),
         (['invalid/path'], SystemExit),
     ],
     'rebuild': [
         ([], None),
         (['--rebuild', '10'], {'rebuild': 10}),
         (['--rebuild', '1'], SystemExit),
         (['--rebuild', '21'], SystemExit),
     ],
@@ -44,25 +44,36 @@ TEMPLATE_TESTS = {
         ([], None),
         (['--talos-profile'], {'gecko-profile': True}),
         (['--geckoProfile'], {'gecko-profile': True}),
         (['--gecko-profile'], {'gecko-profile': True}),
     ],
 }
 
 
-def test_templates(template, args, expected):
+@pytest.fixture
+def template_patch_resolver(patch_resolver):
+    def inner(paths):
+        patch_resolver([], [{'flavor': 'xpcshell', 'srcdir_relpath': path} for path in paths])
+    return inner
+
+
+def test_templates(template_patch_resolver, template, args, expected):
     parser = ArgumentParser()
 
     t = all_templates[template]()
     t.add_arguments(parser)
 
     if inspect.isclass(expected) and issubclass(expected, BaseException):
         with pytest.raises(expected):
             args = parser.parse_args(args)
+            if template == 'path':
+                template_patch_resolver(**vars(args))
             t.context(**vars(args))
     else:
         args = parser.parse_args(args)
+        if template == 'path':
+            template_patch_resolver(**vars(args))
         assert t.context(**vars(args)) == expected
 
 
 if __name__ == '__main__':
     mozunit.main()