Bug 1193215 - Support for passing test directories through mach try, r=chmanchester
☠☠ backed out by 1ffb54341546 ☠ ☠
authorJames Graham <james@hoppipolla.co.uk>
Tue, 22 Sep 2015 17:55:29 +0100
changeset 299670 ced598ee50fe23465f51a929766420bbd9780718
parent 299669 fb3defaea56e02252f9a59dc07db544c824e95ee
child 299671 2e67853b0b70b1a056b68dae94ed2b308ae84097
push id1001
push userraliiev@mozilla.com
push dateMon, 18 Jan 2016 19:06:03 +0000
treeherdermozilla-release@8b89261f3ac4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1193215
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1193215 - Support for passing test directories through mach try, r=chmanchester This adds support for web-platform-tests to mach try. It changes the implementation so that instead of passing paths to manifests, the user passes arbitary paths in the source tree, and tests under that path are run, with test discovery mainly left to the harness.
testing/mach_commands.py
testing/mozharness/configs/b2g/emulator_automation_config.py
testing/mozharness/configs/b2g/mulet_config.py
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
testing/mozharness/mozharness/mozilla/testing/try_tools.py
testing/mozharness/scripts/android_emulator_unittest.py
testing/mozharness/scripts/android_panda.py
testing/mozharness/scripts/androidx86_emulator_unittest.py
testing/mozharness/scripts/b2g_desktop_unittest.py
testing/mozharness/scripts/b2g_emulator_unittest.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/marionette.py
testing/mozharness/scripts/web_platform_tests.py
testing/tools/autotry/autotry.py
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -431,130 +431,150 @@ class JsapiTestsCommand(MachCommandBase)
         jsapi_tests_cmd = [os.path.join(self.bindir, executable_name('jsapi-tests'))]
         if params['test_name']:
             jsapi_tests_cmd.append(params['test_name'])
 
         jsapi_tests_result = subprocess.call(jsapi_tests_cmd)
 
         return jsapi_tests_result
 
+def autotry_parser():
+    from autotry import parser
+    return parser()
 
 @CommandProvider
 class PushToTry(MachCommandBase):
 
     def validate_args(self, paths, tests, tags, builds, platforms):
         if not any([len(paths), tests, tags]):
             print("Paths, tests, or tags must be specified.")
             sys.exit(1)
 
         if platforms is None:
             platforms = os.environ['AUTOTRY_PLATFORM_HINT']
 
+        rv_platforms = []
+        for item in platforms:
+            for platform in item.split(","):
+                if platform:
+                    rv_platforms.append(platform)
+
+        rv_tests = []
+        for item in tests:
+            for test in item.split(","):
+                if test:
+                    rv_tests.append(test)
+
         for p in paths:
             p = os.path.normpath(os.path.abspath(p))
             if not p.startswith(self.topsrcdir):
                 print('Specified path "%s" is outside of the srcdir, unable to'
                       ' specify tests outside of the srcdir' % p)
                 sys.exit(1)
             if len(p) <= len(self.topsrcdir):
                 print('Specified path "%s" is at the top of the srcdir and would'
                       ' select all tests.' % p)
                 sys.exit(1)
 
-        return builds, platforms
+        return builds, rv_platforms, rv_tests
+
+    @Command('try',
+             category='testing',
+             description='Push selected tests to the try server',
+             parser=autotry_parser)
+    def autotry(self, builds=None, platforms=None, paths=None, verbose=None,
+                push=None, tags=None, tests=None, extra_args=None, intersection=False):
+        """Autotry is in beta, please file bugs blocking 1149670.
+
+        Push the current tree to try, with the specified syntax.
+
+        Build options, platforms and regression tests may be selected
+        using the usual try options (-b, -p and -u respectively). In
+        addition, tests in a given directory may be automatically
+        selected by passing that directory as a positional argument to the
+        command. For example:
+
+        mach try -b d -p linux64 dom testing/web-platform/tests/dom
 
-    @Command('try', category='testing', description='Push selected tests to the try server')
-    @CommandArgument('paths', nargs='*', help='Paths to search for tests to run on try.')
-    @CommandArgument('-n', dest='verbose', action='store_true', default=False,
-                     help='Print detailed information about the resulting test selection '
-                          'and commands performed.')
-    @CommandArgument('-p', dest='platforms', required='AUTOTRY_PLATFORM_HINT' not in os.environ,
-                     help='Platforms to run. (required if not found in the environment)')
-    @CommandArgument('-u', dest='tests',
-                     help='Test jobs to run. These will be used in place of suites '
-                          'determined by test paths, if any.')
-    @CommandArgument('--extra', dest='extra_tests',
-                     help='Additional tests to run. These will be added to suites '
-                          'determined by test paths, if any.')
-    @CommandArgument('-b', dest='builds', default='do',
-                     help='Build types to run (d for debug, o for optimized)')
-    @CommandArgument('--tag', dest='tags', action='append',
-                     help='Restrict tests to the given tag (may be specified multiple times)')
-    @CommandArgument('--no-push', dest='push', action='store_false',
-                     help='Do not push to try as a result of running this command (if '
-                          'specified this command will only print calculated try '
-                          'syntax and selection info).')
-    def autotry(self, builds=None, platforms=None, paths=None, verbose=None,
-                extra_tests=None, push=None, tags=None, tests=None):
-        """mach try is under development, please file bugs blocking 1149670.
+        would schedule a try run for linux64 debug consisting of all
+        tests under dom/ and testing/web-platform/tests/dom.
+
+        Test selection using positional arguments is available for
+        mochitests, reftests, xpcshell tests and web-platform-tests.
+
+        Tests may be also filtered by passing --tag to the command,
+        which will run only tests marked as having the specified
+        tags e.g.
 
-        Pushes the specified tests to try. The simplest way to specify tests is
-        by using the -u argument, which will behave as usual for try syntax.
-        This command also provides a mechanism to select test jobs and tests
-        within a job by path based on tests present in the tree under that
-        path. Mochitests, xpcshell tests, and reftests are eligible for
-        selection by this mechanism. Selected tests will be run in a single
-        chunk of the relevant suite, at this time in chunk 1.
+        mach try -b d -p win64 --tag media
+
+        would run all tests tagged 'media' on Windows 64.
 
-        Specifying platforms is still required with the -p argument (a default
-        is taken from the AUTOTRY_PLATFORM_HINT environment variable if set).
+        If both positional arguments or tags and -u are supplied, the
+        suites in -u will be run in full. Where tests are selected by
+        positional argument they will be run in a single chunk.
 
-        Tests may be further filtered by passing one or more --tag to the
-        command. If one or more --tag is specified with out paths or -u,
-        tests with the given tags will be run in a single chunk of
-        applicable suites.
-
-        To run suites in addition to those determined from the tree, they
-        can be passed to the --extra arguent.
+        If no build option is selected, both debug and opt will be
+        scheduled. If no platform is selected a default is taken from
+        the AUTOTRY_PLATFORM_HINT environment variable, if set.
 
         The command requires either its own mercurial extension ("push-to-try",
         installable from mach mercurial-setup) or a git repo using git-cinnabar
         (available at https://github.com/glandium/git-cinnabar).
+
         """
 
         from mozbuild.testing import TestResolver
         from mozbuild.controller.building import BuildDriver
         from autotry import AutoTry
-        import pprint
-
         print("mach try is under development, please file bugs blocking 1149670.")
 
-        builds, platforms = self.validate_args(paths, tests, tags, builds, platforms)
+        if tests is None:
+            tests = []
+
+        builds, platforms, tests = self.validate_args(paths, tests, tags, builds, platforms)
         resolver = self._spawn(TestResolver)
 
         at = AutoTry(self.topsrcdir, resolver, self._mach_context)
-        if at.find_uncommited_changes():
+        if push and at.find_uncommited_changes():
             print('ERROR please commit changes before continuing')
             sys.exit(1)
 
         if paths or tags:
             driver = self._spawn(BuildDriver)
             driver.install_tests(remove=False)
 
-        manifests_by_flavor = at.resolve_manifests(paths=paths, tags=tags)
+            paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
+                     for item in paths]
+            paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags)
+
+            if not paths_by_flavor and not tests:
+                print("No tests were found when attempting to resolve paths:\n\n\t%s" %
+                      paths)
+                sys.exit(1)
 
-        if not manifests_by_flavor and not tests:
-            print("No tests were found when attempting to resolve paths:\n\n\t%s" %
-                  paths)
+            if not intersection:
+                paths_by_flavor = at.remove_duplicates(paths_by_flavor, tests)
+        else:
+            paths_by_flavor = {}
+
+        try:
+            msg = at.calc_try_syntax(platforms, tests, builds, paths_by_flavor, tags,
+                                     extra_args, intersection)
+        except ValueError as e:
+            print(e.message)
             sys.exit(1)
 
-        all_manifests = set()
-        for m in manifests_by_flavor.values():
-            all_manifests |= m
-        all_manifests = list(all_manifests)
-
-        msg = at.calc_try_syntax(platforms, manifests_by_flavor.keys(), tests,
-                                 extra_tests, builds, all_manifests, tags)
-
-        if verbose and manifests_by_flavor:
-            print('Tests from the following manifests will be selected: ')
-            pprint.pprint(manifests_by_flavor)
+        if verbose and paths_by_flavor:
+            print('The following tests will be selected: ')
+            for flavor, paths in paths_by_flavor.iteritems():
+                print("%s: %s" % (flavor, ",".join(paths)))
 
         if verbose or not push:
-            print('The following try syntax was calculated:\n\n\t%s\n' % msg)
+            print('The following try syntax was calculated:\n%s' % msg)
 
         if push:
             at.push_to_try(msg, verbose)
 
         return
 
 
 def get_parser(argv=None):
--- a/testing/mozharness/configs/b2g/emulator_automation_config.py
+++ b/testing/mozharness/configs/b2g/emulator_automation_config.py
@@ -113,18 +113,18 @@ config = {
                 "--busybox=%(busybox)s",
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--quiet",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
-                "%(test_path)s"
             ],
+            "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
         },
         "mochitest-chrome": {
             "options": [
                 "--adbpath=%(adbpath)s",
                 "--b2gpath=%(b2gpath)s",
                 "--emulator=%(emulator)s",
@@ -137,18 +137,18 @@ config = {
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--quiet",
                 "--chrome",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
-                "%(test_path)s"
             ],
+            "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
         },
         "reftest": {
             "options": [
                 "--adbpath=%(adbpath)s",
                 "--b2gpath=%(b2gpath)s",
                 "--emulator=%(emulator)s",
--- a/testing/mozharness/configs/b2g/mulet_config.py
+++ b/testing/mozharness/configs/b2g/mulet_config.py
@@ -15,16 +15,16 @@ config = {
             "options": [
                 "--mulet",
                 "--profile=%(gaia_profile)s",
                 "--appname=%(application)s",
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--symbols-path=%(symbols_path)s",
                 "--enable-oop",
-                "%(test_manifest)s"
             ],
+            "tests": ["%(test_manifest)s"]
         }
     },
     "run_file_names": {
         "reftest": "runreftestb2g.py",
     },
 }
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -251,17 +251,17 @@ config = {
         "xpcshell-addons": {
             "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
                         "--tag=addons",
                         "--manifest=tests/xpcshell/tests/all-test-dirs.list"],
             "tests": []
         },
     },
     "all_cppunittest_suites": {
-        "cppunittest": ["tests/cppunittest"]
+        "cppunittest": {"tests": ["tests/cppunittest"]}
     },
     "all_gtest_suites": {
         "gtest": []
     },
     "all_jittest_suites": {
         "jittest": [],
         "jittest1": ["--total-chunks=2", "--this-chunk=1"],
         "jittest2": ["--total-chunks=2", "--this-chunk=2"],
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -21,17 +21,17 @@ from mozharness.base.python import (
     VirtualenvMixin,
     virtualenv_config_options,
 )
 from mozharness.mozilla.buildbot import BuildbotMixin, TBPL_WARNING
 from mozharness.mozilla.proxxy import Proxxy
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.taskcluster_helper import TaskClusterArtifactFinderMixin
 from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
-from mozharness.mozilla.testing.try_tools import TryToolsMixin
+from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options
 from mozharness.mozilla.tooltool import TooltoolMixin
 
 from mozharness.lib.python.authentication import get_credentials
 
 INSTALLER_SUFFIXES = ('.tar.bz2', '.zip', '.dmg', '.exe', '.apk', '.tar.gz')
 
 testing_config_options = [
     [["--installer-url"],
@@ -78,17 +78,17 @@ testing_config_options = [
       }],
     [["--download-symbols"],
      {"action": "store",
      "dest": "download_symbols",
      "type": "choice",
      "choices": ['ondemand', 'true'],
      "help": "Download and extract crash reporter symbols.",
       }],
-] + copy.deepcopy(virtualenv_config_options)
+] + copy.deepcopy(virtualenv_config_options) + copy.deepcopy(try_config_options)
 
 
 # TestingMixin {{{1
 class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin,
                    TaskClusterArtifactFinderMixin, TooltoolMixin, TryToolsMixin):
     """
     The steps to identify + download the proper bits for [browser] unit
     tests and Talos.
@@ -671,16 +671,45 @@ Did you run with --create-virtualenv? Is
             else:
                 self.warning("minidump stackwalk path was given but couldn't be found. "
                              "Tried looking in '%s'" % abs_minidump_path)
                 # don't burn the job but we should at least turn them orange so it is caught
                 self.buildbot_status(TBPL_WARNING, WARNING)
 
         return self.minidump_stackwalk_path
 
+    def query_options(self, *args, **kwargs):
+        if "str_format_values" in kwargs:
+            str_format_values = kwargs.pop("str_format_values")
+        else:
+            str_format_values = {}
+
+        arguments = []
+
+        for arg in args:
+            if arg is not None:
+                arguments.extend(argument % str_format_values for argument in arg)
+
+        return arguments
+
+    def query_tests_args(self, *args, **kwargs):
+        if "str_format_values" in kwargs:
+            str_format_values = kwargs.pop("str_format_values")
+        else:
+            str_format_values = {}
+
+        arguments = []
+
+        for arg in reversed(args):
+            if arg:
+                arguments.append("--")
+                arguments.extend(argument % str_format_values for argument in arg)
+                break
+
+        return arguments
 
     def _run_cmd_checks(self, suites):
         if not suites:
             return
         dirs = self.query_abs_dirs()
         for suite in suites:
             # XXX platform.architecture() may give incorrect values for some
             # platforms like mac as excutable files may be universal
--- a/testing/mozharness/mozharness/mozilla/testing/try_tools.py
+++ b/testing/mozharness/mozharness/mozilla/testing/try_tools.py
@@ -3,62 +3,91 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import argparse
 import os
 import re
+from collections import defaultdict
 
 from mozharness.base.script import PostScriptAction
 from mozharness.base.transfer import TransferMixin
 
+try_config_options = [
+    [["--try-message"],
+     {"action": "store",
+     "dest": "try_message",
+     "default": None,
+     "help": "try syntax string to select tests to run",
+      }],
+]
+
+test_flavors = {
+    'browser-chrome': {},
+    'chrome': {},
+    'devtools-chrome': {},
+    'mochitest': {},
+    'xpcshell' :{},
+    'reftest': {
+        "path": lambda x: os.path.join("tests", "reftest", "tests", x)
+    },
+    'crashtest': {
+        "path": lambda x: os.path.join("tests", "reftest", "tests", x)
+    },
+    'web-platform-tests': {
+        "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
+    }
+}
 
 class TryToolsMixin(TransferMixin):
     """Utility functions for an interface between try syntax and out test harnesses.
     Requires log and script mixins."""
 
     harness_extra_args = None
-    try_test_paths = []
+    try_test_paths = {}
     known_try_arguments = {
         '--tag': {
             'action': 'append',
             'dest': 'tags',
             'default': None,
         },
     }
 
     def _extract_try_message(self):
         msg = None
-        if self.buildbot_config['sourcestamp']['changes']:
-            msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments']
+        if "try_message" in self.config and self.config["try_message"]:
+            msg = self.config["try_message"]
+        else:
+            if self.buildbot_config['sourcestamp']['changes']:
+                msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments']
 
-        if msg is None or len(msg) == 1024:
-            # This commit message was potentially truncated, get the full message
-            # from hg.
-            props = self.buildbot_config['properties']
-            rev = props['revision']
-            repo = props['repo_path']
-            url = 'https://hg.mozilla.org/%s/json-pushes?changeset=%s&full=1' % (repo, rev)
+            if msg is None or len(msg) == 1024:
+                # This commit message was potentially truncated, get the full message
+                # from hg.
+                props = self.buildbot_config['properties']
+                rev = props['revision']
+                repo = props['repo_path']
+                url = 'https://hg.mozilla.org/%s/json-pushes?changeset=%s&full=1' % (repo, rev)
 
-            pushinfo = self.load_json_from_url(url)
-            for k, v in pushinfo.items():
-                if isinstance(v, dict) and 'changesets' in v:
-                    msg = v['changesets'][-1]['desc']
+                pushinfo = self.load_json_from_url(url)
+                for k, v in pushinfo.items():
+                    if isinstance(v, dict) and 'changesets' in v:
+                        msg = v['changesets'][-1]['desc']
 
-        if not msg and 'try_syntax' in self.buildbot_config['properties']:
-            # If we don't find try syntax in the usual place, check for it in an
-            # alternate property available to tools using self-serve.
-            msg = self.buildbot_config['properties']['try_syntax']
+            if not msg and 'try_syntax' in self.buildbot_config['properties']:
+                # If we don't find try syntax in the usual place, check for it in an
+                # alternate property available to tools using self-serve.
+                msg = self.buildbot_config['properties']['try_syntax']
 
         return msg
 
     @PostScriptAction('download-and-extract')
-    def _set_extra_try_arguments(self, action, success=None):
+    def set_extra_try_arguments(self, action, success=None):
         """Finds a commit message and parses it for extra arguments to pass to the test
         harness command line and test paths used to filter manifests.
 
         Extracting arguments from a commit message taken directly from the try_parser.
         """
         if (not self.buildbot_config or 'properties' not in self.buildbot_config or
                 self.buildbot_config['properties'].get('branch') != 'try'):
             return
@@ -102,17 +131,17 @@ class TryToolsMixin(TransferMixin):
                            'arguments to the harness process.')
             if 'dest' in opts:
                 label_dict[opts['dest']] = label
 
             parser.add_argument(label, **opts)
 
         parser.add_argument('--try-test-paths', nargs='*')
         (args, _) = parser.parse_known_args(all_try_args)
-        self.try_test_paths = args.try_test_paths
+        self.try_test_paths = self._group_test_paths(args.try_test_paths)
         del args.try_test_paths
 
         out_args = []
         # This is a pretty hacky way to echo arguments down to the harness.
         # Hopefully this can be improved once we have a configuration system
         # in tree for harnesses that relies less on a command line.
         for (arg, value) in vars(args).iteritems():
             if value:
@@ -122,96 +151,40 @@ class TryToolsMixin(TransferMixin):
                     out_args.append(label)
                 elif isinstance(value, list):
                     out_args.extend(['%s=%s' % (label, el) for el in value])
                 else:
                     out_args.append('%s=%s' % (label, value))
 
         self.harness_extra_args = out_args
 
-    def _resolve_specified_manifests(self):
-        if not self.try_test_paths:
-            return None
+    def _group_test_paths(self, args):
+        rv = defaultdict(list)
 
-        target_manifests = set(self.try_test_paths)
+        if args is None:
+            return rv
 
-        def filter_ini_manifest(line):
-            # Lines are formatted as [include:<path>], we care about <path>.
-            parts = line.split(':')
-            term = line
-            if len(parts) == 2:
-                term = parts[1]
-            if term.endswith(']'):
-                term = term[:-1]
-            if (term in target_manifests or
-                any(term.startswith(l) for l in target_manifests)):
-                return True
-            return False
+        for item in args:
+            suite, path = item.split(":", 1)
+            rv[suite].append(path)
+        return rv
 
-        def filter_list_manifest(line):
-            # Lines are usually formatted as "include <path>", we care about <path>.
-            parts = line.split()
-            term = line
-            if len(parts) == 2:
-                term = parts[1]
-            # Reftest master manifests also include skip-if lines and relative
-            # paths we aren't doing to resolve here, so unlike above this is just
-            # a substring check.
-            if (term in target_manifests or
-                any(l in term for l in target_manifests)):
-                return True
-            return False
+    def try_args(self, flavor):
+        """Get arguments, test_list derived from try syntax to apply to a command"""
+        # TODO: Detect and reject incompatible arguments
+        args = self.harness_extra_args[:] if self.harness_extra_args else []
 
-        # The master manifests we need to filter for target manifests.
-        # TODO: All this needs to go in a config file somewhere and get sewn
-        # into the job definition so its less likely to break as things are
-        # modified. One straightforward way to achieve this would be with a key
-        # in the tree manifest for the master manifest path, however the current
-        # tree manifests don't distinguish between flavors of mochitests to this
-        # isn't straightforward.
-        master_manifests = [
-            ('mochitest/chrome/chrome.ini', filter_ini_manifest),
-            ('mochitest/browser/browser-chrome.ini', filter_ini_manifest),
-            ('mochitest/tests/mochitest.ini', filter_ini_manifest),
-            ('xpcshell/tests/all-test-dirs.list', filter_ini_manifest),
-            ('xpcshell/tests/xpcshell.ini', filter_ini_manifest),
-            ('reftest/tests/layout/reftests/reftest.list', filter_list_manifest),
-            ('reftest/tests/testing/crashtest/crashtests.list', filter_list_manifest),
-        ]
-
-        dirs = self.query_abs_dirs()
-        tests_dir = dirs.get('abs_test_install_dir',
-                             os.path.join(dirs['abs_work_dir'], 'tests'))
-        master_manifests = [(os.path.join(tests_dir, name), filter_fn) for (name, filter_fn) in
-                            master_manifests]
-        for m, filter_fn in master_manifests:
-            if not os.path.isfile(m):
-                continue
+        if self.try_test_paths.get(flavor):
+            self.info('TinderboxPrint: Tests will be run from the following '
+                      'files: %s.' % ','.join(self.try_test_paths[flavor]))
+            args.extend(['--this-chunk=1', '--total-chunks=1'])
 
-            self.info("Filtering master manifest at: %s" % m)
-            lines = self.read_from_file(m).splitlines()
-
-            out_lines = [line for line in lines if filter_fn(line)]
-
-            self.rmtree(m)
-            self.write_to_file(m, '\n'.join(out_lines))
-
+            path_func = test_flavors[flavor].get("path", lambda x:x)
+            tests = [path_func(item) for item in self.try_test_paths[flavor]]
+        else:
+            tests = []
 
-    def append_harness_extra_args(self, cmd):
-        """Append arguments derived from try syntax to a command."""
-        # TODO: Detect and reject incompatible arguments
-        extra_args = self.harness_extra_args[:] if self.harness_extra_args else []
-        if self.try_test_paths:
-            self._resolve_specified_manifests()
-            self.info('TinderboxPrint: Tests will be run from the following '
-                      'manifests: %s.' % ','.join(self.try_test_paths))
-            extra_args.extend(['--this-chunk=1', '--total-chunks=1'])
+        if args or tests:
+            self.info('TinderboxPrint: The following arguments were forwarded from mozharness '
+                      'to the test command:\nTinderboxPrint: \t%s -- %s' %
+                      (" ".join(args), " ".join(tests)))
 
-        if not extra_args:
-            return cmd
-
-        out_cmd = cmd[:]
-        out_cmd.extend(extra_args)
-        self.info('TinderboxPrint: The following arguments were forwarded from mozharness '
-                  'to the test command:\nTinderboxPrint: \t%s' %
-                  extra_args)
-
-        return out_cmd
+        return args, tests
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -478,24 +478,22 @@ class AndroidEmulatorTest(BlobUploadMixi
 
         for arg in self.test_suite_definitions[self.test_suite]["extra_args"]:
             argname = arg.split('=')[0]
             # only add the extra arg if it wasn't already defined by in-tree configs
             if any(a.split('=')[0] == argname for a in cmd):
                 continue
             cmd.append(arg)
 
-        tests = None
-        if "tests" in self.test_suite_definitions[self.test_suite]:
-            tests = self.test_suite_definitions[self.test_suite]["tests"]
-        elif "tests" in self.config["suite_definitions"][suite_category]:
-            tests = self.config["suite_definitions"][suite_category]["tests"]
-
-        if tests:
-            cmd.extend(tests)
+        try_options, try_tests = self.try_args(suite_category)
+        cmd.extend(try_options)
+        cmd.extend(self.query_tests_args(
+            self.config["suite_definitions"][suite_category].get("tests"),
+            self.test_suite_definitions[self.test_suite].get("tests"),
+            try_tests))
 
         return cmd
 
     def _tooltool_fetch(self, url):
         c = self.config
         dirs = self.query_abs_dirs()
 
         manifest_path = self.download_file(
@@ -659,17 +657,17 @@ class AndroidEmulatorTest(BlobUploadMixi
 
         self.info("Finished installing apps for %s" % self.emulator["name"])
 
     def run_tests(self):
         """
         Run the tests
         """
         cmd = self._build_command()
-        cmd = self.append_harness_extra_args(cmd)
+
         try:
             cwd = self._query_tests_dir()
         except:
             self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
         env = self.query_env()
         if self.query_minidump_stackwalk():
             env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
         env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
--- a/testing/mozharness/scripts/android_panda.py
+++ b/testing/mozharness/scripts/android_panda.py
@@ -212,16 +212,30 @@ class PandaTest(TestingMixin, MercurialS
         """
         if self.config.get('run_all_suites'):
             target_categories = SUITE_CATEGORIES
         else:
             target_categories = [cat for cat in SUITE_CATEGORIES
                                  if self._query_specified_suites(cat) is not None]
         super(PandaTest, self).download_and_extract(suite_categories=target_categories)
 
+    def _query_try_flavor(self, category, suite):
+        flavors = {
+            "mochitest": [("plain.*", "mochitest"),
+                          ("browser-chrome.*", "browser-chrome"),
+                          ("mochitest-devtools-chrome.*", "devtools-chrome"),
+                          ("chrome", "chrome")],
+            "xpcshell": [("xpcshell", "xpcshell")],
+            "reftest": [("reftest", "reftest"),
+                        ("crashtest", "crashtest")]
+        }
+        for suite_pattern, flavor in flavors.get(category, []):
+            if re.compile(suite_pattern).match(suite):
+                return flavor
+
     def _run_category_suites(self, suite_category, preflight_run_method=None):
         """run suite(s) to a specific category"""
 
         env = self.query_env(partial_env={'DM_TRANS': "sut", 'TEST_DEVICE': self.mozpool_device})
         self.info("Running tests...")
 
         suites = self._query_specified_suites(suite_category)
         level = INFO
@@ -241,21 +255,23 @@ class PandaTest(TestingMixin, MercurialS
                 if 'robocop' in suite:
                     self._download_robocop_apk()
                 if 'jittest' in suite:
                     should_install_app = False
 
                 if should_install_app:
                     self._install_app()
                 cmd = abs_base_cmd[:]
-                replace_dict = {}
-                for arg in suites[suite]:
-                    cmd.append(arg % replace_dict)
+
+                flavor = self._query_try_flavor(suite_category, suite)
+                try_options, try_tests = self.try_args(flavor)
 
-                cmd = self.append_harness_extra_args(cmd)
+                cmd.extend(self.query_options(suites[suite],
+                                              try_options))
+                cmd.extend(self.query_tests_args(try_tests))
 
                 tests = self.config["suite_definitions"][suite_category].get("tests", [])
                 cmd += tests
 
                 tbpl_status, log_level = None, None
                 error_list = BaseErrorList + [{
                     'regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"),
                     'level': ERROR,
--- a/testing/mozharness/scripts/androidx86_emulator_unittest.py
+++ b/testing/mozharness/scripts/androidx86_emulator_unittest.py
@@ -447,20 +447,22 @@ class AndroidEmulatorTest(BlobUploadMixi
 
         for arg in self.test_suite_definitions[suite_name]["extra_args"]:
             argname = arg.split('=')[0]
             # only add the extra arg if it wasn't already defined by in-tree configs
             if any(a.split('=')[0] == argname for a in cmd):
                 continue
             cmd.append(arg)
 
-        if "tests" in self.test_suite_definitions[suite_name]:
-            cmd.extend(self.suite_definitions[suite_name]["tests"])
-        elif "tests" in c["suite_definitions"][suite_category]:
-            cmd.extend(self.config["suite_definitions"][suite_category]["tests"])
+        try_options, try_tests = self.try_args(suite_category)
+        cmd.extend(try_options)
+        cmd.extend(self.query_tests_args(
+            self.config["suite_definitions"][suite_category].get("tests"),
+            self.test_suite_definitions[self.test_suite].get("tests"),
+            try_tests))
 
         return cmd
 
     def preflight_run_tests(self):
         super(AndroidEmulatorTest, self).preflight_run_tests()
 
         if not os.path.isfile(self.adb_path):
             self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path)
@@ -472,17 +474,16 @@ class AndroidEmulatorTest(BlobUploadMixi
         We return a dictionary with the following information:
          - subprocess object that is running the test on the emulator
          - the filename where the stdout is going to
          - the stdout where the output is going to
          - the suite name that is associated
         """
         dirs = self.query_abs_dirs()
         cmd = self._build_command(self.emulators[emulator_index], suite_name)
-        cmd = self.append_harness_extra_args(cmd)
 
         try:
             cwd = self._query_tests_dir(suite_name)
         except:
             self.fatal("Don't know how to run --test-suite '%s'!" % suite_name)
 
         env = self.query_env()
         if self.query_minidump_stackwalk():
--- a/testing/mozharness/scripts/b2g_desktop_unittest.py
+++ b/testing/mozharness/scripts/b2g_desktop_unittest.py
@@ -134,16 +134,17 @@ class B2GDesktopTest(BlobUploadMixin, Te
         dirs = self.query_abs_dirs()
         cmd = [self.query_python_path('python')]
         cmd.append(self.config["run_file_names"][suite])
 
         raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
                                     '%s_raw.log' % suite)
         error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
                                           '%s_errorsummary.log' % suite)
+
         str_format_values = {
             'application': self.binary_path,
             'test_manifest': self.test_manifest,
             'symbols_path': self.symbols_path,
             'gaia_profile': self.gaia_profile,
             'utility_path': os.path.join(dirs['abs_tests_dir'], 'bin'),
             'total_chunks': self.config.get('total_chunks'),
             'this_chunk': self.config.get('this_chunk'),
@@ -151,26 +152,25 @@ class B2GDesktopTest(BlobUploadMixin, Te
             'browser_arg': self.config.get('browser_arg'),
             'raw_log_file': raw_log_file,
             'error_summary_file': error_summary_file,
         }
 
         if suite not in self.config["suite_definitions"]:
             self.fatal("'%s' not defined in the config!" % suite)
 
-        options = self.config["suite_definitions"][suite]["options"]
-        if options:
-            for option in options:
-                option = option % str_format_values
-                if not option.endswith('None'):
-                    cmd.append(option)
+        try_options, try_tests = self.try_args(suite)
 
-        tests = self.config["suite_definitions"][suite].get("tests")
-        if tests:
-            cmd.extend(tests)
+        cmd.extend(self.query_options(self.config["suite_definitions"][suite]["options"],
+                                      try_options,
+                                      str_format_values=str_format_values))
+
+        cmd.extend(self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
+                                         try_tests,
+                                         str_format_values=str_format_values))
 
         return cmd
 
     def download_and_extract(self):
         suite_category = self.config['test_suite']
         super(B2GDesktopTest, self).download_and_extract(suite_categories=[suite_category])
 
     def preflight_run_tests(self):
@@ -195,17 +195,16 @@ class B2GDesktopTest(BlobUploadMixin, Te
         error_list = self.error_list
         error_list.extend(BaseErrorList)
 
         suite = self.config['test_suite']
         if suite not in self.test_suites:
             self.fatal("Don't know how to run --test-suite '%s'!" % suite)
 
         cmd = self._query_abs_base_cmd(suite)
-        cmd = self.append_harness_extra_args(cmd)
 
         cwd = dirs['abs_%s_dir' % suite]
 
         # TODO we probably have to move some of the code in
         # scripts/desktop_unittest.py and scripts/marionette.py to
         # mozharness.mozilla.testing.unittest so we can share it.
         # In the short term, I'm ok with some duplication of code if it
         # expedites things; please file bugs to merge if that happens.
--- a/testing/mozharness/scripts/b2g_emulator_unittest.py
+++ b/testing/mozharness/scripts/b2g_emulator_unittest.py
@@ -268,28 +268,29 @@ class B2GEmulatorTest(TestingMixin, VCSM
             'this_chunk': self.config.get('this_chunk'),
             'test_path': self.config.get('test_path'),
             'certificate_path': dirs['abs_certs_dir'],
             'raw_log_file': raw_log_file,
             'error_summary_file': error_summary_file,
         }
 
         if suite not in self.config["suite_definitions"]:
-            self.fatal("Key '%s' not defined in the config!" % suite)
+            self.fatal("'%s' not defined in the config!" % suite)
+
+        try_options, try_tests = self.try_args(suite)
 
-        options = self.config["suite_definitions"][suite]["options"]
-        if options:
-            for option in options:
-                option = option % str_format_values
-                if not option.endswith('None'):
-                    cmd.append(option)
+        options = self.query_options(self.config["suite_definitions"][suite]["options"],
+                                     try_options,
+                                     str_format_values=str_format_values)
+        cmd.extend(opt for opt in options if not opt.endswith('None'))
 
-        tests = self.config["suite_definitions"][suite].get("tests", [])
-        if tests:
-            cmd.extend(tests)
+        tests = self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
+                                      try_tests,
+                                      str_format_values=str_format_values)
+        cmd.extend(opt for opt in tests if not opt.endswith('None'))
 
         return cmd
 
     def _query_adb(self):
         return self.which('adb') or \
             os.getenv('ADB_PATH') or \
             os.path.join(self.query_abs_dirs()['abs_b2g-distro_dir'],
                          'out', 'host', 'linux-x86', 'bin', 'adb')
@@ -336,18 +337,16 @@ class B2GEmulatorTest(TestingMixin, VCSM
         error_list = self.error_list
         error_list.extend(BaseErrorList)
 
         suite = self.config['test_suite']
         if suite not in self.test_suites:
             self.fatal("Don't know how to run --test-suite '%s'!" % suite)
 
         cmd = self._query_abs_base_cmd(suite)
-        cmd = self.append_harness_extra_args(cmd)
-
         cwd = dirs['abs_%s_dir' % suite]
 
         # TODO we probably have to move some of the code in
         # scripts/desktop_unittest.py and scripts/marionette.py to
         # mozharness.mozilla.testing.unittest so we can share it.
         # In the short term, I'm ok with some duplication of code if it
         # expedites things; please file bugs to merge if that happens.
 
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -29,17 +29,16 @@ from mozharness.mozilla.mozbase import M
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 
 SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill', 'webapprt']
 
-
 # DesktopUnittest {{{1
 class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin, CodeCoverageMixin):
     config_options = [
         [['--mochitest-suite', ], {
             "action": "extend",
             "dest": "specified_mochitest_suites",
             "type": "string",
             "help": "Specify which mochi suite to run. "
@@ -294,76 +293,52 @@ class DesktopUnittest(TestingMixin, Merc
                 self.fatal("self.installer_url was found but symbols_url could \
                         not be determined")
         else:
             self.fatal("self.installer_url was not found in self.config")
         self.info("setting symbols_url as %s" % (symbols_url))
         self.symbols_url = symbols_url
         return self.symbols_url
 
-    def get_webapprt_path(self, res_dir, mochitest_dir):
-        """Get the path to the webapp runtime binary.
-        On Mac, we copy the stub from the resources dir to the test app bundle,
-        since we have to run it from the executable directory of a bundle
-        in order for its windows to appear.  Ideally, the build system would do
-        this for us at build time, and we should find a way for it to do that.
-        """
-        exe_suffix = self.config.get('exe_suffix', '')
-        app_name = 'webapprt-stub' + exe_suffix
-        app_path = os.path.join(res_dir, app_name)
-        if self._is_darwin():
-            mac_dir_name = os.path.join(
-                mochitest_dir,
-                'webapprtChrome',
-                'webapprt',
-                'test',
-                'chrome',
-                'TestApp.app',
-                'Contents',
-                'MacOS')
-            mac_app_name = 'webapprt' + exe_suffix
-            mac_app_path = os.path.join(mac_dir_name, mac_app_name)
-            self.copyfile(app_path, mac_app_path, copystat=True)
-            return mac_app_path
-        return app_path
-
     def _query_abs_base_cmd(self, suite_category, suite):
         if self.binary_path:
             c = self.config
             dirs = self.query_abs_dirs()
             run_file = c['run_file_names'][suite_category]
             base_cmd = [self.query_python_path('python'), '-u']
             base_cmd.append(os.path.join(dirs["abs_%s_dir" % suite_category], run_file))
             abs_app_dir = self.query_abs_app_dir()
             abs_res_dir = self.query_abs_res_dir()
 
+            webapprt_path = os.path.join(os.path.dirname(self.binary_path),
+                                         'webapprt-stub')
+            if c.get('exe_suffix'):
+                webapprt_path += c['exe_suffix']
+
             raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
                                         '%s_raw.log' % suite)
 
             error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
                                               '%s_errorsummary.log' % suite)
             str_format_values = {
                 'binary_path': self.binary_path,
                 'symbols_path': self._query_symbols_url(),
                 'abs_app_dir': abs_app_dir,
-                'abs_res_dir': abs_res_dir,
+                'app_path': webapprt_path,
                 'raw_log_file': raw_log_file,
                 'error_summary_file': error_summary_file,
                 'gtest_dir': os.path.join(dirs['abs_test_install_dir'],
                                           'gtest'),
             }
 
             # TestingMixin._download_and_extract_symbols() will set
             # self.symbols_path when downloading/extracting.
             if self.symbols_path:
                 str_format_values['symbols_path'] = self.symbols_path
 
-            if suite_category == 'webapprt':
-                str_format_values['app_path'] = self.get_webapprt_path(abs_res_dir, dirs['abs_mochitest_dir'])
-
             if c['e10s']:
                 base_cmd.append('--e10s')
 
             if c.get('strict_content_sandbox'):
                 if suite_category == "mochitest":
                     base_cmd.append('--strict-content-sandbox')
                 else:
                     self.fatal("--strict-content-sandbox only works with mochitest suites.")
@@ -393,17 +368,24 @@ class DesktopUnittest(TestingMixin, Merc
                         base_cmd.append(option)
                 return base_cmd
             else:
                 self.warning("Suite options for %s could not be determined."
                              "\nIf you meant to have options for this suite, "
                              "please make sure they are specified in your "
                              "config under %s_options" %
                              (suite_category, suite_category))
-                return base_cmd
+
+
+            for option in options:
+                option = option % str_format_values
+                if not option.endswith('None'):
+                    base_cmd.append(option)
+
+            return base_cmd
         else:
             self.fatal("'binary_path' could not be determined.\n This should "
                        "be like '/path/build/application/firefox/firefox'"
                        "\nIf you are running this script without the 'install' "
                        "action (where binary_path is set), please ensure you are"
                        " either:\n(1) specifying it in the config file under "
                        "binary_path\n(2) specifying it on command line with the"
                        " '--binary-path' flag")
@@ -429,16 +411,30 @@ class DesktopUnittest(TestingMixin, Merc
                 suites = dict((key, all_suites.get(key)) for key in
                               specified_suites if key in all_suites.keys())
         else:
             if c.get('run_all_suites'):  # needed if you dont specify any suites
                 suites = all_suites
 
         return suites
 
+    def _query_try_flavor(self, category, suite):
+        flavors = {
+            "mochitest": [("plain.*", "mochitest"),
+                          ("browser-chrome.*", "browser-chrome"),
+                          ("mochitest-devtools-chrome.*", "devtools-chrome"),
+                          ("chrome", "chrome")],
+            "xpcshell": [("xpcshell", "xpcshell")],
+            "reftest": [("reftest", "reftest"),
+                        ("crashtest", "crashtest")]
+        }
+        for suite_pattern, flavor in flavors.get(category, []):
+            if re.compile(suite_pattern).match(suite):
+                return flavor
+
     # Actions {{{2
 
     # clobber defined in BaseScript, deletes mozharness/build if exists
     # read_buildbot_config is in BuildbotMixin.
     # postflight_read_buildbot_config is in TestingMixin.
     # preflight_download_and_extract is in TestingMixin.
     # create_virtualenv is in VirtualenvMixin.
     # preflight_install is in TestingMixin.
@@ -583,25 +579,32 @@ class DesktopUnittest(TestingMixin, Merc
 
                     # Mac specific, but points to abs_app_dir on other
                     # platforms.
                     'abs_res_dir': abs_res_dir,
                 }
                 options_list = []
                 env = {}
                 if isinstance(suites[suite], dict):
-                    options_list = suites[suite]['options'] + suites[suite].get("tests", [])
+                    options_list = suites[suite].get('options', [])
+                    tests_list = suites[suite].get('tests', [])
                     env = copy.deepcopy(suites[suite].get('env', {}))
                 else:
                     options_list = suites[suite]
+                    tests_list = []
 
-                for arg in options_list:
-                    cmd.append(arg % replace_dict)
+                flavor = self._query_try_flavor(suite_category, suite)
+                try_options, try_tests = self.try_args(flavor)
 
-                cmd = self.append_harness_extra_args(cmd)
+                cmd.extend(self.query_options(options_list,
+                                              try_options,
+                                              str_format_values=replace_dict))
+                cmd.extend(self.query_tests_args(tests_list,
+                                                 try_tests,
+                                                 str_format_values=replace_dict))
 
                 suite_name = suite_category + '-' + suite
                 tbpl_status, log_level = None, None
                 error_list = BaseErrorList + [{
                     'regex': re.compile(r'''PROCESS-CRASH.*application crashed'''),
                     'level': ERROR,
                 }]
                 parser = self.get_test_output_parser(suite_category,
--- a/testing/mozharness/scripts/marionette.py
+++ b/testing/mozharness/scripts/marionette.py
@@ -434,17 +434,20 @@ class MarionetteTest(TestingMixin, Mercu
         for s in self.config["suite_definitions"][options_group]["options"]:
             cmd.append(s % config_fmt_args)
 
         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
             # Make sure that the logging directory exists
             self.fatal("Could not create blobber upload directory")
 
         cmd.append(manifest)
-        cmd = self.append_harness_extra_args(cmd)
+
+        try_options, try_tests = self.try_args("marionette")
+        cmd.extend(self.query_tests_args(try_tests,
+                                         str_format_values=config_fmt_args))
 
         env = {}
         if self.query_minidump_stackwalk():
             env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
         if self.config.get('gaiatest'):
             env['GAIATEST_ACKNOWLEDGED_RISKS'] = '1'
             env['GAIATEST_SKIP_WARNING'] = '1'
         env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -110,69 +110,74 @@ class WebPlatformTest(TestingMixin, Merc
             self.fatal("Binary path could not be determined")
             #And exit
 
         c = self.config
         dirs = self.query_abs_dirs()
         abs_app_dir = self.query_abs_app_dir()
         run_file_name = "runtests.py"
 
-        base_cmd = [self.query_python_path('python'), '-u']
-        base_cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
+        cmd = [self.query_python_path('python'), '-u']
+        cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
 
         # Make sure that the logging directory exists
         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
             self.fatal("Could not create blobber upload directory")
             # Exit
 
-        base_cmd += ["--log-raw=-",
-                     "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
-                                                   "wpt_raw.log"),
-                     "--binary=%s" % self.binary_path,
-                     "--symbols-path=%s" % self.query_symbols_url(),
-                     "--stackwalk-binary=%s" % self.query_minidump_stackwalk()]
+        cmd += ["--log-raw=-",
+                "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
+                                              "wpt_raw.log"),
+                "--binary=%s" % self.binary_path,
+                "--symbols-path=%s" % self.query_symbols_url(),
+                "--stackwalk-binary=%s" % self.query_minidump_stackwalk()]
 
         for test_type in c.get("test_type", []):
-            base_cmd.append("--test-type=%s" % test_type)
+            cmd.append("--test-type=%s" % test_type)
 
         if c.get("e10s"):
-            base_cmd.append("--e10s")
+            cmd.append("--e10s")
 
         for opt in ["total_chunks", "this_chunk"]:
             val = c.get(opt)
             if val:
-                base_cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
+                cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
 
         options = list(c.get("options", []))
 
         str_format_values = {
             'binary_path': self.binary_path,
             'test_path': dirs["abs_wpttest_dir"],
             'test_install_path': dirs["abs_test_install_dir"],
             'abs_app_dir': abs_app_dir,
             'abs_work_dir': dirs["abs_work_dir"]
             }
 
-        opt_cmd = [item % str_format_values for item in options]
+        try_options, try_tests = self.try_args("web-platform-tests")
 
-        return base_cmd + opt_cmd
+        cmd.extend(self.query_options(options,
+                                      try_options,
+                                      str_format_values=str_format_values))
+        cmd.extend(self.query_tests_args(try_tests,
+                                         str_format_values=str_format_values))
+
+        return cmd
 
     def download_and_extract(self):
         super(WebPlatformTest, self).download_and_extract(
             target_unzip_dirs=["bin/*",
                                "config/*",
                                "mozbase/*",
                                "marionette/*",
                                "web-platform/*"],
             suite_categories=["web-platform"])
 
     def run_tests(self):
         dirs = self.query_abs_dirs()
         cmd = self._query_cmd()
-        cmd = self.append_harness_extra_args(cmd)
 
         parser = StructuredOutputParser(config=self.config,
                                         log_obj=self.log_obj)
 
         env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}
         env = self.query_env(partial_env=env, log_level=INFO)
 
         return_code = self.run_command(cmd,
--- a/testing/tools/autotry/autotry.py
+++ b/testing/tools/autotry/autotry.py
@@ -1,98 +1,149 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import argparse
 import sys
 import os
 import itertools
 import subprocess
 import which
 
 from collections import defaultdict
 
+import ConfigParser
 
-TRY_SYNTAX_TMPL = """
-try: -b %s -p %s -u %s -t none %s %s
-"""
+
+def parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.')
+    parser.add_argument('-p', dest='platforms', action="append",
+                        help='Platforms to run. (required if not found in the environment)')
+    parser.add_argument('-u', dest='tests', action="append",
+                        help='Test suites to run in their entirety')
+    parser.add_argument('-b', dest='builds', default='do',
+                        help='Build types to run (d for debug, o for optimized)')
+    parser.add_argument('--tag', dest='tags', action='append',
+                        help='Restrict tests to the given tag (may be specified multiple times)')
+    parser.add_argument('--and', action='store_true', dest="intersection",
+                        help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments')
+    parser.add_argument('--no-push', dest='push', action='store_false',
+                        help='Do not push to try as a result of running this command (if '
+                        'specified this command will only print calculated try '
+                        'syntax and selection info).')
+    parser.add_argument('extra_args', nargs=argparse.REMAINDER,
+                        help='Extra arguments to put in the try push')
+    parser.add_argument('-v', "--verbose", dest='verbose', action='store_true', default=False,
+                        help='Print detailed information about the resulting test selection '
+                        'and commands performed.')
+    return parser
+
 
 class AutoTry(object):
 
-    test_flavors = [
-        'browser-chrome',
-        'chrome',
-        'devtools-chrome',
-        'mochitest',
-        'xpcshell',
-        'reftest',
-        'crashtest',
-    ]
+    # Maps from flavors to the job names needed to run that flavour
+    flavor_jobs = {
+        'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
+        'xpcshell': ['xpcshell'],
+        'chrome': ['mochitest-o'],
+        'browser-chrome': ['mochitest-browser-chrome-1',
+                           'mochitest-e10s-browser-chrome-1'],
+        'devtools-chrome': ['mochitest-dt',
+                            'mochitest-e10s-devtools-chrome'],
+        'crashtest': ['crashtest', 'crashtest-e10s'],
+        'reftest': ['reftest', 'reftest-e10s'],
+        'web-platform-tests': ['web-platform-tests-1'],
+    }
+
+    flavor_suites = {
+        "mochitest": "mochitests",
+        "xpcshell": "xpcshell",
+        "chrome": "mochitest-o",
+        "browser-chrome": "mochitest-bc",
+        "devtools-chrome": "mochitest-dt",
+        "crashtest": "crashtest",
+        "reftest": "reftest",
+        "web-platform-tests": "web-platform-tests",
+    }
 
     def __init__(self, topsrcdir, resolver, mach_context):
         self.topsrcdir = topsrcdir
         self.resolver = resolver
         self.mach_context = mach_context
 
         if os.path.exists(os.path.join(self.topsrcdir, '.hg')):
             self._use_git = False
         else:
             self._use_git = True
 
-    def resolve_manifests(self, paths=None, tags=None):
-        if not (paths or tags):
-            return {}
+    def paths_by_flavor(self, paths=None, tags=None):
+        paths_by_flavor = defaultdict(set)
 
-        tests = list(self.resolver.resolve_tests(tags=tags,
-                                                 paths=paths,
-                                                 cwd=self.mach_context.cwd))
-        manifests_by_flavor = defaultdict(set)
+        if not (paths or tags):
+            return dict(paths_by_flavor)
+
+        tests = list(self.resolver.resolve_tests(paths=paths,
+                                                 tags=tags))
 
         for t in tests:
-            if t['flavor'] in AutoTry.test_flavors:
+            if t['flavor'] in self.flavor_suites:
                 flavor = t['flavor']
                 if 'subsuite' in t and t['subsuite'] == 'devtools':
                     flavor = 'devtools-chrome'
-                manifest = os.path.relpath(t['manifest'], self.topsrcdir)
-                manifests_by_flavor[flavor].add(manifest)
 
-        return dict(manifests_by_flavor)
+                for path in paths:
+                    if flavor in ["crashtest", "reftest"]:
+                        manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
+                        if manifest_relpath.startswith(path):
+                            paths_by_flavor[flavor].add(manifest_relpath)
+                    else:
+                        if t['file_relpath'].startswith(path):
+                            paths_by_flavor[flavor].add(path)
 
-    def calc_try_syntax(self, platforms, flavors, tests, extra_tests, builds,
-                        manifests, tags):
+        return dict(paths_by_flavor)
+
+    def remove_duplicates(self, paths_by_flavor, tests):
+        rv = {}
+        for item in paths_by_flavor:
+            if self.flavor_suites[item] not in tests:
+                rv[item] = paths_by_flavor[item].copy()
+        return rv
 
-        # Maps from flavors to the try syntax snippets implied by that flavor.
-        # TODO: put selected tests under their own builder/label to avoid
-        # confusion reading results on treeherder.
-        flavor_suites = {
-            'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
-            'xpcshell': ['xpcshell'],
-            'chrome': ['mochitest-o'],
-            'browser-chrome': ['mochitest-browser-chrome-1',
-                               'mochitest-e10s-browser-chrome-1'],
-            'devtools-chrome': ['mochitest-dt',
-                                'mochitest-e10s-devtools-chrome'],
-            'crashtest': ['crashtest', 'crashtest-e10s'],
-            'reftest': ['reftest', 'reftest-e10s'],
-        }
+    def calc_try_syntax(self, platforms, tests, builds, paths_by_flavor, tags, extra_args,
+                        intersection):
+        parts = ["try:", "-b", builds, "-p", ",".join(platforms)]
+
+        suites = set(tests) if not intersection else set()
+        paths = set()
+        for flavor, flavor_tests in paths_by_flavor.iteritems():
+            suite = self.flavor_suites[flavor]
+            if suite not in suites and (not intersection or suite in tests):
+                for job_name in self.flavor_jobs[flavor]:
+                    for test in flavor_tests:
+                        paths.add("%s:%s" % (flavor, test))
+                    suites.add(job_name)
+
+        if not suites:
+            raise ValueError("No tests found matching filters")
+
+        parts.append("-u")
+        parts.append(",".join(sorted(suites)))
 
         if tags:
-            tags = ' '.join('--tag %s' % t for t in tags)
-        else:
-            tags = ''
+            parts.append(' '.join('--tag %s' % t for t in tags))
+
+        if extra_args is not None:
+            parts.extend(extra_args)
 
-        if not tests:
-            tests = ','.join(itertools.chain(*(flavor_suites[f] for f in flavors)))
-            if extra_tests:
-                tests += ',%s' % (extra_tests)
+        if paths:
+            parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
 
-        manifests = ' '.join(manifests)
-        if manifests:
-            manifests = '--try-test-paths %s' % manifests
-        return TRY_SYNTAX_TMPL % (builds, platforms, tests, manifests, tags)
+        return " ".join(parts)
 
     def _run_git(self, *args):
         args = ['git'] + list(args)
         ret = subprocess.call(args)
         if ret:
             print('ERROR git command %s returned %s' %
                   (args, ret))
             sys.exit(1)