Backed out 4 changesets (bug 1193264, bug 1193215, bug 1188483, bug 1204120) for test bustage and bustage on a CLOSED TREE
authorCarsten "Tomcat" Book <cbook@mozilla.com>
Thu, 24 Sep 2015 13:58:20 +0200
changeset 299390 0a5b448cd09ea78e7ef930c3910b488ac27efebc
parent 299389 a9a4292b2df3f147501d13871885abc3d1dea7b6
child 299391 e15e24b44323bb0f2fae6c22c03f9cc08044bb73
push id1001
push userraliiev@mozilla.com
push dateMon, 18 Jan 2016 19:06:03 +0000
treeherdermozilla-release@8b89261f3ac4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1193264, 1193215, 1188483, 1204120
milestone44.0a1
backs outa9a4292b2df3f147501d13871885abc3d1dea7b6
9748ff0d2836c7a62da2f404906af6bf1b96c9ce
a0b952bb6620f2833fe87328ed9aae1e6ff8c344
63c61416c2f7911a3bdf68a319bca14bbcb48e53
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 4 changesets (bug 1193264, bug 1193215, bug 1188483, bug 1204120) for test bustage and bustage on a CLOSED TREE Backed out changeset a9a4292b2df3 (bug 1188483) Backed out changeset 9748ff0d2836 (bug 1204120) Backed out changeset a0b952bb6620 (bug 1193264) Backed out changeset 63c61416c2f7 (bug 1193215)
testing/mach_commands.py
testing/mozharness/configs/b2g/emulator_automation_config.py
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/mozharness/mozilla/buildbot.py
testing/mozharness/mozharness/mozilla/l10n/locales.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
testing/mozharness/mozharness/mozilla/testing/try_tools.py
testing/mozharness/scripts/android_emulator_unittest.py
testing/mozharness/scripts/android_panda.py
testing/mozharness/scripts/b2g_desktop_unittest.py
testing/mozharness/scripts/b2g_emulator_unittest.py
testing/mozharness/scripts/desktop_l10n.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/marionette.py
testing/mozharness/scripts/web_platform_tests.py
testing/tools/autotry/autotry.py
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -5,17 +5,16 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
 import sys
 import tempfile
 import subprocess
 import shutil
-from collections import defaultdict
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 from mozbuild.base import MachCommandBase
@@ -432,198 +431,135 @@ class JsapiTestsCommand(MachCommandBase)
         jsapi_tests_cmd = [os.path.join(self.bindir, executable_name('jsapi-tests'))]
         if params['test_name']:
             jsapi_tests_cmd.append(params['test_name'])
 
         jsapi_tests_result = subprocess.call(jsapi_tests_cmd)
 
         return jsapi_tests_result
 
-def autotry_parser():
-    from autotry import arg_parser
-    return arg_parser()
 
 @CommandProvider
 class PushToTry(MachCommandBase):
-    def normalise_list(self, items, allow_subitems=False):
-        from autotry import parse_arg
 
-        rv = defaultdict(list)
-        for item in items:
-            parsed = parse_arg(item)
-            for key, values in parsed.iteritems():
-                rv[key].extend(values)
-
-        if not allow_subitems:
-            if not all(item == [] for item in rv.itervalues()):
-                raise ValueError("Unexpected subitems in argument")
-            return rv.keys()
-        else:
-            return rv
-
-    def validate_args(self, **kwargs):
-        if not kwargs["paths"] and not kwargs["tests"] and not kwargs["tags"]:
-            print("Paths, tags, or tests must be specified as an argument to autotry.")
+    def validate_args(self, paths, tests, tags, builds, platforms):
+        if not any([len(paths), tests, tags]):
+            print("Paths, tests, or tags must be specified.")
             sys.exit(1)
 
-        if kwargs["platforms"] is None:
-            print("Platforms must be specified as an argument to autotry")
-            sys.exit(1)
-
-        try:
-            platforms = self.normalise_list(kwargs["platforms"])
-        except ValueError as e:
-            print("Error parsing -p argument:\n%s" % e.message)
-            sys.exit(1)
+        if platforms is None:
+            platforms = os.environ['AUTOTRY_PLATFORM_HINT']
 
-        try:
-            tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
-                     if kwargs["tests"] else {})
-        except ValueError as e:
-            print("Error parsing -u argument:\n%s" % e.message)
-            sys.exit(1)
-
-        try:
-            talos = self.normalise_list(kwargs["talos"]) if kwargs["talos"] else []
-        except ValueError as e:
-            print("Error parsing -t argument:\n%s" % e.message)
-            sys.exit(1)
-
-        paths = []
-        for p in kwargs["paths"]:
+        for p in paths:
             p = os.path.normpath(os.path.abspath(p))
             if not p.startswith(self.topsrcdir):
                 print('Specified path "%s" is outside of the srcdir, unable to'
                       ' specify tests outside of the srcdir' % p)
                 sys.exit(1)
             if len(p) <= len(self.topsrcdir):
                 print('Specified path "%s" is at the top of the srcdir and would'
                       ' select all tests.' % p)
                 sys.exit(1)
-            paths.append(os.path.relpath(p, self.topsrcdir))
 
-        try:
-            tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
-        except ValueError as e:
-            print("Error parsing --tags argument:\n%s" % e.message)
-            sys.exit(1)
-
-        return kwargs["builds"], platforms, tests, talos, paths, tags, kwargs["extra_args"]
-
-
-    @Command('try',
-             category='testing',
-             description='Push selected tests to the try server',
-             parser=autotry_parser)
-
-    def autotry(self, **kwargs):
-        """Autotry is in beta, please file bugs blocking 1149670.
-
-        Push the current tree to try, with the specified syntax.
+        return builds, platforms
 
-        Build options, platforms and regression tests may be selected
-        using the usual try options (-b, -p and -u respectively). In
-        addition, tests in a given directory may be automatically
-        selected by passing that directory as a positional argument to the
-        command. For example:
-
-        mach try -b d -p linux64 dom testing/web-platform/tests/dom
-
-        would schedule a try run for linux64 debug consisting of all
-        tests under dom/ and testing/web-platform/tests/dom.
-
-        Test selection using positional arguments is available for
-        mochitests, reftests, xpcshell tests and web-platform-tests.
+    @Command('try', category='testing', description='Push selected tests to the try server')
+    @CommandArgument('paths', nargs='*', help='Paths to search for tests to run on try.')
+    @CommandArgument('-n', dest='verbose', action='store_true', default=False,
+                     help='Print detailed information about the resulting test selection '
+                          'and commands performed.')
+    @CommandArgument('-p', dest='platforms', required='AUTOTRY_PLATFORM_HINT' not in os.environ,
+                     help='Platforms to run. (required if not found in the environment)')
+    @CommandArgument('-u', dest='tests',
+                     help='Test jobs to run. These will be used in place of suites '
+                          'determined by test paths, if any.')
+    @CommandArgument('--extra', dest='extra_tests',
+                     help='Additional tests to run. These will be added to suites '
+                          'determined by test paths, if any.')
+    @CommandArgument('-b', dest='builds', default='do',
+                     help='Build types to run (d for debug, o for optimized)')
+    @CommandArgument('--tag', dest='tags', action='append',
+                     help='Restrict tests to the given tag (may be specified multiple times)')
+    @CommandArgument('--no-push', dest='push', action='store_false',
+                     help='Do not push to try as a result of running this command (if '
+                          'specified this command will only print calculated try '
+                          'syntax and selection info).')
+    def autotry(self, builds=None, platforms=None, paths=None, verbose=None,
+                extra_tests=None, push=None, tags=None, tests=None):
+        """mach try is under development, please file bugs blocking 1149670.
 
-        Tests may be also filtered by passing --tag to the command,
-        which will run only tests marked as having the specified
-        tags e.g.
-
-        mach try -b d -p win64 --tag media
-
-        would run all tests tagged 'media' on Windows 64.
+        Pushes the specified tests to try. The simplest way to specify tests is
+        by using the -u argument, which will behave as usual for try syntax.
+        This command also provides a mechanism to select test jobs and tests
+        within a job by path based on tests present in the tree under that
+        path. Mochitests, xpcshell tests, and reftests are eligible for
+        selection by this mechanism. Selected tests will be run in a single
+        chunk of the relevant suite, at this time in chunk 1.
 
-        If both positional arguments or tags and -u are supplied, the
-        suites in -u will be run in full. Where tests are selected by
-        positional argument they will be run in a single chunk.
+        Specifying platforms is still required with the -p argument (a default
+        is taken from the AUTOTRY_PLATFORM_HINT environment variable if set).
 
-        If no build option is selected, both debug and opt will be
-        scheduled. If no platform is selected a default is taken from
-        the AUTOTRY_PLATFORM_HINT environment variable, if set.
+        Tests may be further filtered by passing one or more --tag to the
+        command. If one or more --tag is specified with out paths or -u,
+        tests with the given tags will be run in a single chunk of
+        applicable suites.
+
+        To run suites in addition to those determined from the tree, they
+        can be passed to the --extra arguent.
 
         The command requires either its own mercurial extension ("push-to-try",
         installable from mach mercurial-setup) or a git repo using git-cinnabar
         (available at https://github.com/glandium/git-cinnabar).
-
         """
 
         from mozbuild.testing import TestResolver
         from mozbuild.controller.building import BuildDriver
         from autotry import AutoTry
+        import pprint
 
         print("mach try is under development, please file bugs blocking 1149670.")
 
+        builds, platforms = self.validate_args(paths, tests, tags, builds, platforms)
         resolver = self._spawn(TestResolver)
-        at = AutoTry(self.topsrcdir, resolver, self._mach_context)
-
-        if kwargs["load"] is not None:
-            defaults = at.load_config(kwargs["load"])
 
-            if defaults is None:
-                print("No saved configuration called %s found in autotry.ini" % kwargs["load"],
-                      file=sys.stderr)
-
-            for key, value in kwargs.iteritems():
-                if value in (None, []) and key in defaults:
-                    kwargs[key] = defaults[key]
-
-        builds, platforms, tests, talos, paths, tags, extra_args = self.validate_args(**kwargs)
-
-        if kwargs["push"] and at.find_uncommited_changes():
+        at = AutoTry(self.topsrcdir, resolver, self._mach_context)
+        if at.find_uncommited_changes():
             print('ERROR please commit changes before continuing')
             sys.exit(1)
 
         if paths or tags:
             driver = self._spawn(BuildDriver)
             driver.install_tests(remove=False)
 
-            paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
-                     for item in paths]
-            paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags)
-
-            if not paths_by_flavor and not tests:
-                print("No tests were found when attempting to resolve paths:\n\n\t%s" %
-                      paths)
-                sys.exit(1)
+        manifests_by_flavor = at.resolve_manifests(paths=paths, tags=tags)
 
-            if not kwargs["intersection"]:
-                paths_by_flavor = at.remove_duplicates(paths_by_flavor, tests)
-        else:
-            paths_by_flavor = {}
-
-        try:
-            msg = at.calc_try_syntax(platforms, tests, talos, builds, paths_by_flavor, tags,
-                                     extra_args, kwargs["intersection"])
-        except ValueError as e:
-            print(e.message)
+        if not manifests_by_flavor and not tests:
+            print("No tests were found when attempting to resolve paths:\n\n\t%s" %
+                  paths)
             sys.exit(1)
 
-        if kwargs["verbose"] and paths_by_flavor:
-            print('The following tests will be selected: ')
-            for flavor, paths in paths_by_flavor.iteritems():
-                print("%s: %s" % (flavor, ",".join(paths)))
+        all_manifests = set()
+        for m in manifests_by_flavor.values():
+            all_manifests |= m
+        all_manifests = list(all_manifests)
+
+        msg = at.calc_try_syntax(platforms, manifests_by_flavor.keys(), tests,
+                                 extra_tests, builds, all_manifests, tags)
 
-        if kwargs["verbose"] or not kwargs["push"]:
-            print('The following try syntax was calculated:\n%s' % msg)
+        if verbose and manifests_by_flavor:
+            print('Tests from the following manifests will be selected: ')
+            pprint.pprint(manifests_by_flavor)
 
-        if kwargs["push"]:
-            at.push_to_try(msg, kwargs["verbose"])
+        if verbose or not push:
+            print('The following try syntax was calculated:\n\n\t%s\n' % msg)
 
-        if kwargs["save"] is not None:
-            at.save_config(kwargs["save"], msg)
+        if push:
+            at.push_to_try(msg, verbose)
+
+        return
 
 
 def get_parser(argv=None):
     parser = ArgumentParser()
     parser.add_argument(dest="suite_name",
                         nargs=1,
                         choices=['mochitest'],
                         type=str,
--- a/testing/mozharness/configs/b2g/emulator_automation_config.py
+++ b/testing/mozharness/configs/b2g/emulator_automation_config.py
@@ -113,18 +113,18 @@ config = {
                 "--busybox=%(busybox)s",
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--quiet",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
+                "%(test_path)s"
             ],
-            "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
         },
         "mochitest-chrome": {
             "options": [
                 "--adbpath=%(adbpath)s",
                 "--b2gpath=%(b2gpath)s",
                 "--emulator=%(emulator)s",
@@ -137,18 +137,18 @@ config = {
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--quiet",
                 "--chrome",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
+                "%(test_path)s"
             ],
-            "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
         },
         "reftest": {
             "options": [
                 "--adbpath=%(adbpath)s",
                 "--b2gpath=%(b2gpath)s",
                 "--emulator=%(emulator)s",
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -251,17 +251,17 @@ config = {
         "xpcshell-addons": {
             "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
                         "--tag=addons",
                         "--manifest=tests/xpcshell/tests/all-test-dirs.list"],
             "tests": []
         },
     },
     "all_cppunittest_suites": {
-        "cppunittest": {"tests": ["tests/cppunittest"]}
+        "cppunittest": ["tests/cppunittest"]
     },
     "all_gtest_suites": {
         "gtest": []
     },
     "all_jittest_suites": {
         "jittest": [],
         "jittest1": ["--total-chunks=2", "--this-chunk=1"],
         "jittest2": ["--total-chunks=2", "--this-chunk=2"],
--- a/testing/mozharness/mozharness/mozilla/buildbot.py
+++ b/testing/mozharness/mozharness/mozilla/buildbot.py
@@ -44,17 +44,17 @@ EXIT_STATUS_DICT = {
     TBPL_EXCEPTION: 3,
     TBPL_RETRY: 4,
 }
 TBPL_WORST_LEVEL_TUPLE = (TBPL_RETRY, TBPL_EXCEPTION, TBPL_FAILURE,
                           TBPL_WARNING, TBPL_SUCCESS)
 
 
 class BuildbotMixin(object):
-    buildbot_config = {}
+    buildbot_config = None
     buildbot_properties = {}
     worst_buildbot_status = TBPL_SUCCESS
 
     def read_buildbot_config(self):
         c = self.config
         if not c.get("buildbot_json_path"):
             # If we need to fail out, add postflight_read_buildbot_config()
             self.info("buildbot_json_path is not set.  Skipping...")
--- a/testing/mozharness/mozharness/mozilla/l10n/locales.py
+++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py
@@ -24,90 +24,44 @@ class LocalesMixin(ChunkingMixin):
     def __init__(self, **kwargs):
         """ Mixins generally don't have an __init__.
         This breaks super().__init__() for children.
         However, this is needed to override the query_abs_dirs()
         """
         self.abs_dirs = None
         self.locales = None
         self.gecko_locale_revisions = None
-        self.l10n_revisions = {}
 
     def query_locales(self):
         if self.locales is not None:
             return self.locales
         c = self.config
+        locales = c.get("locales", None)
         ignore_locales = c.get("ignore_locales", [])
         additional_locales = c.get("additional_locales", [])
-        # List of locales can be set by using different methods in the
-        # following order:
-        # 1. "locales" buildbot property: a string of locale:revision separated
-        # by space
-        # 2. "MOZ_LOCALES" env variable: a string of locale:revision separated
-        # by space
-        # 3. self.config["locales"] which can be either coming from the config
-        # or from --locale command line argument
-        # 4. using self.config["locales_file"] l10n changesets file
-        locales = None
 
-        # Buildbot property
-        if hasattr(self, 'read_buildbot_config'):
-            self.read_buildbot_config()
-            locales = self.buildbot_config.get("locales")
-            if locales:
-                self.info("Using locales from buildbot: %s" % locales)
-                locales = locales.split()
-        else:
-            self.info("'read_buildbot_config()' is missing, ignoring buildbot"
-                      " properties")
-
-        # Environment variable
-        if not locales and "MOZ_LOCALES" in os.environ:
-            self.debug("Using locales from environment: %s" %
-                       os.environ["MOZ_LOCALES"])
-            locales = os.environ["MOZ_LOCALES"].split()
-
-        # Command line or config
-        if not locales and c.get("locales", None):
-            locales = c["locales"]
-            self.debug("Using locales from config/CLI: %s" % locales)
-
-        # parse locale:revision if set
-        if locales:
-            for l in locales:
-                if ":" in l:
-                    # revision specified in locale string
-                    locale, revision = l.split(":", 1)
-                    self.debug("Using %s:%s" % (locale, revision))
-                    self.l10n_revisions[locale] = revision
-            # clean up locale by removing revisions
-            locales = [l.split(":")[0] for l in locales]
-
-        if not locales and 'locales_file' in c:
-            locales_file = os.path.join(c['base_work_dir'], c['work_dir'],
-                                        c['locales_file'])
-            locales = self.parse_locales_file(locales_file)
-
-        if not locales:
-            self.fatal("No locales set!")
-
+        if locales is None:
+            locales = []
+            if 'locales_file' in c:
+                # Best way to get abs/relative path to this?
+                locales_file = os.path.join(c['base_work_dir'], c['work_dir'],
+                                            c['locales_file'])
+                locales = self.parse_locales_file(locales_file)
+            else:
+                self.fatal("No way to determine locales!")
         for locale in ignore_locales:
             if locale in locales:
                 self.debug("Ignoring locale %s." % locale)
                 locales.remove(locale)
-                if locale in self.l10n_revisions:
-                    del self.l10n_revisions[locale]
-
         for locale in additional_locales:
             if locale not in locales:
                 self.debug("Adding locale %s." % locale)
                 locales.append(locale)
-
-        if not locales:
-            return None
+        if locales is None:
+            return
         if 'total_locale_chunks' and 'this_locale_chunk' in c:
             self.debug("Pre-chunking locale list: %s" % str(locales))
             locales = self.query_chunked_list(locales,
                                               c['this_locale_chunk'],
                                               c['total_locale_chunks'],
                                               sort=True)
             self.debug("Post-chunking locale list: %s" % locales)
         self.locales = locales
@@ -120,24 +74,25 @@ class LocalesMixin(ChunkingMixin):
 
     def parse_locales_file(self, locales_file):
         locales = []
         c = self.config
         platform = c.get("locales_platform", None)
 
         if locales_file.endswith('json'):
             locales_json = parse_config_file(locales_file)
+            self.locale_dict = {}
             for locale in locales_json.keys():
                 if isinstance(locales_json[locale], dict):
                     if platform and platform not in locales_json[locale]['platforms']:
                         continue
-                    self.l10n_revisions[locale] = locales_json[locale]['revision']
+                    self.locale_dict[locale] = locales_json[locale]['revision']
                 else:
                     # some other way of getting this?
-                    self.l10n_revisions[locale] = 'default'
+                    self.locale_dict[locale] = 'default'
                 locales.append(locale)
         else:
             locales = self.read_from_file(locales_file).split()
         return locales
 
     def run_compare_locales(self, locale, halt_on_failure=False):
         dirs = self.query_abs_dirs()
         compare_locales_script = os.path.join(dirs['abs_compare_locales_dir'],
@@ -217,18 +172,18 @@ class LocalesMixin(ChunkingMixin):
             self.vcs_checkout_repos(repos, tag_override=c.get('tag_override'))
         # Pull locales
         locales = self.query_locales()
         locale_repos = []
         if c.get("user_repo_override"):
             hg_l10n_base = hg_l10n_base % {"user_repo_override": c["user_repo_override"]}
         for locale in locales:
             tag = c.get('hg_l10n_tag', 'default')
-            if self.l10n_revisions.get(locale):
-                tag = self.l10n_revisions[locale]
+            if hasattr(self, 'locale_dict'):
+                tag = self.locale_dict[locale]
             locale_repos.append({
                 'repo': "%s/%s" % (hg_l10n_base, locale),
                 'tag': tag,
                 'vcs': vcs
             })
         revs = self.vcs_checkout_repos(repo_list=locale_repos,
                                        parent_dir=parent_dir,
                                        tag_override=c.get('tag_override'))
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -21,17 +21,17 @@ from mozharness.base.python import (
     VirtualenvMixin,
     virtualenv_config_options,
 )
 from mozharness.mozilla.buildbot import BuildbotMixin, TBPL_WARNING
 from mozharness.mozilla.proxxy import Proxxy
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.taskcluster_helper import TaskClusterArtifactFinderMixin
 from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
-from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options
+from mozharness.mozilla.testing.try_tools import TryToolsMixin
 from mozharness.mozilla.tooltool import TooltoolMixin
 
 from mozharness.lib.python.authentication import get_credentials
 
 INSTALLER_SUFFIXES = ('.tar.bz2', '.zip', '.dmg', '.exe', '.apk', '.tar.gz')
 
 testing_config_options = [
     [["--installer-url"],
@@ -78,17 +78,17 @@ testing_config_options = [
       }],
     [["--download-symbols"],
      {"action": "store",
      "dest": "download_symbols",
      "type": "choice",
      "choices": ['ondemand', 'true'],
      "help": "Download and extract crash reporter symbols.",
       }],
-] + copy.deepcopy(virtualenv_config_options) + copy.deepcopy(try_config_options)
+] + copy.deepcopy(virtualenv_config_options)
 
 
 # TestingMixin {{{1
 class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin,
                    TaskClusterArtifactFinderMixin, TooltoolMixin, TryToolsMixin):
     """
     The steps to identify + download the proper bits for [browser] unit
     tests and Talos.
@@ -671,45 +671,16 @@ Did you run with --create-virtualenv? Is
             else:
                 self.warning("minidump stackwalk path was given but couldn't be found. "
                              "Tried looking in '%s'" % abs_minidump_path)
                 # don't burn the job but we should at least turn them orange so it is caught
                 self.buildbot_status(TBPL_WARNING, WARNING)
 
         return self.minidump_stackwalk_path
 
-    def query_options(self, *args, **kwargs):
-        if "str_format_values" in kwargs:
-            str_format_values = kwargs.pop("str_format_values")
-        else:
-            str_format_values = {}
-
-        arguments = []
-
-        for arg in args:
-            if arg is not None:
-                arguments.extend(argument % str_format_values for argument in arg)
-
-        return arguments
-
-    def query_tests_args(self, *args, **kwargs):
-        if "str_format_values" in kwargs:
-            str_format_values = kwargs.pop("str_format_values")
-        else:
-            str_format_values = {}
-
-        arguments = []
-
-        for arg in reversed(args):
-            if arg:
-                arguments.append("--")
-                arguments.extend(argument % str_format_values for argument in arg)
-                break
-
-        return arguments
 
     def _run_cmd_checks(self, suites):
         if not suites:
             return
         dirs = self.query_abs_dirs()
         for suite in suites:
             # XXX platform.architecture() may give incorrect values for some
             # platforms like mac as excutable files may be universal
--- a/testing/mozharness/mozharness/mozilla/testing/try_tools.py
+++ b/testing/mozharness/mozharness/mozilla/testing/try_tools.py
@@ -3,91 +3,62 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import argparse
 import os
 import re
-from collections import defaultdict
 
 from mozharness.base.script import PostScriptAction
 from mozharness.base.transfer import TransferMixin
 
-try_config_options = [
-    [["--try-message"],
-     {"action": "store",
-     "dest": "try_message",
-     "default": None,
-     "help": "try syntax string to select tests to run",
-      }],
-]
-
-test_flavors = {
-    'browser-chrome': {},
-    'chrome': {},
-    'devtools-chrome': {},
-    'mochitest': {},
-    'xpcshell' :{},
-    'reftest': {
-        "path": lambda x: os.path.join("tests", "reftest", "tests", x)
-    },
-    'crashtest': {
-        "path": lambda x: os.path.join("tests", "reftest", "tests", x)
-    },
-    'web-platform-tests': {
-        "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
-    }
-}
 
 class TryToolsMixin(TransferMixin):
     """Utility functions for an interface between try syntax and out test harnesses.
     Requires log and script mixins."""
 
     harness_extra_args = None
-    try_test_paths = {}
+    try_test_paths = []
     known_try_arguments = {
         '--tag': {
             'action': 'append',
             'dest': 'tags',
             'default': None,
         },
     }
 
     def _extract_try_message(self):
         msg = None
-        if "try_message" in self.config and self.config["try_message"]:
-            msg = self.config["try_message"]
-        else:
-            if self.buildbot_config['sourcestamp']['changes']:
-                msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments']
+        if self.buildbot_config['sourcestamp']['changes']:
+            msg = self.buildbot_config['sourcestamp']['changes'][-1]['comments']
 
-            if msg is None or len(msg) == 1024:
-                # This commit message was potentially truncated, get the full message
-                # from hg.
-                props = self.buildbot_config['properties']
-                rev = props['revision']
-                repo = props['repo_path']
-                url = 'https://hg.mozilla.org/%s/json-pushes?changeset=%s&full=1' % (repo, rev)
+        if msg is None or len(msg) == 1024:
+            # This commit message was potentially truncated, get the full message
+            # from hg.
+            props = self.buildbot_config['properties']
+            rev = props['revision']
+            repo = props['repo_path']
+            url = 'https://hg.mozilla.org/%s/json-pushes?changeset=%s&full=1' % (repo, rev)
 
-                pushinfo = self.load_json_from_url(url)
-                for k, v in pushinfo.items():
-                    if isinstance(v, dict) and 'changesets' in v:
-                        msg = v['changesets'][-1]['desc']
+            pushinfo = self.load_json_from_url(url)
+            for k, v in pushinfo.items():
+                if isinstance(v, dict) and 'changesets' in v:
+                    msg = v['changesets'][-1]['desc']
 
-            if not msg and 'try_syntax' in self.buildbot_config['properties']:
-                # If we don't find try syntax in the usual place, check for it in an
-                # alternate property available to tools using self-serve.
-                msg = self.buildbot_config['properties']['try_syntax']
+        if not msg and 'try_syntax' in self.buildbot_config['properties']:
+            # If we don't find try syntax in the usual place, check for it in an
+            # alternate property available to tools using self-serve.
+            msg = self.buildbot_config['properties']['try_syntax']
 
         return msg
 
     @PostScriptAction('download-and-extract')
-    def set_extra_try_arguments(self, action, success=None):
+    def _set_extra_try_arguments(self, action, success=None):
         """Finds a commit message and parses it for extra arguments to pass to the test
         harness command line and test paths used to filter manifests.
 
         Extracting arguments from a commit message taken directly from the try_parser.
         """
         if (not self.buildbot_config or 'properties' not in self.buildbot_config or
                 self.buildbot_config['properties'].get('branch') != 'try'):
             return
@@ -131,17 +102,17 @@ class TryToolsMixin(TransferMixin):
                            'arguments to the harness process.')
             if 'dest' in opts:
                 label_dict[opts['dest']] = label
 
             parser.add_argument(label, **opts)
 
         parser.add_argument('--try-test-paths', nargs='*')
         (args, _) = parser.parse_known_args(all_try_args)
-        self.try_test_paths = self._group_test_paths(args.try_test_paths)
+        self.try_test_paths = args.try_test_paths
         del args.try_test_paths
 
         out_args = []
         # This is a pretty hacky way to echo arguments down to the harness.
         # Hopefully this can be improved once we have a configuration system
         # in tree for harnesses that relies less on a command line.
         for (arg, value) in vars(args).iteritems():
             if value:
@@ -151,40 +122,96 @@ class TryToolsMixin(TransferMixin):
                     out_args.append(label)
                 elif isinstance(value, list):
                     out_args.extend(['%s=%s' % (label, el) for el in value])
                 else:
                     out_args.append('%s=%s' % (label, value))
 
         self.harness_extra_args = out_args
 
-    def _group_test_paths(self, args):
-        rv = defaultdict(list)
+    def _resolve_specified_manifests(self):
+        if not self.try_test_paths:
+            return None
 
-        if args is None:
-            return rv
+        target_manifests = set(self.try_test_paths)
 
-        for item in args:
-            suite, path = item.split(":", 1)
-            rv[suite].append(path)
-        return rv
+        def filter_ini_manifest(line):
+            # Lines are formatted as [include:<path>], we care about <path>.
+            parts = line.split(':')
+            term = line
+            if len(parts) == 2:
+                term = parts[1]
+            if term.endswith(']'):
+                term = term[:-1]
+            if (term in target_manifests or
+                any(term.startswith(l) for l in target_manifests)):
+                return True
+            return False
 
-    def try_args(self, flavor):
-        """Get arguments, test_list derived from try syntax to apply to a command"""
-        # TODO: Detect and reject incompatible arguments
-        args = self.harness_extra_args[:] if self.harness_extra_args else []
+        def filter_list_manifest(line):
+            # Lines are usually formatted as "include <path>", we care about <path>.
+            parts = line.split()
+            term = line
+            if len(parts) == 2:
+                term = parts[1]
+            # Reftest master manifests also include skip-if lines and relative
+            # paths we aren't doing to resolve here, so unlike above this is just
+            # a substring check.
+            if (term in target_manifests or
+                any(l in term for l in target_manifests)):
+                return True
+            return False
 
-        if self.try_test_paths.get(flavor):
-            self.info('TinderboxPrint: Tests will be run from the following '
-                      'files: %s.' % ','.join(self.try_test_paths[flavor]))
-            args.extend(['--this-chunk=1', '--total-chunks=1'])
+        # The master manifests we need to filter for target manifests.
+        # TODO: All this needs to go in a config file somewhere and get sewn
+        # into the job definition so its less likely to break as things are
+        # modified. One straightforward way to achieve this would be with a key
+        # in the tree manifest for the master manifest path, however the current
+        # tree manifests don't distinguish between flavors of mochitests to this
+        # isn't straightforward.
+        master_manifests = [
+            ('mochitest/chrome/chrome.ini', filter_ini_manifest),
+            ('mochitest/browser/browser-chrome.ini', filter_ini_manifest),
+            ('mochitest/tests/mochitest.ini', filter_ini_manifest),
+            ('xpcshell/tests/all-test-dirs.list', filter_ini_manifest),
+            ('xpcshell/tests/xpcshell.ini', filter_ini_manifest),
+            ('reftest/tests/layout/reftests/reftest.list', filter_list_manifest),
+            ('reftest/tests/testing/crashtest/crashtests.list', filter_list_manifest),
+        ]
+
+        dirs = self.query_abs_dirs()
+        tests_dir = dirs.get('abs_test_install_dir',
+                             os.path.join(dirs['abs_work_dir'], 'tests'))
+        master_manifests = [(os.path.join(tests_dir, name), filter_fn) for (name, filter_fn) in
+                            master_manifests]
+        for m, filter_fn in master_manifests:
+            if not os.path.isfile(m):
+                continue
 
-            path_func = test_flavors[flavor].get("path", lambda x:x)
-            tests = [path_func(item) for item in self.try_test_paths[flavor]]
-        else:
-            tests = []
+            self.info("Filtering master manifest at: %s" % m)
+            lines = self.read_from_file(m).splitlines()
+
+            out_lines = [line for line in lines if filter_fn(line)]
+
+            self.rmtree(m)
+            self.write_to_file(m, '\n'.join(out_lines))
+
 
-        if args or tests:
-            self.info('TinderboxPrint: The following arguments were forwarded from mozharness '
-                      'to the test command:\nTinderboxPrint: \t%s -- %s' %
-                      (" ".join(args), " ".join(tests)))
+    def append_harness_extra_args(self, cmd):
+        """Append arguments derived from try syntax to a command."""
+        # TODO: Detect and reject incompatible arguments
+        extra_args = self.harness_extra_args[:] if self.harness_extra_args else []
+        if self.try_test_paths:
+            self._resolve_specified_manifests()
+            self.info('TinderboxPrint: Tests will be run from the following '
+                      'manifests: %s.' % ','.join(self.try_test_paths))
+            extra_args.extend(['--this-chunk=1', '--total-chunks=1'])
 
-        return args, tests
+        if not extra_args:
+            return cmd
+
+        out_cmd = cmd[:]
+        out_cmd.extend(extra_args)
+        self.info('TinderboxPrint: The following arguments were forwarded from mozharness '
+                  'to the test command:\nTinderboxPrint: \t%s' %
+                  extra_args)
+
+        return out_cmd
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -478,22 +478,24 @@ class AndroidEmulatorTest(BlobUploadMixi
 
         for arg in self.test_suite_definitions[self.test_suite]["extra_args"]:
             argname = arg.split('=')[0]
             # only add the extra arg if it wasn't already defined by in-tree configs
             if any(a.split('=')[0] == argname for a in cmd):
                 continue
             cmd.append(arg)
 
-        try_options, try_tests = self.try_args(suite_category)
-        cmd.extend(try_options)
-        cmd.extend(self.query_tests_args(
-            self.config["suite_definitions"][suite_category].get("tests"),
-            self.test_suite_definitions[self.test_suite].get("tests"),
-            try_tests))
+        tests = None
+        if "tests" in self.test_suite_definitions[self.test_suite]:
+            tests = self.test_suite_definitions[self.test_suite]["tests"]
+        elif "tests" in self.config["suite_definitions"][suite_category]:
+            tests = self.config["suite_definitions"][suite_category]["tests"]
+
+        if tests:
+            cmd.extend(tests)
 
         return cmd
 
     def _tooltool_fetch(self, url):
         c = self.config
         dirs = self.query_abs_dirs()
 
         manifest_path = self.download_file(
@@ -657,17 +659,17 @@ class AndroidEmulatorTest(BlobUploadMixi
 
         self.info("Finished installing apps for %s" % self.emulator["name"])
 
     def run_tests(self):
         """
         Run the tests
         """
         cmd = self._build_command()
-
+        cmd = self.append_harness_extra_args(cmd)
         try:
             cwd = self._query_tests_dir()
         except:
             self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
         env = self.query_env()
         if self.query_minidump_stackwalk():
             env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
         env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
--- a/testing/mozharness/scripts/android_panda.py
+++ b/testing/mozharness/scripts/android_panda.py
@@ -212,30 +212,16 @@ class PandaTest(TestingMixin, MercurialS
         """
         if self.config.get('run_all_suites'):
             target_categories = SUITE_CATEGORIES
         else:
             target_categories = [cat for cat in SUITE_CATEGORIES
                                  if self._query_specified_suites(cat) is not None]
         super(PandaTest, self).download_and_extract(suite_categories=target_categories)
 
-    def _query_try_flavor(self, category, suite):
-        flavors = {
-            "mochitest": [("plain.*", "mochitest"),
-                          ("browser-chrome.*", "browser-chrome"),
-                          ("mochitest-devtools-chrome.*", "devtools-chrome"),
-                          ("chrome", "chrome")],
-            "xpcshell": [("xpcshell", "xpcshell")],
-            "reftest": [("reftest", "reftest"),
-                        ("crashtest", "crashtest")]
-        }
-        for suite_pattern, flavor in flavors.get(category, []):
-            if re.compile(suite_pattern).match(suite):
-                return flavor
-
     def _run_category_suites(self, suite_category, preflight_run_method=None):
         """run suite(s) to a specific category"""
 
         env = self.query_env(partial_env={'DM_TRANS': "sut", 'TEST_DEVICE': self.mozpool_device})
         self.info("Running tests...")
 
         suites = self._query_specified_suites(suite_category)
         level = INFO
@@ -255,23 +241,21 @@ class PandaTest(TestingMixin, MercurialS
                 if 'robocop' in suite:
                     self._download_robocop_apk()
                 if 'jittest' in suite:
                     should_install_app = False
 
                 if should_install_app:
                     self._install_app()
                 cmd = abs_base_cmd[:]
-
-                flavor = self._query_try_flavor(suite_category, suite)
-                try_options, try_tests = self.try_args(flavor)
+                replace_dict = {}
+                for arg in suites[suite]:
+                    cmd.append(arg % replace_dict)
 
-                cmd.extend(self.query_options(suites[suite],
-                                              try_options))
-                cmd.extend(self.query_tests_args(try_tests))
+                cmd = self.append_harness_extra_args(cmd)
 
                 tests = self.config["suite_definitions"][suite_category].get("tests", [])
                 cmd += tests
 
                 tbpl_status, log_level = None, None
                 error_list = BaseErrorList + [{
                     'regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"),
                     'level': ERROR,
--- a/testing/mozharness/scripts/b2g_desktop_unittest.py
+++ b/testing/mozharness/scripts/b2g_desktop_unittest.py
@@ -134,17 +134,16 @@ class B2GDesktopTest(BlobUploadMixin, Te
         dirs = self.query_abs_dirs()
         cmd = [self.query_python_path('python')]
         cmd.append(self.config["run_file_names"][suite])
 
         raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
                                     '%s_raw.log' % suite)
         error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
                                           '%s_errorsummary.log' % suite)
-
         str_format_values = {
             'application': self.binary_path,
             'test_manifest': self.test_manifest,
             'symbols_path': self.symbols_path,
             'gaia_profile': self.gaia_profile,
             'utility_path': os.path.join(dirs['abs_tests_dir'], 'bin'),
             'total_chunks': self.config.get('total_chunks'),
             'this_chunk': self.config.get('this_chunk'),
@@ -152,25 +151,26 @@ class B2GDesktopTest(BlobUploadMixin, Te
             'browser_arg': self.config.get('browser_arg'),
             'raw_log_file': raw_log_file,
             'error_summary_file': error_summary_file,
         }
 
         if suite not in self.config["suite_definitions"]:
             self.fatal("'%s' not defined in the config!" % suite)
 
-        try_options, try_tests = self.try_args(suite_name)
+        options = self.config["suite_definitions"][suite]["options"]
+        if options:
+            for option in options:
+                option = option % str_format_values
+                if not option.endswith('None'):
+                    cmd.append(option)
 
-        cmd.extend(self.query_tests_args(self.config["suite_definitions"][suite]["options"],
-                                         try_options,
-                                         str_format_values=str_format_values))
-
-        cmd.extend(self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
-                                         try_tests,
-                                         str_format_values=str_format_values)
+        tests = self.config["suite_definitions"][suite].get("tests")
+        if tests:
+            cmd.extend(tests)
 
         return cmd
 
     def download_and_extract(self):
         suite_category = self.config['test_suite']
         super(B2GDesktopTest, self).download_and_extract(suite_categories=[suite_category])
 
     def preflight_run_tests(self):
@@ -195,16 +195,17 @@ class B2GDesktopTest(BlobUploadMixin, Te
         error_list = self.error_list
         error_list.extend(BaseErrorList)
 
         suite = self.config['test_suite']
         if suite not in self.test_suites:
             self.fatal("Don't know how to run --test-suite '%s'!" % suite)
 
         cmd = self._query_abs_base_cmd(suite)
+        cmd = self.append_harness_extra_args(cmd)
 
         cwd = dirs['abs_%s_dir' % suite]
 
         # TODO we probably have to move some of the code in
         # scripts/desktop_unittest.py and scripts/marionette.py to
         # mozharness.mozilla.testing.unittest so we can share it.
         # In the short term, I'm ok with some duplication of code if it
         # expedites things; please file bugs to merge if that happens.
--- a/testing/mozharness/scripts/b2g_emulator_unittest.py
+++ b/testing/mozharness/scripts/b2g_emulator_unittest.py
@@ -268,29 +268,28 @@ class B2GEmulatorTest(TestingMixin, VCSM
             'this_chunk': self.config.get('this_chunk'),
             'test_path': self.config.get('test_path'),
             'certificate_path': dirs['abs_certs_dir'],
             'raw_log_file': raw_log_file,
             'error_summary_file': error_summary_file,
         }
 
         if suite not in self.config["suite_definitions"]:
-            self.fatal("'%s' not defined in the config!" % suite)
-
-        try_options, try_tests = self.try_args(suite)
+            self.fatal("Key '%s' not defined in the config!" % suite)
 
-        options = self.query_options(self.config["suite_definitions"][suite]["options"],
-                                     try_options,
-                                     str_format_values=str_format_values)
-        cmd.extend(opt for opt in options if not opt.endswith('None'))
+        options = self.config["suite_definitions"][suite]["options"]
+        if options:
+            for option in options:
+                option = option % str_format_values
+                if not option.endswith('None'):
+                    cmd.append(option)
 
-        tests = self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
-                                      try_tests,
-                                      str_format_values=str_format_values)
-        cmd.extend(opt for opt in tests if not opt.endswith('None'))
+        tests = self.config["suite_definitions"][suite].get("tests", [])
+        if tests:
+            cmd.extend(tests)
 
         return cmd
 
     def _query_adb(self):
         return self.which('adb') or \
             os.getenv('ADB_PATH') or \
             os.path.join(self.query_abs_dirs()['abs_b2g-distro_dir'],
                          'out', 'host', 'linux-x86', 'bin', 'adb')
@@ -337,16 +336,18 @@ class B2GEmulatorTest(TestingMixin, VCSM
         error_list = self.error_list
         error_list.extend(BaseErrorList)
 
         suite = self.config['test_suite']
         if suite not in self.test_suites:
             self.fatal("Don't know how to run --test-suite '%s'!" % suite)
 
         cmd = self._query_abs_base_cmd(suite)
+        cmd = self.append_harness_extra_args(cmd)
+
         cwd = dirs['abs_%s_dir' % suite]
 
         # TODO we probably have to move some of the code in
         # scripts/desktop_unittest.py and scripts/marionette.py to
         # mozharness.mozilla.testing.unittest so we can share it.
         # In the short term, I'm ok with some duplication of code if it
         # expedites things; please file bugs to merge if that happens.
 
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -102,18 +102,17 @@ class DesktopSingleLocale(LocalesMixin, 
          "dest": "config_files",
          "type": "string",
          "help": "Specify the platform configuration file"}
     ], [
         ['--locale', ],
         {"action": "extend",
          "dest": "locales",
          "type": "string",
-         "help": "Specify the locale(s) to sign and update. Optionally pass"
-                 " revision separated by colon, en-GB:default."}
+         "help": "Specify the locale(s) to sign and update"}
     ], [
         ['--locales-file', ],
         {"action": "store",
          "dest": "locales_file",
          "type": "string",
          "help": "Specify a file to determine which locales to sign and update"}
     ], [
         ['--tag-override', ],
@@ -329,23 +328,16 @@ class DesktopSingleLocale(LocalesMixin, 
             return self.bootstrap_env
         config = self.config
         replace_dict = self.query_abs_dirs()
         bootstrap_env = self.query_env(partial_env=config.get("bootstrap_env"),
                                        replace_dict=replace_dict)
         if config.get('en_us_binary_url') and \
            config.get('release_config_file'):
             bootstrap_env['EN_US_BINARY_URL'] = config['en_us_binary_url']
-        # Override en_us_binary_url if passed as a buildbot property
-        self.read_buildbot_config()
-        if self.buildbot_config.get("en_us_binary_url"):
-            self.info("Overriding en_us_binary_url with %s" %
-                      self.buildbot_config["en_us_binary_url"])
-            bootstrap_env['EN_US_BINARY_URL'] = \
-                self.buildbot_config["en_us_binary_url"]
         if 'MOZ_SIGNING_SERVERS' in os.environ:
             sign_cmd = self.query_moz_sign_cmd(formats=None)
             sign_cmd = subprocess.list2cmdline(sign_cmd)
             # windows fix
             bootstrap_env['MOZ_SIGN_CMD'] = sign_cmd.replace('\\', '\\\\\\\\')
         for binary in self._mar_binaries():
             # "mar -> MAR" and 'mar.exe -> MAR' (windows)
             name = binary.replace('.exe', '')
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -29,16 +29,17 @@ from mozharness.mozilla.mozbase import M
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 
 SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill', 'webapprt']
 
+
 # DesktopUnittest {{{1
 class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin, CodeCoverageMixin):
     config_options = [
         [['--mochitest-suite', ], {
             "action": "extend",
             "dest": "specified_mochitest_suites",
             "type": "string",
             "help": "Specify which mochi suite to run. "
@@ -293,52 +294,76 @@ class DesktopUnittest(TestingMixin, Merc
                 self.fatal("self.installer_url was found but symbols_url could \
                         not be determined")
         else:
             self.fatal("self.installer_url was not found in self.config")
         self.info("setting symbols_url as %s" % (symbols_url))
         self.symbols_url = symbols_url
         return self.symbols_url
 
+    def get_webapprt_path(self, res_dir, mochitest_dir):
+        """Get the path to the webapp runtime binary.
+        On Mac, we copy the stub from the resources dir to the test app bundle,
+        since we have to run it from the executable directory of a bundle
+        in order for its windows to appear.  Ideally, the build system would do
+        this for us at build time, and we should find a way for it to do that.
+        """
+        exe_suffix = self.config.get('exe_suffix', '')
+        app_name = 'webapprt-stub' + exe_suffix
+        app_path = os.path.join(res_dir, app_name)
+        if self._is_darwin():
+            mac_dir_name = os.path.join(
+                mochitest_dir,
+                'webapprtChrome',
+                'webapprt',
+                'test',
+                'chrome',
+                'TestApp.app',
+                'Contents',
+                'MacOS')
+            mac_app_name = 'webapprt' + exe_suffix
+            mac_app_path = os.path.join(mac_dir_name, mac_app_name)
+            self.copyfile(app_path, mac_app_path, copystat=True)
+            return mac_app_path
+        return app_path
+
     def _query_abs_base_cmd(self, suite_category, suite):
         if self.binary_path:
             c = self.config
             dirs = self.query_abs_dirs()
             run_file = c['run_file_names'][suite_category]
             base_cmd = [self.query_python_path('python'), '-u']
             base_cmd.append(os.path.join(dirs["abs_%s_dir" % suite_category], run_file))
             abs_app_dir = self.query_abs_app_dir()
             abs_res_dir = self.query_abs_res_dir()
 
-            webapprt_path = os.path.join(os.path.dirname(self.binary_path),
-                                         'webapprt-stub')
-            if c.get('exe_suffix'):
-                webapprt_path += c['exe_suffix']
-
             raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
                                         '%s_raw.log' % suite)
 
             error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
                                               '%s_errorsummary.log' % suite)
             str_format_values = {
                 'binary_path': self.binary_path,
                 'symbols_path': self._query_symbols_url(),
                 'abs_app_dir': abs_app_dir,
-                'app_path': webapprt_path,
+                'abs_res_dir': abs_res_dir,
                 'raw_log_file': raw_log_file,
                 'error_summary_file': error_summary_file,
                 'gtest_dir': os.path.join(dirs['abs_test_install_dir'],
                                           'gtest'),
             }
 
             # TestingMixin._download_and_extract_symbols() will set
             # self.symbols_path when downloading/extracting.
             if self.symbols_path:
                 str_format_values['symbols_path'] = self.symbols_path
 
+            if suite_category == 'webapprt':
+                str_format_values['app_path'] = self.get_webapprt_path(abs_res_dir, dirs['abs_mochitest_dir'])
+
             if c['e10s']:
                 base_cmd.append('--e10s')
 
             if c.get('strict_content_sandbox'):
                 if suite_category == "mochitest":
                     base_cmd.append('--strict-content-sandbox')
                 else:
                     self.fatal("--strict-content-sandbox only works with mochitest suites.")
@@ -368,24 +393,17 @@ class DesktopUnittest(TestingMixin, Merc
                         base_cmd.append(option)
                 return base_cmd
             else:
                 self.warning("Suite options for %s could not be determined."
                              "\nIf you meant to have options for this suite, "
                              "please make sure they are specified in your "
                              "config under %s_options" %
                              (suite_category, suite_category))
-
-
-            for option in options:
-                option = option % str_format_values
-                if not option.endswith('None'):
-                    base_cmd.append(option)
-
-            return base_cmd
+                return base_cmd
         else:
             self.fatal("'binary_path' could not be determined.\n This should "
                        "be like '/path/build/application/firefox/firefox'"
                        "\nIf you are running this script without the 'install' "
                        "action (where binary_path is set), please ensure you are"
                        " either:\n(1) specifying it in the config file under "
                        "binary_path\n(2) specifying it on command line with the"
                        " '--binary-path' flag")
@@ -411,30 +429,16 @@ class DesktopUnittest(TestingMixin, Merc
                 suites = dict((key, all_suites.get(key)) for key in
                               specified_suites if key in all_suites.keys())
         else:
             if c.get('run_all_suites'):  # needed if you dont specify any suites
                 suites = all_suites
 
         return suites
 
-    def _query_try_flavor(self, category, suite):
-        flavors = {
-            "mochitest": [("plain.*", "mochitest"),
-                          ("browser-chrome.*", "browser-chrome"),
-                          ("mochitest-devtools-chrome.*", "devtools-chrome"),
-                          ("chrome", "chrome")],
-            "xpcshell": [("xpcshell", "xpcshell")],
-            "reftest": [("reftest", "reftest"),
-                        ("crashtest", "crashtest")]
-        }
-        for suite_pattern, flavor in flavors.get(category, []):
-            if re.compile(suite_pattern).match(suite):
-                return flavor
-
     # Actions {{{2
 
     # clobber defined in BaseScript, deletes mozharness/build if exists
     # read_buildbot_config is in BuildbotMixin.
     # postflight_read_buildbot_config is in TestingMixin.
     # preflight_download_and_extract is in TestingMixin.
     # create_virtualenv is in VirtualenvMixin.
     # preflight_install is in TestingMixin.
@@ -579,32 +583,25 @@ class DesktopUnittest(TestingMixin, Merc
 
                     # Mac specific, but points to abs_app_dir on other
                     # platforms.
                     'abs_res_dir': abs_res_dir,
                 }
                 options_list = []
                 env = {}
                 if isinstance(suites[suite], dict):
-                    options_list = suites[suite].get('options', [])
-                    tests_list = suites[suite].get('tests', [])
+                    options_list = suites[suite]['options'] + suites[suite].get("tests", [])
                     env = copy.deepcopy(suites[suite].get('env', {}))
                 else:
                     options_list = suites[suite]
-                    tests_list = []
-
-                flavor = self._query_try_flavor(suite_category, suite)
-                try_options, try_tests = self.try_args(flavor)
 
-                cmd.extend(self.query_options(options_list,
-                                              try_options,
-                                              str_format_values=replace_dict))
-                cmd.extend(self.query_tests_args(tests_list,
-                                                 try_tests,
-                                                 str_format_values=replace_dict))
+                for arg in options_list:
+                    cmd.append(arg % replace_dict)
+
+                cmd = self.append_harness_extra_args(cmd)
 
                 suite_name = suite_category + '-' + suite
                 tbpl_status, log_level = None, None
                 error_list = BaseErrorList + [{
                     'regex': re.compile(r'''PROCESS-CRASH.*application crashed'''),
                     'level': ERROR,
                 }]
                 parser = self.get_test_output_parser(suite_category,
--- a/testing/mozharness/scripts/marionette.py
+++ b/testing/mozharness/scripts/marionette.py
@@ -434,20 +434,17 @@ class MarionetteTest(TestingMixin, Mercu
         for s in self.config["suite_definitions"][options_group]["options"]:
             cmd.append(s % config_fmt_args)
 
         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
             # Make sure that the logging directory exists
             self.fatal("Could not create blobber upload directory")
 
         cmd.append(manifest)
-
-        try_options, try_tests = self.try_args("marionette")
-        cmd.extend(self.query_tests_args(try_tests,
-                                         str_format_values=config_fmt_args))
+        cmd = self.append_harness_extra_args(cmd)
 
         env = {}
         if self.query_minidump_stackwalk():
             env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
         if self.config.get('gaiatest'):
             env['GAIATEST_ACKNOWLEDGED_RISKS'] = '1'
             env['GAIATEST_SKIP_WARNING'] = '1'
         env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -110,74 +110,69 @@ class WebPlatformTest(TestingMixin, Merc
             self.fatal("Binary path could not be determined")
             #And exit
 
         c = self.config
         dirs = self.query_abs_dirs()
         abs_app_dir = self.query_abs_app_dir()
         run_file_name = "runtests.py"
 
-        cmd = [self.query_python_path('python'), '-u']
-        cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
+        base_cmd = [self.query_python_path('python'), '-u']
+        base_cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
 
         # Make sure that the logging directory exists
         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
             self.fatal("Could not create blobber upload directory")
             # Exit
 
-        cmd += ["--log-raw=-",
-                "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
-                                              "wpt_raw.log"),
-                "--binary=%s" % self.binary_path,
-                "--symbols-path=%s" % self.query_symbols_url(),
-                "--stackwalk-binary=%s" % self.query_minidump_stackwalk()]
+        base_cmd += ["--log-raw=-",
+                     "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
+                                                   "wpt_raw.log"),
+                     "--binary=%s" % self.binary_path,
+                     "--symbols-path=%s" % self.query_symbols_url(),
+                     "--stackwalk-binary=%s" % self.query_minidump_stackwalk()]
 
         for test_type in c.get("test_type", []):
-            cmd.append("--test-type=%s" % test_type)
+            base_cmd.append("--test-type=%s" % test_type)
 
         if c.get("e10s"):
-            cmd.append("--e10s")
+            base_cmd.append("--e10s")
 
         for opt in ["total_chunks", "this_chunk"]:
             val = c.get(opt)
             if val:
-                cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
+                base_cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
 
         options = list(c.get("options", []))
 
         str_format_values = {
             'binary_path': self.binary_path,
             'test_path': dirs["abs_wpttest_dir"],
             'test_install_path': dirs["abs_test_install_dir"],
             'abs_app_dir': abs_app_dir,
             'abs_work_dir': dirs["abs_work_dir"]
             }
 
-        try_options, try_tests = self.try_args("web-platform-tests")
+        opt_cmd = [item % str_format_values for item in options]
 
-        cmd.extend(self.query_options(options,
-                                      try_options,
-                                      str_format_values=str_format_values))
-        cmd.extend(self.query_tests_args(try_tests,
-                                         str_format_values=str_format_values))
-
-        return cmd
+        return base_cmd + opt_cmd
 
     def download_and_extract(self):
         super(WebPlatformTest, self).download_and_extract(
             target_unzip_dirs=["bin/*",
                                "config/*",
                                "mozbase/*",
                                "marionette/*",
                                "web-platform/*"],
             suite_categories=["web-platform"])
 
     def run_tests(self):
         dirs = self.query_abs_dirs()
         cmd = self._query_cmd()
+        cmd = self.append_harness_extra_args(cmd)
 
         parser = StructuredOutputParser(config=self.config,
                                         log_obj=self.log_obj)
 
         env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}
         env = self.query_env(partial_env=env, log_level=INFO)
 
         return_code = self.run_command(cmd,
--- a/testing/tools/autotry/autotry.py
+++ b/testing/tools/autotry/autotry.py
@@ -1,290 +1,98 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import argparse
-import itertools
+import sys
 import os
-import re
+import itertools
 import subprocess
-import sys
 import which
 
 from collections import defaultdict
 
-import ConfigParser
 
-
-def arg_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.')
-    parser.add_argument('-b', dest='builds', default='do',
-                        help='Build types to run (d for debug, o for optimized).')
-    parser.add_argument('-p', dest='platforms', action="append",
-                        help='Platforms to run (required if not found in the environment).')
-    parser.add_argument('-u', dest='tests', action="append",
-                        help='Test suites to run in their entirety.')
-    parser.add_argument('-t', dest="talos", action="append",
-                        help='Talos suites to run.')
-    parser.add_argument('--tag', dest='tags', action='append',
-                        help='Restrict tests to the given tag (may be specified multiple times).')
-    parser.add_argument('--and', action='store_true', dest="intersection",
-                        help='When -u and paths are supplied run only the intersection of the tests specified by the two arguments.')
-    parser.add_argument('--no-push', dest='push', action='store_false',
-                        help='Do not push to try as a result of running this command (if '
-                        'specified this command will only print calculated try '
-                        'syntax and selection info).')
-    parser.add_argument('--save', dest="save", action='store',
-                        help="Save the command line arguments for future use with --preset.")
-    parser.add_argument('--preset', dest="load", action='store',
-                        help="Load a saved set of arguments. Additional arguments will override saved ones.")
-    parser.add_argument('extra_args', nargs=argparse.REMAINDER,
-                        help='Extra arguments to put in the try push.')
-    parser.add_argument('-v', "--verbose", dest='verbose', action='store_true', default=False,
-                        help='Print detailed information about the resulting test selection '
-                        'and commands performed.')
-    return parser
-
-class TryArgumentTokenizer(object):
-    symbols = [("seperator", ","),
-               ("list_start", "\["),
-               ("list_end", "\]"),
-               ("item", "([^,\[\]\s][^,\[\]]+)"),
-               ("space", "\s+")]
-    token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
-
-    def tokenize(self, data):
-        for match in self.token_re.finditer(data):
-            symbol = match.lastgroup
-            data = match.group(symbol)
-            if symbol == "space":
-                pass
-            else:
-                yield symbol, data
-
-class TryArgumentParser(object):
-    """Simple three-state parser for handling expressions
-    of the from "foo[sub item, another], bar,baz". This takes
-    input from the TryArgumentTokenizer and runs through a small
-    state machine, returning a dictionary of {top-level-item:[sub_items]}
-    i.e. the above would result in
-    {"foo":["sub item", "another"], "bar": [], "baz": []}
-    In the case of invalid input a ValueError is raised."""
-
-    EOF = object()
-
-    def __init__(self):
-        self.reset()
-
-    def reset(self):
-        self.tokens = None
-        self.current_item = None
-        self.data = {}
-        self.token = None
-        self.state = None
-
-    def parse(self, tokens):
-        self.reset()
-        self.tokens = tokens
-        self.consume()
-        self.state = self.item_state
-        while self.token[0] != self.EOF:
-            self.state()
-        return self.data
-
-    def consume(self):
-        try:
-            self.token = self.tokens.next()
-        except StopIteration:
-            self.token = (self.EOF, None)
-
-    def expect(self, *types):
-        if self.token[0] not in types:
-            raise ValueError("Error parsing try string, unexpected %s" % (self.token[0]))
-
-    def item_state(self):
-        self.expect("item")
-        value = self.token[1].strip()
-        if value not in self.data:
-            self.data[value] = []
-        self.current_item = value
-        self.consume()
-        if self.token[0] == "seperator":
-            self.consume()
-        elif self.token[0] == "list_start":
-            self.consume()
-            self.state = self.subitem_state
-        elif self.token[0] == self.EOF:
-            pass
-        else:
-            raise ValueError
-
-    def subitem_state(self):
-        self.expect("item")
-        value = self.token[1].strip()
-        self.data[self.current_item].append(value)
-        self.consume()
-        if self.token[0] == "seperator":
-            self.consume()
-        elif self.token[0] == "list_end":
-            self.consume()
-            self.state = self.after_list_end_state
-        else:
-            raise ValueError
-
-    def after_list_end_state(self):
-        self.expect("seperator")
-        self.consume()
-        self.state = self.item_state
-
-def parse_arg(arg):
-    tokenizer = TryArgumentTokenizer()
-    parser = TryArgumentParser()
-    return parser.parse(tokenizer.tokenize(arg))
+TRY_SYNTAX_TMPL = """
+try: -b %s -p %s -u %s -t none %s %s
+"""
 
 class AutoTry(object):
 
-    # Maps from flavors to the job names needed to run that flavour
-    flavor_jobs = {
-        'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
-        'xpcshell': ['xpcshell'],
-        'chrome': ['mochitest-o'],
-        'browser-chrome': ['mochitest-browser-chrome-1',
-                           'mochitest-e10s-browser-chrome-1'],
-        'devtools-chrome': ['mochitest-dt',
-                            'mochitest-e10s-devtools-chrome'],
-        'crashtest': ['crashtest', 'crashtest-e10s'],
-        'reftest': ['reftest', 'reftest-e10s'],
-        'web-platform-tests': ['web-platform-tests-1'],
-    }
-
-    flavor_suites = {
-        "mochitest": "mochitests",
-        "xpcshell": "xpcshell",
-        "chrome": "mochitest-o",
-        "browser-chrome": "mochitest-bc",
-        "devtools-chrome": "mochitest-dt",
-        "crashtest": "crashtest",
-        "reftest": "reftest",
-        "web-platform-tests": "web-platform-tests",
-    }
+    test_flavors = [
+        'browser-chrome',
+        'chrome',
+        'devtools-chrome',
+        'mochitest',
+        'xpcshell',
+        'reftest',
+        'crashtest',
+    ]
 
     def __init__(self, topsrcdir, resolver, mach_context):
         self.topsrcdir = topsrcdir
         self.resolver = resolver
         self.mach_context = mach_context
 
         if os.path.exists(os.path.join(self.topsrcdir, '.hg')):
             self._use_git = False
         else:
             self._use_git = True
 
-    @property
-    def config_path(self):
-        return os.path.join(self.mach_context.state_dir, "autotry.ini")
-
-    def load_config(self, name):
-        config = ConfigParser.RawConfigParser()
-        success = config.read([self.config_path])
-        if not success:
-            return None
-
-        try:
-            data = config.get("try", name)
-        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
-            return None
-
-        kwargs = vars(arg_parser().parse_args(data.split()))
-
-        return kwargs
+    def resolve_manifests(self, paths=None, tags=None):
+        if not (paths or tags):
+            return {}
 
-    def save_config(self, name, data):
-        assert data.startswith("try: ")
-        data = data[len("try: "):]
-
-        parser = ConfigParser.RawConfigParser()
-        parser.read([self.config_path])
-
-        if not parser.has_section("try"):
-            parser.add_section("try")
-
-        parser.set("try", name, data)
-
-        with open(self.config_path, "w") as f:
-            parser.write(f)
-
-    def paths_by_flavor(self, paths=None, tags=None):
-        paths_by_flavor = defaultdict(set)
-
-        if not (paths or tags):
-            return dict(paths_by_flavor)
-
-        tests = list(self.resolver.resolve_tests(paths=paths,
-                                                 tags=tags))
+        tests = list(self.resolver.resolve_tests(tags=tags,
+                                                 paths=paths,
+                                                 cwd=self.mach_context.cwd))
+        manifests_by_flavor = defaultdict(set)
 
         for t in tests:
-            if t['flavor'] in self.flavor_suites:
+            if t['flavor'] in AutoTry.test_flavors:
                 flavor = t['flavor']
                 if 'subsuite' in t and t['subsuite'] == 'devtools':
                     flavor = 'devtools-chrome'
+                manifest = os.path.relpath(t['manifest'], self.topsrcdir)
+                manifests_by_flavor[flavor].add(manifest)
 
-                for path in paths:
-                    if flavor in ["crashtest", "reftest"]:
-                        manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
-                        if manifest_relpath.startswith(path):
-                            paths_by_flavor[flavor].add(manifest_relpath)
-                    else:
-                        if t['file_relpath'].startswith(path):
-                            paths_by_flavor[flavor].add(path)
+        return dict(manifests_by_flavor)
 
-        return dict(paths_by_flavor)
-
-    def remove_duplicates(self, paths_by_flavor, tests):
-        rv = {}
-        for item in paths_by_flavor:
-            if self.flavor_suites[item] not in tests:
-                rv[item] = paths_by_flavor[item].copy()
-        return rv
+    def calc_try_syntax(self, platforms, flavors, tests, extra_tests, builds,
+                        manifests, tags):
 
-    def calc_try_syntax(self, platforms, tests, talos, builds, paths_by_flavor, tags,
-                        extra_args, intersection):
-        parts = ["try:", "-b", builds, "-p", ",".join(platforms)]
-
-        suites = tests if not intersection else {}
-        paths = set()
-        for flavor, flavor_tests in paths_by_flavor.iteritems():
-            suite = self.flavor_suites[flavor]
-            if suite not in suites and (not intersection or suite in tests):
-                for job_name in self.flavor_jobs[flavor]:
-                    for test in flavor_tests:
-                        paths.add("%s:%s" % (flavor, test))
-                    suites[job_name] = tests.get(suite, [])
-
-        if not suites:
-            raise ValueError("No tests found matching filters")
-
-        parts.append("-u")
-        parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
-                              for k,v in sorted(suites.items())) if suites else "none")
-
-        parts.append("-t")
-        parts.append(",".join(talos) if talos else "none")
+        # Maps from flavors to the try syntax snippets implied by that flavor.
+        # TODO: put selected tests under their own builder/label to avoid
+        # confusion reading results on treeherder.
+        flavor_suites = {
+            'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
+            'xpcshell': ['xpcshell'],
+            'chrome': ['mochitest-o'],
+            'browser-chrome': ['mochitest-browser-chrome-1',
+                               'mochitest-e10s-browser-chrome-1'],
+            'devtools-chrome': ['mochitest-dt',
+                                'mochitest-e10s-devtools-chrome'],
+            'crashtest': ['crashtest', 'crashtest-e10s'],
+            'reftest': ['reftest', 'reftest-e10s'],
+        }
 
         if tags:
-            parts.append(' '.join('--tag %s' % t for t in tags))
-
-        if extra_args is not None:
-            parts.extend(extra_args)
+            tags = ' '.join('--tag %s' % t for t in tags)
+        else:
+            tags = ''
 
-        if paths:
-            parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
+        if not tests:
+            tests = ','.join(itertools.chain(*(flavor_suites[f] for f in flavors)))
+            if extra_tests:
+                tests += ',%s' % (extra_tests)
 
-        return " ".join(parts)
+        manifests = ' '.join(manifests)
+        if manifests:
+            manifests = '--try-test-paths %s' % manifests
+        return TRY_SYNTAX_TMPL % (builds, platforms, tests, manifests, tags)
 
     def _run_git(self, *args):
         args = ['git'] + list(args)
         ret = subprocess.call(args)
         if ret:
             print('ERROR git command %s returned %s' %
                   (args, ret))
             sys.exit(1)