Bug 1293295 - Replace all mochitest 'flavor' options with a single --flavor argument, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 08 Aug 2016 11:48:49 -0400
changeset 334505 43ebf330ad942617aa8526294493a3dd25c0fc8d
parent 334504 a7a5fd1b36e553163533f4ff112354be2b310a95
child 334506 ff904be8285df7c54096349d0965b0895c43287a
push id10033
push userraliiev@mozilla.com
push dateMon, 19 Sep 2016 13:50:26 +0000
treeherdermozilla-aurora@5dddbefdf759 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1293295, 1293259
milestone51.0a1
Bug 1293295 - Replace all mochitest 'flavor' options with a single --flavor argument, r=jmaher This accomplishes three things: 1) Easier to use CLI when running without the benefit of testing/mochitest/mach_commands.py 2) Guarantees these arguments are mutually exclusive 3) Simplifies a bunch of logic in the test harness The primary motivation for this change is to slightly improve the UX when running mochitest from a taskcluster interactive loaner. However, this is more of a bandaid solution that was easy to implement before the proper fix in bug 1293259 can be landed. MozReview-Commit-ID: IeHBGrJ0Sji
b2g/build.mk
browser/build.mk
mobile/android/build.mk
testing/mochitest/mach_commands.py
testing/mochitest/mochitest_options.py
testing/mochitest/redirect.html
testing/mochitest/runtests.py
testing/mochitest/runtestsb2g.py
testing/mochitest/runtestsremote.py
testing/mozharness/configs/android/androidarm.py
testing/mozharness/configs/android/androidarm_4_3.py
testing/mozharness/configs/b2g/emulator_automation_config.py
testing/mozharness/configs/b2g/taskcluster_emulator_automation.py
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/configs/unittests/mac_unittest.py
testing/mozharness/configs/unittests/win_unittest.py
--- a/b2g/build.mk
+++ b/b2g/build.mk
@@ -16,16 +16,16 @@ install::
 
 upload::
 	@$(MAKE) -C b2g/installer upload
 
 ifdef ENABLE_TESTS
 # Implemented in testing/testsuite-targets.mk
 
 mochitest-browser-chrome:
-	$(RUN_MOCHITEST) --browser-chrome
+	$(RUN_MOCHITEST) --flavor=browser
 	$(CHECK_TEST_ERROR)
 
 mochitest:: mochitest-browser-chrome
 
 .PHONY: mochitest-browser-chrome
 endif
 
--- a/browser/build.mk
+++ b/browser/build.mk
@@ -40,16 +40,16 @@ hg-bundle::
 
 l10n-check::
 	@$(MAKE) -C browser/locales l10n-check
 
 ifdef ENABLE_TESTS
 # Implemented in testing/testsuite-targets.mk
 
 mochitest-browser-chrome:
-	$(RUN_MOCHITEST) --browser-chrome
+	$(RUN_MOCHITEST) --flavor=browser
 	$(CHECK_TEST_ERROR)
 
 mochitest:: mochitest-browser-chrome
 
 .PHONY: mochitest-browser-chrome
 
 endif
--- a/mobile/android/build.mk
+++ b/mobile/android/build.mk
@@ -48,17 +48,17 @@ deb: package
 
 upload::
 	@$(MAKE) -C mobile/android/installer upload
 
 ifdef ENABLE_TESTS
 # Implemented in testing/testsuite-targets.mk
 
 mochitest-browser-chrome:
-	$(RUN_MOCHITEST) --browser-chrome
+	$(RUN_MOCHITEST) --flavor=browser
 	$(CHECK_TEST_ERROR)
 
 mochitest:: mochitest-browser-chrome
 
 .PHONY: mochitest-browser-chrome
 endif
 
 ifeq ($(OS_TARGET),Linux)
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -85,49 +85,49 @@ ALL_FLAVORS = {
         'aliases': ('plain', 'mochitest'),
         'enabled_apps': ('firefox', 'b2g', 'android', 'mulet'),
     },
     'chrome': {
         'suite': 'chrome',
         'aliases': ('chrome', 'mochitest-chrome'),
         'enabled_apps': ('firefox', 'mulet', 'b2g', 'android'),
         'extra_args': {
-            'chrome': True,
+            'flavor': 'chrome',
         }
     },
     'browser-chrome': {
         'suite': 'browser',
         'aliases': ('browser', 'browser-chrome', 'mochitest-browser-chrome', 'bc'),
         'enabled_apps': ('firefox',),
         'extra_args': {
-            'browserChrome': True,
+            'flavor': 'browser',
         }
     },
     'jetpack-package': {
         'suite': 'jetpack-package',
         'aliases': ('jetpack-package', 'mochitest-jetpack-package', 'jpp'),
         'enabled_apps': ('firefox',),
         'extra_args': {
-            'jetpackPackage': True,
+            'flavor': 'jetpack-package',
         }
     },
     'jetpack-addon': {
         'suite': 'jetpack-addon',
         'aliases': ('jetpack-addon', 'mochitest-jetpack-addon', 'jpa'),
         'enabled_apps': ('firefox',),
         'extra_args': {
-            'jetpackAddon': True,
+            'flavor': 'jetpack-addon',
         }
     },
     'a11y': {
         'suite': 'a11y',
         'aliases': ('a11y', 'mochitest-a11y', 'accessibility'),
         'enabled_apps': ('firefox',),
         'extra_args': {
-            'a11y': True,
+            'flavor': 'a11y',
         }
     },
 }
 
 SUPPORTED_APPS = ['firefox', 'b2g', 'android', 'mulet']
 SUPPORTED_FLAVORS = list(chain.from_iterable([f['aliases'] for f in ALL_FLAVORS.values()]))
 CANONICAL_FLAVORS = sorted([f['aliases'][0] for f in ALL_FLAVORS.values()])
 
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -81,27 +81,33 @@ class ArgumentContainer():
     def get_full_path(self, path, cwd):
         """Get an absolute path relative to cwd."""
         return os.path.normpath(os.path.join(cwd, os.path.expanduser(path)))
 
 
 class MochitestArguments(ArgumentContainer):
     """General mochitest arguments."""
 
+    FLAVORS = ('a11y', 'browser', 'chrome', 'jetpack-addon', 'jetpack-package', 'plain')
     LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL")
-    LEVEL_STRING = ", ".join(LOG_LEVELS)
 
     args = [
         [["test_paths"],
          {"nargs": "*",
           "metavar": "TEST",
           "default": [],
           "help": "Test to run. Can be a single test file or a directory of tests "
                   "(to run recursively). If omitted, the entire suite is run.",
           }],
+        [["-f", "--flavor"],
+         {"default": "plain",
+          "choices": FLAVORS,
+          "help": "Mochitest flavor to run, one of {}. Defaults to 'plain'.".format(FLAVORS),
+          "suppress": build_obj is not None,
+          }],
         [["--keep-open"],
          {"nargs": "?",
           "type": strtobool,
           "const": "true",
           "default": None,
           "help": "Always keep the browser open after tests complete. Or always close the "
                   "browser with --keep-open=false",
           }],
@@ -177,23 +183,18 @@ class MochitestArguments(ArgumentContain
          {"action": "store_true",
           "help": "Shuffle execution order of tests.",
           "default": False,
           }],
         [["--console-level"],
          {"dest": "consoleLevel",
           "choices": LOG_LEVELS,
           "default": "INFO",
-          "help": "One of %s to determine the level of console logging." % LEVEL_STRING,
-          "suppress": True,
-          }],
-        [["--chrome"],
-         {"action": "store_true",
-          "default": False,
-          "help": "Run chrome mochitests.",
+          "help": "One of {} to determine the level of console logging.".format(
+                  ', '.join(LOG_LEVELS)),
           "suppress": True,
           }],
         [["--bisect-chunk"],
          {"dest": "bisectChunk",
           "default": None,
           "help": "Specify the failing test name to find the previous tests that may be "
                   "causing the failure.",
           }],
@@ -202,48 +203,21 @@ class MochitestArguments(ArgumentContain
           "default": "",
           "help": "Start running the test sequence at this test.",
           }],
         [["--end-at"],
          {"dest": "endAt",
           "default": "",
           "help": "Stop running the test sequence at this test.",
           }],
-        [["--browser-chrome"],
-         {"action": "store_true",
-          "dest": "browserChrome",
-          "default": False,
-          "help": "run browser chrome Mochitests",
-          "suppress": True,
-          }],
         [["--subsuite"],
          {"default": None,
           "help": "Subsuite of tests to run. Unlike tags, subsuites also remove tests from "
                   "the default set. Only one can be specified at once.",
           }],
-        [["--jetpack-package"],
-         {"action": "store_true",
-          "dest": "jetpackPackage",
-          "help": "Run jetpack package tests.",
-          "default": False,
-          "suppress": True,
-          }],
-        [["--jetpack-addon"],
-         {"action": "store_true",
-          "dest": "jetpackAddon",
-          "help": "Run jetpack addon tests.",
-          "default": False,
-          "suppress": True,
-          }],
-        [["--a11y"],
-         {"action": "store_true",
-          "help": "Run accessibility Mochitests.",
-          "default": False,
-          "suppress": True,
-          }],
         [["--setenv"],
          {"action": "append",
           "dest": "environment",
           "metavar": "NAME=VALUE",
           "default": [],
           "help": "Sets the given variable in the application's environment.",
           }],
         [["--exclude-extension"],
--- a/testing/mochitest/redirect.html
+++ b/testing/mochitest/redirect.html
@@ -1,17 +1,17 @@
 <html>
 <head>
   <title>redirecting...</title>
 
   <script type="text/javascript">
     function redirect(aURL)
     {
-      // We create a listener for this event in browser-test.js 
-      // which will get picked up when specifying --chrome or --a11y
+      // We create a listener for this event in browser-test.js which will
+      // get picked up when specifying --flavor=chrome or --flavor=a11y
       var event = new CustomEvent("contentEvent", {
         bubbles: true,
         detail: {
           "data": aURL + location.search,
           "type": "loadURI"
         }
       });
       document.dispatchEvent(event);
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -614,18 +614,17 @@ class MochitestBase(object):
             options.logFile = ""
         if not hasattr(options, 'fileLevel'):
             options.fileLevel = 'INFO'
 
         # allow relative paths for logFile
         if options.logFile:
             options.logFile = self.getLogFilePath(options.logFile)
 
-        if options.browserChrome or options.chrome or \
-           options.a11y or options.jetpackPackage or options.jetpackAddon:
+        if options.flavor in ('a11y', 'browser', 'chrome', 'jetpack-addon', 'jetpack-package'):
             self.makeTestConfig(options)
         else:
             if options.autorun:
                 self.urlOpts.append("autorun=1")
             if options.timeout:
                 self.urlOpts.append("timeout=%d" % options.timeout)
             if options.maxTimeouts:
                 self.urlOpts.append("maxTimeouts=%d" % options.maxTimeouts)
@@ -684,68 +683,59 @@ class MochitestBase(object):
                         options.dumpOutputDirectory))
             if options.dumpAboutMemoryAfterTest:
                 self.urlOpts.append("dumpAboutMemoryAfterTest=true")
             if options.dumpDMDAfterTest:
                 self.urlOpts.append("dumpDMDAfterTest=true")
             if options.debugger:
                 self.urlOpts.append("interactiveDebugger=true")
 
-    def getTestFlavor(self, options):
-        if options.browserChrome:
-            return "browser-chrome"
-        elif options.jetpackPackage:
-            return "jetpack-package"
-        elif options.jetpackAddon:
-            return "jetpack-addon"
-        elif options.chrome:
-            return "chrome"
-        elif options.a11y:
-            return "a11y"
-        else:
-            return "mochitest"
+    def normflavor(self, flavor):
+        """
+        In some places the string 'browser-chrome' is expected instead of
+        'browser' and 'mochitest' instead of 'plain'. Normalize the flavor
+        strings for those instances.
+        """
+        # TODO Use consistent flavor strings everywhere and remove this
+        if flavor == 'browser':
+            return 'browser-chrome'
+        elif flavor == 'plain':
+            return 'mochitest'
+        return flavor
 
     # This check can be removed when bug 983867 is fixed.
     def isTest(self, options, filename):
         allow_js_css = False
-        if options.browserChrome:
+        if options.flavor == 'browser':
             allow_js_css = True
             testPattern = re.compile(r"browser_.+\.js")
-        elif options.jetpackPackage:
+        elif options.flavor == 'jetpack-package':
             allow_js_css = True
             testPattern = re.compile(r"test-.+\.js")
-        elif options.jetpackAddon:
+        elif options.flavor == 'jetpack-addon':
             testPattern = re.compile(r".+\.xpi")
-        elif options.chrome or options.a11y:
+        elif options.flavor in ('a11y', 'chrome'):
             testPattern = re.compile(r"(browser|test)_.+\.(xul|html|js|xhtml)")
         else:
             testPattern = re.compile(r"test_")
 
         if not allow_js_css and (".js" in filename or ".css" in filename):
             return False
 
         pathPieces = filename.split("/")
 
         return (testPattern.match(pathPieces[-1]) and
                 not re.search(r'\^headers\^$', filename))
 
     def setTestRoot(self, options):
-        if options.browserChrome:
-            if options.immersiveMode:
+        if options.flavor != 'plain':
+            self.testRoot = options.flavor
+
+            if options.flavor == 'browser' and options.immersiveMode:
                 self.testRoot = 'metro'
-            else:
-                self.testRoot = 'browser'
-        elif options.jetpackPackage:
-            self.testRoot = 'jetpack-package'
-        elif options.jetpackAddon:
-            self.testRoot = 'jetpack-addon'
-        elif options.a11y:
-            self.testRoot = 'a11y'
-        elif options.chrome:
-            self.testRoot = 'chrome'
         else:
             self.testRoot = self.TEST_PATH
         self.testRootAbs = os.path.join(SCRIPT_DIR, self.testRoot)
 
     def buildTestURL(self, options):
         testHost = "http://mochi.test:8888"
         testURL = "/".join([testHost, self.TEST_PATH])
 
@@ -755,19 +745,19 @@ class MochitestBase(object):
                     self.oldcwd,
                     os.path.dirname(__file__),
                     self.TEST_PATH,
                     options.test_paths[0])):
                 testURL = "/".join([testURL, os.path.dirname(options.test_paths[0])])
             else:
                 testURL = "/".join([testURL, options.test_paths[0]])
 
-        if options.chrome or options.a11y:
+        if options.flavor in ('a11y', 'chrome'):
             testURL = "/".join([testHost, self.CHROME_PATH])
-        elif options.browserChrome or options.jetpackPackage or options.jetpackAddon:
+        elif options.flavor in ('browser', 'jetpack-addon', 'jetpack-package'):
             testURL = "about:blank"
         if options.nested_oop:
             testURL = "/".join([testHost, self.NESTED_OOP_TEST_PATH])
         return testURL
 
     def buildTestPath(self, options, testsToFilter=None, disabled=True):
         """ Build the url path to the specific test harness and test file or directory
             Build a manifest of tests to run and write out a json file for the harness to read
@@ -1061,18 +1051,17 @@ toolbar#nav-bar {
             # Add chunking filters if specified
             if options.totalChunks:
                 if options.chunkByRuntime:
                     runtime_file = self.resolve_runtime_file(options)
                     if not os.path.exists(runtime_file):
                         self.log.warning("runtime file %s not found; defaulting to chunk-by-dir" %
                                          runtime_file)
                         options.chunkByRuntime = None
-                        flavor = self.getTestFlavor(options)
-                        if flavor in ('browser-chrome', 'devtools-chrome'):
+                        if options.flavor == 'browser':
                             # these values match current mozharness configs
                             options.chunkbyDir = 5
                         else:
                             options.chunkByDir = 4
 
                 if options.chunkByDir:
                     filters.append(chunk_by_dir(options.thisChunk,
                                                 options.totalChunks,
@@ -1149,17 +1138,17 @@ toolbar#nav-bar {
         elif options.manifestFile and os.path.isfile(os.path.join(SCRIPT_DIR, options.manifestFile)):
             manifestFileAbs = os.path.abspath(
                 os.path.join(
                     SCRIPT_DIR,
                     options.manifestFile))
             assert manifestFileAbs.startswith(SCRIPT_DIR)
             manifest = TestManifest([manifestFileAbs], strict=False)
         else:
-            masterName = self.getTestFlavor(options) + '.ini'
+            masterName = self.normflavor(options.flavor) + '.ini'
             masterPath = os.path.join(SCRIPT_DIR, self.testRoot, masterName)
 
             if os.path.exists(masterPath):
                 manifest = TestManifest([masterPath], strict=False)
             else:
                 self._log.warning(
                     'TestManifest masterPath %s does not exist' %
                     masterPath)
@@ -1205,17 +1194,17 @@ toolbar#nav-bar {
         # via the commandline at your own risk.
         browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
 
         # When creating child processes on Windows pre-Vista (e.g. Windows XP) we
         # don't normally inherit stdout/err handles, because you can only do it by
         # inheriting all other inheritable handles as well.
         # We need to inherit them for plain mochitests for test logging purposes, so
         # we do so on the basis of a specific environment variable.
-        if self.getTestFlavor(options) == "mochitest":
+        if options.flavor == 'plain':
             browserEnv["MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA"] = "1"
 
         # interpolate environment passed with options
         try:
             browserEnv.update(
                 dict(
                     parseKeyValue(
                         options.environment,
@@ -1629,17 +1618,17 @@ class MochitestDesktop(MochitestBase):
                       "-w", pwfilePath, "-d", certdbPath],
                      env=toolsEnv)
 
         os.unlink(pwfilePath)
         return 0
 
     def buildProfile(self, options):
         """ create the profile and add optional chrome bits and files if requested """
-        if options.browserChrome and options.timeout:
+        if options.flavor == 'browser' and options.timeout:
             options.extraPrefs.append(
                 "testing.browserTestHarness.timeout=%d" %
                 options.timeout)
         options.extraPrefs.append(
             "browser.tabs.remote.autostart=%s" %
             ('true' if options.e10s else 'false'))
         if options.strictContentSandbox:
             options.extraPrefs.append("security.sandbox.content.level=1")
@@ -2106,31 +2095,38 @@ class MochitestDesktop(MochitestBase):
         """
         self.expectedError.clear()
         self.result.clear()
         options.manifestFile = None
         options.profilePath = None
         self.urlOpts = []
 
     def resolve_runtime_file(self, options):
+        """
+        Return a path to the runtimes file for a given flavor and
+        subsuite.
+        """
+        template = "mochitest{e10s}-{suite_slug}.runtimes.json"
         data_dir = os.path.join(SCRIPT_DIR, 'runtimes')
 
-        flavor = self.getTestFlavor(options)
-        if flavor == 'browser-chrome' and options.subsuite == 'devtools':
-            flavor = 'devtools-chrome'
-        elif flavor == 'mochitest':
-            flavor = 'plain'
+        # Determine the suite slug in the runtimes file name
+        slug = self.normflavor(options.flavor)
+        if slug == 'browser-chrome' and options.subsuite == 'devtools':
+            slug = 'devtools-chrome'
+        elif slug == 'mochitest':
+            slug = 'plain'
             if options.subsuite:
-                flavor = options.subsuite
-
-        base = 'mochitest'
+                slug = options.subsuite
+
+        e10s = ''
         if options.e10s:
-            base = '{}-e10s'.format(base)
-        return os.path.join(data_dir, '{}-{}.runtimes.json'.format(
-            base, flavor))
+            e10s = '-e10s'
+
+        return os.path.join(data_dir, template.format(
+            e10s=e10s, suite_slug=slug))
 
     def normalize_paths(self, paths):
         # Normalize test paths so they are relative to test root
         norm_paths = []
         for p in paths:
             abspath = os.path.abspath(os.path.join(self.oldcwd, p))
             if abspath.startswith(self.testRootAbs):
                 norm_paths.append(os.path.relpath(abspath, self.testRootAbs))
@@ -2190,17 +2186,17 @@ class MochitestDesktop(MochitestBase):
 
         return result
 
     def runTests(self, options):
         """ Prepare, configure, run tests and cleanup """
 
         # a11y and chrome tests don't run with e10s enabled in CI. Need to set
         # this here since |mach mochitest| sets the flavor after argument parsing.
-        if options.a11y or options.chrome:
+        if options.flavor in ('a11y', 'chrome'):
             options.e10s = False
         mozinfo.update({"e10s": options.e10s})  # for test manifest parsing.
 
         self.setTestRoot(options)
 
         # Despite our efforts to clean up servers started by this script, in practice
         # we still see infrequent cases where a process is orphaned and interferes
         # with future tests, typically because the old server is keeping the port in use.
@@ -2236,17 +2232,17 @@ class MochitestDesktop(MochitestBase):
             self.message_logger.dump_buffered()
 
             if result == -1:
                 break
 
         e10s_mode = "e10s" if options.e10s else "non-e10s"
 
         # printing total number of tests
-        if options.browserChrome:
+        if options.flavor == 'browser':
             print "TEST-INFO | checking window state"
             print "Browser Chrome Test Summary"
             print "\tPassed: %s" % self.countpass
             print "\tFailed: %s" % self.countfail
             print "\tTodo: %s" % self.counttodo
             print "\tMode: %s" % e10s_mode
             print "*** End BrowserChrome Test Results ***"
         else:
@@ -2381,19 +2377,19 @@ class MochitestDesktop(MochitestBase):
                 timeout = options.timeout + 30
             elif options.debugger or not options.autorun:
                 timeout = None
             else:
                 timeout = 330.0  # default JS harness timeout is 300 seconds
 
             # detect shutdown leaks for m-bc runs
             detectShutdownLeaks = mozinfo.info[
-                "debug"] and options.browserChrome
-
-            self.start_script_args.append(self.getTestFlavor(options))
+                "debug"] and options.flavor == 'browser'
+
+            self.start_script_args.append(self.normflavor(options.flavor))
             marionette_args = {
                 'symbols_path': options.symbolsPath,
                 'socket_timeout': options.marionette_socket_timeout,
                 'port_timeout': options.marionette_port_timeout,
             }
 
             if options.marionette:
                 host, port = options.marionette.split(':')
@@ -2672,23 +2668,17 @@ def run_test_harness(parser, options):
     logger_options = {
         key: value for key, value in vars(options).iteritems()
         if key.startswith('log') or key == 'valgrind'}
 
     runner = MochitestDesktop(logger_options)
 
     options.runByDir = False
 
-    if runner.getTestFlavor(options) == 'mochitest':
-        options.runByDir = True
-
-    if runner.getTestFlavor(options) == 'browser-chrome':
-        options.runByDir = True
-
-    if runner.getTestFlavor(options) == 'chrome':
+    if options.flavor in ('plain', 'browser', 'chrome'):
         options.runByDir = True
 
     if mozinfo.info.get('buildapp') == 'mulet':
         options.runByDir = False
 
     result = runner.runTests(options)
 
     if runner.mozLogs:
--- a/testing/mochitest/runtestsb2g.py
+++ b/testing/mochitest/runtestsb2g.py
@@ -152,17 +152,17 @@ class MochitestB2G(MochitestBase):
 
             self.remote_log = posixpath.join(self.app_ctx.remote_test_root,
                                              'log', 'mochitest.log')
             if not self.app_ctx.dm.dirExists(
                 posixpath.dirname(
                     self.remote_log)):
                 self.app_ctx.dm.mkDirs(self.remote_log)
 
-            if options.chrome:
+            if options.flavor == 'chrome':
                 # Update chrome manifest file in profile with correct path.
                 self.writeChromeManifest(options)
 
             self.leak_report_file = posixpath.join(
                 self.app_ctx.remote_test_root,
                 'log',
                 'runtests_leaks.log')
 
@@ -195,17 +195,17 @@ class MochitestB2G(MochitestBase):
             # prevent us from trying to pass in an instance of TestManifest via url param.
             manifestFile = options.manifestFile
             options.manifestFile = None
             self.buildURLOptions(options, {'MOZ_HIDE_RESULTS_TABLE': '1'})
             options.manifestFile = manifestFile
 
             self.start_script_args.append(not options.emulator)
             self.start_script_args.append(options.wifi)
-            self.start_script_args.append(options.chrome)
+            self.start_script_args.append(options.flavor == 'chrome')
 
             self.runner.start(outputTimeout=timeout)
 
             self.marionette.wait_for_port()
             self.marionette.start_session()
             self.marionette.set_context(self.marionette.CONTEXT_CHROME)
 
             # Disable offline status management (bug 777145), otherwise the network
@@ -227,17 +227,17 @@ class MochitestB2G(MochitestBase):
                     .getService(Components.interfaces.mozIJSSubScriptLoader);
                   loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserver.jsm",
                     testUtils);
                   testUtils.specialPowersObserver = new testUtils.SpecialPowersObserver();
                   testUtils.specialPowersObserver.init();
                 }
                 """)
 
-            if options.chrome:
+            if options.flavor == 'chrome':
                 self.app_ctx.dm.removeDir(self.remote_chrome_test_dir)
                 self.app_ctx.dm.mkDir(self.remote_chrome_test_dir)
                 local = MochitestBase.getChromeTestDir(self, options)
                 local = os.path.join(local, "chrome")
                 remote = self.remote_chrome_test_dir
                 self.log.info(
                     "pushing %s to %s on device..." %
                     (local, remote))
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -187,17 +187,17 @@ class MochiRemote(MochitestDesktop):
         options.profilePath = self.remoteProfile
         return manifest
 
     def addChromeToProfile(self, options):
         manifest = MochitestDesktop.addChromeToProfile(self, options)
 
         # Support Firefox (browser), B2G (shell), SeaMonkey (navigator), and Webapp
         # Runtime (webapp).
-        if options.chrome:
+        if options.flavor == 'chrome':
             # append overlay to chrome.manifest
             chrome = "overlay chrome://browser/content/browser.xul chrome://mochikit/content/browser-test-overlay.xul"
             path = os.path.join(options.profilePath, 'extensions', 'staged',
                                 'mochikit@mozilla.org', 'chrome.manifest')
             with open(path, "a") as f:
                 f.write(chrome)
         return manifest
 
@@ -221,17 +221,17 @@ class MochiRemote(MochitestDesktop):
         options.profilePath = self.remoteProfile
         options.logFile = self.localLog
         return retVal
 
     def getChromeTestDir(self, options):
         local = super(MochiRemote, self).getChromeTestDir(options)
         local = os.path.join(local, "chrome")
         remote = self.remoteChromeTestDir
-        if options.chrome:
+        if options.flavor == 'chrome':
             self.log.info("pushing %s to %s on device..." % (local, remote))
             self._dm.pushDir(local, remote)
         return remote
 
     def getLogFilePath(self, logFile):
         return logFile
 
     def printDeviceInfo(self, printLogcat=False):
@@ -301,17 +301,17 @@ def run_test_harness(parser, options):
     auto = RemoteAutomation(None, "fennec", processArgs=process_args)
 
     if options is None:
         raise ValueError("Invalid options specified, use --help for a list of valid options")
 
     options.runByDir = False
     # roboextender is used by mochitest-chrome tests like test_java_addons.html,
     # but not by any plain mochitests
-    if not options.chrome:
+    if options.flavor != 'chrome':
         options.extensionsToExclude.append('roboextender@mozilla.org')
 
     dm = options.dm
     auto.setDeviceManager(dm)
     mochitest = MochiRemote(auto, dm, options)
 
     log = mochitest.log
     message_logger.logger = log
--- a/testing/mozharness/configs/android/androidarm.py
+++ b/testing/mozharness/configs/android/androidarm.py
@@ -320,17 +320,17 @@ config = {
             "extra_args": ["--total-chunks=16", "--this-chunk=15"],
         },
         "mochitest-16": {
             "category": "mochitest",
             "extra_args": ["--total-chunks=16", "--this-chunk=16"],
         },
         "mochitest-chrome": {
             "category": "mochitest",
-            "extra_args": ["--chrome"],
+            "extra_args": ["--flavor=chrome"],
         },
         "mochitest-media-1": {
             "category": "mochitest-media",
             "extra_args": ["--this-chunk=1"],
         },
         "mochitest-media-2": {
             "category": "mochitest-media",
             "extra_args": ["--this-chunk=2"],
--- a/testing/mozharness/configs/android/androidarm_4_3.py
+++ b/testing/mozharness/configs/android/androidarm_4_3.py
@@ -118,17 +118,17 @@ config = {
                 "--certificate-path=%(certs_path)s",
                 "--symbols-path=%(symbols_path)s",
                 "--quiet",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--extra-profile-file=fonts",
                 "--extra-profile-file=hyphenation",
                 "--screenshot-on-fail",
-                "--chrome",
+                "--flavor=chrome",
             ],
         },
         "mochitest-plain-gpu": {
             "run_filename": "runtestsremote.py",
             "testsdir": "mochitest",
             "options": [
                 "--dm_trans=adb",
                 "--app=%(app)s",
--- a/testing/mozharness/configs/b2g/emulator_automation_config.py
+++ b/testing/mozharness/configs/b2g/emulator_automation_config.py
@@ -135,17 +135,17 @@ config = {
                 "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s",
                 "--utility-path=%(utility_path)s",
                 "--symbols-path=%(symbols_path)s",
                 "--busybox=%(busybox)s",
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--quiet",
-                "--chrome",
+                "--flavor=chrome",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
             ],
             "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
--- a/testing/mozharness/configs/b2g/taskcluster_emulator_automation.py
+++ b/testing/mozharness/configs/b2g/taskcluster_emulator_automation.py
@@ -123,17 +123,17 @@ config = {
                 "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s",
                 "--utility-path=%(utility_path)s",
                 "--symbols-path=%(symbols_path)s",
                 "--busybox=%(busybox)s",
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--quiet",
-                "--chrome",
+                "--flavor=chrome",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
             ],
             "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
@@ -187,9 +187,9 @@ config = {
                 "--emulator=%(emulator)s",
                 "--homedir=%(homedir)s"
             ],
             "run_filename": "runtests.py",
             "testsdir": "marionette/harness/marionette"
         }
     },
     "vcs_output_timeout": 1760,
-}
\ No newline at end of file
+}
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -196,34 +196,34 @@ config = {
                            "--valgrind-supp-files=" + VALGRIND_SUPP_ARCH +
                                "," + VALGRIND_SUPP_CROSS_ARCH,
                            "--timeout=900", "--max-timeouts=50"],
         "plain": [],
         "plain-gpu": ["--subsuite=gpu"],
         "plain-clipboard": ["--subsuite=clipboard"],
         "plain-chunked": ["--chunk-by-dir=4"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--chrome"],
-        "chrome-gpu": ["--chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--browser-chrome"],
-        "browser-chrome-gpu": ["--browser-chrome", "--subsuite=gpu"],
-        "browser-chrome-clipboard": ["--browser-chrome", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--browser-chrome", "--chunk-by-runtime"],
-        "browser-chrome-addons": ["--browser-chrome", "--chunk-by-runtime", "--tag=addons"],
-        "browser-chrome-coverage": ["--browser-chrome", "--chunk-by-runtime", "--timeout=1200"],
-        "browser-chrome-screenshots": ["--browser-chrome", "--subsuite=screenshots"],
+        "chrome": ["--flavor=chrome"],
+        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "browser-chrome": ["--flavor=browser"],
+        "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
+        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
+        "browser-chrome-coverage": ["--flavor=browser", "--chunk-by-runtime", "--timeout=1200"],
+        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
         "mochitest-gl": ["--subsuite=webgl"],
-        "mochitest-devtools-chrome": ["--browser-chrome", "--subsuite=devtools"],
-        "mochitest-devtools-chrome-chunked": ["--browser-chrome", "--subsuite=devtools", "--chunk-by-runtime"],
-        "jetpack-package": ["--jetpack-package"],
-        "jetpack-package-clipboard": ["--jetpack-package", "--subsuite=clipboard"],
-        "jetpack-addon": ["--jetpack-addon"],
-        "a11y": ["--a11y"],
+        "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
+        "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
+        "jetpack-package": ["--flavor=jetpack-package"],
+        "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
+        "jetpack-addon": ["--flavor=jetpack-addon"],
+        "a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "reftest": {
             "options": ["--suite=reftest"],
             "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
         "crashtest": {
--- a/testing/mozharness/configs/unittests/mac_unittest.py
+++ b/testing/mozharness/configs/unittests/mac_unittest.py
@@ -144,33 +144,33 @@ config = {
     },
     # local mochi suites
     "all_mochitest_suites": {
         "plain": [],
         "plain-gpu": ["--subsuite=gpu"],
         "plain-clipboard": ["--subsuite=clipboard"],
         "plain-chunked": ["--chunk-by-dir=4"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--chrome"],
-        "chrome-gpu": ["--chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--browser-chrome"],
-        "browser-chrome-gpu": ["--browser-chrome", "--subsuite=gpu"],
-        "browser-chrome-clipboard": ["--browser-chrome", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--browser-chrome", "--chunk-by-runtime"],
-        "browser-chrome-addons": ["--browser-chrome", "--chunk-by-runtime", "--tag=addons"],
-        "browser-chrome-screenshots": ["--browser-chrome", "--subsuite=screenshots"],
+        "chrome": ["--flavor=chrome"],
+        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "browser-chrome": ["--flavor=browser"],
+        "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
+        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
+        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
         "mochitest-gl": ["--subsuite=webgl"],
-        "mochitest-devtools-chrome": ["--browser-chrome", "--subsuite=devtools"],
-        "mochitest-devtools-chrome-chunked": ["--browser-chrome", "--subsuite=devtools", "--chunk-by-runtime"],
-        "jetpack-package": ["--jetpack-package"],
-        "jetpack-package-clipboard": ["--jetpack-package", "--subsuite=clipboard"],
-        "jetpack-addon": ["--jetpack-addon"],
-        "a11y": ["--a11y"],
+        "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
+        "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
+        "jetpack-package": ["--flavor=jetpack-package"],
+        "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
+        "jetpack-addon": ["--flavor=jetpack-addon"],
+        "a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "reftest": {
             'options': ["--suite=reftest"],
             'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
         "crashtest": {
--- a/testing/mozharness/configs/unittests/win_unittest.py
+++ b/testing/mozharness/configs/unittests/win_unittest.py
@@ -158,34 +158,34 @@ config = {
     # local mochi suites
     "all_mochitest_suites":
     {
         "plain": [],
         "plain-gpu": ["--subsuite=gpu"],
         "plain-clipboard": ["--subsuite=clipboard"],
         "plain-chunked": ["--chunk-by-dir=4"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--chrome"],
-        "chrome-gpu": ["--chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--browser-chrome"],
-        "browser-chrome-gpu": ["--browser-chrome", "--subsuite=gpu"],
-        "browser-chrome-clipboard": ["--browser-chrome", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--browser-chrome", "--chunk-by-runtime"],
-        "browser-chrome-addons": ["--browser-chrome", "--chunk-by-runtime", "--tag=addons"],
-        "browser-chrome-screenshots": ["--browser-chrome", "--subsuite=screenshots"],
+        "chrome": ["--flavor=chrome"],
+        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "browser-chrome": ["--flavor=browser"],
+        "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
+        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
+        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
         "mochitest-gl": ["--subsuite=webgl"],
-        "mochitest-devtools-chrome": ["--browser-chrome", "--subsuite=devtools"],
-        "mochitest-devtools-chrome-chunked": ["--browser-chrome", "--subsuite=devtools", "--chunk-by-runtime"],
-        "mochitest-metro-chrome": ["--browser-chrome", "--metro-immersive"],
-        "jetpack-package": ["--jetpack-package"],
-        "jetpack-package-clipboard": ["--jetpack-package", "--subsuite=clipboard"],
-        "jetpack-addon": ["--jetpack-addon"],
-        "a11y": ["--a11y"],
+        "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
+        "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
+        "mochitest-metro-chrome": ["--flavor=browser", "--metro-immersive"],
+        "jetpack-package": ["--flavor=jetpack-package"],
+        "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
+        "jetpack-addon": ["--flavor=jetpack-addon"],
+        "a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "reftest": {
             'options': ["--suite=reftest"],
             'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
         "crashtest": {