Bug 1293295 - Replace all mochitest 'flavor' options with a single --flavor argument, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 08 Aug 2016 11:48:49 -0400
changeset 350041 43ebf330ad942617aa8526294493a3dd25c0fc8d
parent 350040 a7a5fd1b36e553163533f4ff112354be2b310a95
child 350042 ff904be8285df7c54096349d0965b0895c43287a
push id6570
push userraliiev@mozilla.com
push dateMon, 14 Nov 2016 12:26:13 +0000
treeherdermozilla-beta@f455459b2ae5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1293295, 1293259
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1293295 - Replace all mochitest 'flavor' options with a single --flavor argument, r=jmaher This accomplishes three things: 1) Easier to use CLI when running without the benefit of testing/mochitest/mach_commands.py 2) Guarantees these arguments are mutually exclusive 3) Simplifies a bunch of logic in the test harness The primary motivation for this change is to slightly improve the UX when running mochitest from a taskcluster interactive loaner. However, this is more of a bandaid solution that was easy to implement before the proper fix in bug 1293259 can be landed. MozReview-Commit-ID: IeHBGrJ0Sji
b2g/build.mk
browser/build.mk
mobile/android/build.mk
testing/mochitest/mach_commands.py
testing/mochitest/mochitest_options.py
testing/mochitest/redirect.html
testing/mochitest/runtests.py
testing/mochitest/runtestsb2g.py
testing/mochitest/runtestsremote.py
testing/mozharness/configs/android/androidarm.py
testing/mozharness/configs/android/androidarm_4_3.py
testing/mozharness/configs/b2g/emulator_automation_config.py
testing/mozharness/configs/b2g/taskcluster_emulator_automation.py
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/configs/unittests/mac_unittest.py
testing/mozharness/configs/unittests/win_unittest.py
--- a/b2g/build.mk
+++ b/b2g/build.mk
@@ -16,16 +16,16 @@ install::
 
 upload::
 	@$(MAKE) -C b2g/installer upload
 
 ifdef ENABLE_TESTS
 # Implemented in testing/testsuite-targets.mk
 
 mochitest-browser-chrome:
-	$(RUN_MOCHITEST) --browser-chrome
+	$(RUN_MOCHITEST) --flavor=browser
 	$(CHECK_TEST_ERROR)
 
 mochitest:: mochitest-browser-chrome
 
 .PHONY: mochitest-browser-chrome
 endif
 
--- a/browser/build.mk
+++ b/browser/build.mk
@@ -40,16 +40,16 @@ hg-bundle::
 
 l10n-check::
 	@$(MAKE) -C browser/locales l10n-check
 
 ifdef ENABLE_TESTS
 # Implemented in testing/testsuite-targets.mk
 
 mochitest-browser-chrome:
-	$(RUN_MOCHITEST) --browser-chrome
+	$(RUN_MOCHITEST) --flavor=browser
 	$(CHECK_TEST_ERROR)
 
 mochitest:: mochitest-browser-chrome
 
 .PHONY: mochitest-browser-chrome
 
 endif
--- a/mobile/android/build.mk
+++ b/mobile/android/build.mk
@@ -48,17 +48,17 @@ deb: package
 
 upload::
 	@$(MAKE) -C mobile/android/installer upload
 
 ifdef ENABLE_TESTS
 # Implemented in testing/testsuite-targets.mk
 
 mochitest-browser-chrome:
-	$(RUN_MOCHITEST) --browser-chrome
+	$(RUN_MOCHITEST) --flavor=browser
 	$(CHECK_TEST_ERROR)
 
 mochitest:: mochitest-browser-chrome
 
 .PHONY: mochitest-browser-chrome
 endif
 
 ifeq ($(OS_TARGET),Linux)
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -85,49 +85,49 @@ ALL_FLAVORS = {
         'aliases': ('plain', 'mochitest'),
         'enabled_apps': ('firefox', 'b2g', 'android', 'mulet'),
     },
     'chrome': {
         'suite': 'chrome',
         'aliases': ('chrome', 'mochitest-chrome'),
         'enabled_apps': ('firefox', 'mulet', 'b2g', 'android'),
         'extra_args': {
-            'chrome': True,
+            'flavor': 'chrome',
         }
     },
     'browser-chrome': {
         'suite': 'browser',
         'aliases': ('browser', 'browser-chrome', 'mochitest-browser-chrome', 'bc'),
         'enabled_apps': ('firefox',),
         'extra_args': {
-            'browserChrome': True,
+            'flavor': 'browser',
         }
     },
     'jetpack-package': {
         'suite': 'jetpack-package',
         'aliases': ('jetpack-package', 'mochitest-jetpack-package', 'jpp'),
         'enabled_apps': ('firefox',),
         'extra_args': {
-            'jetpackPackage': True,
+            'flavor': 'jetpack-package',
         }
     },
     'jetpack-addon': {
         'suite': 'jetpack-addon',
         'aliases': ('jetpack-addon', 'mochitest-jetpack-addon', 'jpa'),
         'enabled_apps': ('firefox',),
         'extra_args': {
-            'jetpackAddon': True,
+            'flavor': 'jetpack-addon',
         }
     },
     'a11y': {
         'suite': 'a11y',
         'aliases': ('a11y', 'mochitest-a11y', 'accessibility'),
         'enabled_apps': ('firefox',),
         'extra_args': {
-            'a11y': True,
+            'flavor': 'a11y',
         }
     },
 }
 
 SUPPORTED_APPS = ['firefox', 'b2g', 'android', 'mulet']
 SUPPORTED_FLAVORS = list(chain.from_iterable([f['aliases'] for f in ALL_FLAVORS.values()]))
 CANONICAL_FLAVORS = sorted([f['aliases'][0] for f in ALL_FLAVORS.values()])
 
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -81,27 +81,33 @@ class ArgumentContainer():
     def get_full_path(self, path, cwd):
         """Get an absolute path relative to cwd."""
         return os.path.normpath(os.path.join(cwd, os.path.expanduser(path)))
 
 
 class MochitestArguments(ArgumentContainer):
     """General mochitest arguments."""
 
+    FLAVORS = ('a11y', 'browser', 'chrome', 'jetpack-addon', 'jetpack-package', 'plain')
     LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL")
-    LEVEL_STRING = ", ".join(LOG_LEVELS)
 
     args = [
         [["test_paths"],
          {"nargs": "*",
           "metavar": "TEST",
           "default": [],
           "help": "Test to run. Can be a single test file or a directory of tests "
                   "(to run recursively). If omitted, the entire suite is run.",
           }],
+        [["-f", "--flavor"],
+         {"default": "plain",
+          "choices": FLAVORS,
+          "help": "Mochitest flavor to run, one of {}. Defaults to 'plain'.".format(FLAVORS),
+          "suppress": build_obj is not None,
+          }],
         [["--keep-open"],
          {"nargs": "?",
           "type": strtobool,
           "const": "true",
           "default": None,
           "help": "Always keep the browser open after tests complete. Or always close the "
                   "browser with --keep-open=false",
           }],
@@ -177,23 +183,18 @@ class MochitestArguments(ArgumentContain
          {"action": "store_true",
           "help": "Shuffle execution order of tests.",
           "default": False,
           }],
         [["--console-level"],
          {"dest": "consoleLevel",
           "choices": LOG_LEVELS,
           "default": "INFO",
-          "help": "One of %s to determine the level of console logging." % LEVEL_STRING,
-          "suppress": True,
-          }],
-        [["--chrome"],
-         {"action": "store_true",
-          "default": False,
-          "help": "Run chrome mochitests.",
+          "help": "One of {} to determine the level of console logging.".format(
+                  ', '.join(LOG_LEVELS)),
           "suppress": True,
           }],
         [["--bisect-chunk"],
          {"dest": "bisectChunk",
           "default": None,
           "help": "Specify the failing test name to find the previous tests that may be "
                   "causing the failure.",
           }],
@@ -202,48 +203,21 @@ class MochitestArguments(ArgumentContain
           "default": "",
           "help": "Start running the test sequence at this test.",
           }],
         [["--end-at"],
          {"dest": "endAt",
           "default": "",
           "help": "Stop running the test sequence at this test.",
           }],
-        [["--browser-chrome"],
-         {"action": "store_true",
-          "dest": "browserChrome",
-          "default": False,
-          "help": "run browser chrome Mochitests",
-          "suppress": True,
-          }],
         [["--subsuite"],
          {"default": None,
           "help": "Subsuite of tests to run. Unlike tags, subsuites also remove tests from "
                   "the default set. Only one can be specified at once.",
           }],
-        [["--jetpack-package"],
-         {"action": "store_true",
-          "dest": "jetpackPackage",
-          "help": "Run jetpack package tests.",
-          "default": False,
-          "suppress": True,
-          }],
-        [["--jetpack-addon"],
-         {"action": "store_true",
-          "dest": "jetpackAddon",
-          "help": "Run jetpack addon tests.",
-          "default": False,
-          "suppress": True,
-          }],
-        [["--a11y"],
-         {"action": "store_true",
-          "help": "Run accessibility Mochitests.",
-          "default": False,
-          "suppress": True,
-          }],
         [["--setenv"],
          {"action": "append",
           "dest": "environment",
           "metavar": "NAME=VALUE",
           "default": [],
           "help": "Sets the given variable in the application's environment.",
           }],
         [["--exclude-extension"],
--- a/testing/mochitest/redirect.html
+++ b/testing/mochitest/redirect.html
@@ -1,17 +1,17 @@
 <html>
 <head>
   <title>redirecting...</title>
 
   <script type="text/javascript">
     function redirect(aURL)
     {
-      // We create a listener for this event in browser-test.js 
-      // which will get picked up when specifying --chrome or --a11y
+      // We create a listener for this event in browser-test.js which will
+      // get picked up when specifying --flavor=chrome or --flavor=a11y
       var event = new CustomEvent("contentEvent", {
         bubbles: true,
         detail: {
           "data": aURL + location.search,
           "type": "loadURI"
         }
       });
       document.dispatchEvent(event);
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -614,18 +614,17 @@ class MochitestBase(object):
             options.logFile = ""
         if not hasattr(options, 'fileLevel'):
             options.fileLevel = 'INFO'
 
         # allow relative paths for logFile
         if options.logFile:
             options.logFile = self.getLogFilePath(options.logFile)
 
-        if options.browserChrome or options.chrome or \
-           options.a11y or options.jetpackPackage or options.jetpackAddon:
+        if options.flavor in ('a11y', 'browser', 'chrome', 'jetpack-addon', 'jetpack-package'):
             self.makeTestConfig(options)
         else:
             if options.autorun:
                 self.urlOpts.append("autorun=1")
             if options.timeout:
                 self.urlOpts.append("timeout=%d" % options.timeout)
             if options.maxTimeouts:
                 self.urlOpts.append("maxTimeouts=%d" % options.maxTimeouts)
@@ -684,68 +683,59 @@ class MochitestBase(object):
                         options.dumpOutputDirectory))
             if options.dumpAboutMemoryAfterTest:
                 self.urlOpts.append("dumpAboutMemoryAfterTest=true")
             if options.dumpDMDAfterTest:
                 self.urlOpts.append("dumpDMDAfterTest=true")
             if options.debugger:
                 self.urlOpts.append("interactiveDebugger=true")
 
-    def getTestFlavor(self, options):
-        if options.browserChrome:
-            return "browser-chrome"
-        elif options.jetpackPackage:
-            return "jetpack-package"
-        elif options.jetpackAddon:
-            return "jetpack-addon"
-        elif options.chrome:
-            return "chrome"
-        elif options.a11y:
-            return "a11y"
-        else:
-            return "mochitest"
+    def normflavor(self, flavor):
+        """
+        In some places the string 'browser-chrome' is expected instead of
+        'browser' and 'mochitest' instead of 'plain'. Normalize the flavor
+        strings for those instances.
+        """
+        # TODO Use consistent flavor strings everywhere and remove this
+        if flavor == 'browser':
+            return 'browser-chrome'
+        elif flavor == 'plain':
+            return 'mochitest'
+        return flavor
 
     # This check can be removed when bug 983867 is fixed.
     def isTest(self, options, filename):
         allow_js_css = False
-        if options.browserChrome:
+        if options.flavor == 'browser':
             allow_js_css = True
             testPattern = re.compile(r"browser_.+\.js")
-        elif options.jetpackPackage:
+        elif options.flavor == 'jetpack-package':
             allow_js_css = True
             testPattern = re.compile(r"test-.+\.js")
-        elif options.jetpackAddon:
+        elif options.flavor == 'jetpack-addon':
             testPattern = re.compile(r".+\.xpi")
-        elif options.chrome or options.a11y:
+        elif options.flavor in ('a11y', 'chrome'):
             testPattern = re.compile(r"(browser|test)_.+\.(xul|html|js|xhtml)")
         else:
             testPattern = re.compile(r"test_")
 
         if not allow_js_css and (".js" in filename or ".css" in filename):
             return False
 
         pathPieces = filename.split("/")
 
         return (testPattern.match(pathPieces[-1]) and
                 not re.search(r'\^headers\^$', filename))
 
     def setTestRoot(self, options):
-        if options.browserChrome:
-            if options.immersiveMode:
+        if options.flavor != 'plain':
+            self.testRoot = options.flavor
+
+            if options.flavor == 'browser' and options.immersiveMode:
                 self.testRoot = 'metro'
-            else:
-                self.testRoot = 'browser'
-        elif options.jetpackPackage:
-            self.testRoot = 'jetpack-package'
-        elif options.jetpackAddon:
-            self.testRoot = 'jetpack-addon'
-        elif options.a11y:
-            self.testRoot = 'a11y'
-        elif options.chrome:
-            self.testRoot = 'chrome'
         else:
             self.testRoot = self.TEST_PATH
         self.testRootAbs = os.path.join(SCRIPT_DIR, self.testRoot)
 
     def buildTestURL(self, options):
         testHost = "http://mochi.test:8888"
         testURL = "/".join([testHost, self.TEST_PATH])
 
@@ -755,19 +745,19 @@ class MochitestBase(object):
                     self.oldcwd,
                     os.path.dirname(__file__),
                     self.TEST_PATH,
                     options.test_paths[0])):
                 testURL = "/".join([testURL, os.path.dirname(options.test_paths[0])])
             else:
                 testURL = "/".join([testURL, options.test_paths[0]])
 
-        if options.chrome or options.a11y:
+        if options.flavor in ('a11y', 'chrome'):
             testURL = "/".join([testHost, self.CHROME_PATH])
-        elif options.browserChrome or options.jetpackPackage or options.jetpackAddon:
+        elif options.flavor in ('browser', 'jetpack-addon', 'jetpack-package'):
             testURL = "about:blank"
         if options.nested_oop:
             testURL = "/".join([testHost, self.NESTED_OOP_TEST_PATH])
         return testURL
 
     def buildTestPath(self, options, testsToFilter=None, disabled=True):
         """ Build the url path to the specific test harness and test file or directory
             Build a manifest of tests to run and write out a json file for the harness to read
@@ -1061,18 +1051,17 @@ toolbar#nav-bar {
             # Add chunking filters if specified
             if options.totalChunks:
                 if options.chunkByRuntime:
                     runtime_file = self.resolve_runtime_file(options)
                     if not os.path.exists(runtime_file):
                         self.log.warning("runtime file %s not found; defaulting to chunk-by-dir" %
                                          runtime_file)
                         options.chunkByRuntime = None
-                        flavor = self.getTestFlavor(options)
-                        if flavor in ('browser-chrome', 'devtools-chrome'):
+                        if options.flavor == 'browser':
                             # these values match current mozharness configs
                             options.chunkbyDir = 5
                         else:
                             options.chunkByDir = 4
 
                 if options.chunkByDir:
                     filters.append(chunk_by_dir(options.thisChunk,
                                                 options.totalChunks,
@@ -1149,17 +1138,17 @@ toolbar#nav-bar {
         elif options.manifestFile and os.path.isfile(os.path.join(SCRIPT_DIR, options.manifestFile)):
             manifestFileAbs = os.path.abspath(
                 os.path.join(
                     SCRIPT_DIR,
                     options.manifestFile))
             assert manifestFileAbs.startswith(SCRIPT_DIR)
             manifest = TestManifest([manifestFileAbs], strict=False)
         else:
-            masterName = self.getTestFlavor(options) + '.ini'
+            masterName = self.normflavor(options.flavor) + '.ini'
             masterPath = os.path.join(SCRIPT_DIR, self.testRoot, masterName)
 
             if os.path.exists(masterPath):
                 manifest = TestManifest([masterPath], strict=False)
             else:
                 self._log.warning(
                     'TestManifest masterPath %s does not exist' %
                     masterPath)
@@ -1205,17 +1194,17 @@ toolbar#nav-bar {
         # via the commandline at your own risk.
         browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
 
         # When creating child processes on Windows pre-Vista (e.g. Windows XP) we
         # don't normally inherit stdout/err handles, because you can only do it by
         # inheriting all other inheritable handles as well.
         # We need to inherit them for plain mochitests for test logging purposes, so
         # we do so on the basis of a specific environment variable.
-        if self.getTestFlavor(options) == "mochitest":
+        if options.flavor == 'plain':
             browserEnv["MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA"] = "1"
 
         # interpolate environment passed with options
         try:
             browserEnv.update(
                 dict(
                     parseKeyValue(
                         options.environment,
@@ -1629,17 +1618,17 @@ class MochitestDesktop(MochitestBase):
                       "-w", pwfilePath, "-d", certdbPath],
                      env=toolsEnv)
 
         os.unlink(pwfilePath)
         return 0
 
     def buildProfile(self, options):
         """ create the profile and add optional chrome bits and files if requested """
-        if options.browserChrome and options.timeout:
+        if options.flavor == 'browser' and options.timeout:
             options.extraPrefs.append(
                 "testing.browserTestHarness.timeout=%d" %
                 options.timeout)
         options.extraPrefs.append(
             "browser.tabs.remote.autostart=%s" %
             ('true' if options.e10s else 'false'))
         if options.strictContentSandbox:
             options.extraPrefs.append("security.sandbox.content.level=1")
@@ -2106,31 +2095,38 @@ class MochitestDesktop(MochitestBase):
         """
         self.expectedError.clear()
         self.result.clear()
         options.manifestFile = None
         options.profilePath = None
         self.urlOpts = []
 
     def resolve_runtime_file(self, options):
+        """
+        Return a path to the runtimes file for a given flavor and
+        subsuite.
+        """
+        template = "mochitest{e10s}-{suite_slug}.runtimes.json"
         data_dir = os.path.join(SCRIPT_DIR, 'runtimes')
 
-        flavor = self.getTestFlavor(options)
-        if flavor == 'browser-chrome' and options.subsuite == 'devtools':
-            flavor = 'devtools-chrome'
-        elif flavor == 'mochitest':
-            flavor = 'plain'
+        # Determine the suite slug in the runtimes file name
+        slug = self.normflavor(options.flavor)
+        if slug == 'browser-chrome' and options.subsuite == 'devtools':
+            slug = 'devtools-chrome'
+        elif slug == 'mochitest':
+            slug = 'plain'
             if options.subsuite:
-                flavor = options.subsuite
-
-        base = 'mochitest'
+                slug = options.subsuite
+
+        e10s = ''
         if options.e10s:
-            base = '{}-e10s'.format(base)
-        return os.path.join(data_dir, '{}-{}.runtimes.json'.format(
-            base, flavor))
+            e10s = '-e10s'
+
+        return os.path.join(data_dir, template.format(
+            e10s=e10s, suite_slug=slug))
 
     def normalize_paths(self, paths):
         # Normalize test paths so they are relative to test root
         norm_paths = []
         for p in paths:
             abspath = os.path.abspath(os.path.join(self.oldcwd, p))
             if abspath.startswith(self.testRootAbs):
                 norm_paths.append(os.path.relpath(abspath, self.testRootAbs))
@@ -2190,17 +2186,17 @@ class MochitestDesktop(MochitestBase):
 
         return result
 
     def runTests(self, options):
         """ Prepare, configure, run tests and cleanup """
 
         # a11y and chrome tests don't run with e10s enabled in CI. Need to set
         # this here since |mach mochitest| sets the flavor after argument parsing.
-        if options.a11y or options.chrome:
+        if options.flavor in ('a11y', 'chrome'):
             options.e10s = False
         mozinfo.update({"e10s": options.e10s})  # for test manifest parsing.
 
         self.setTestRoot(options)
 
         # Despite our efforts to clean up servers started by this script, in practice
         # we still see infrequent cases where a process is orphaned and interferes
         # with future tests, typically because the old server is keeping the port in use.
@@ -2236,17 +2232,17 @@ class MochitestDesktop(MochitestBase):
             self.message_logger.dump_buffered()
 
             if result == -1:
                 break
 
         e10s_mode = "e10s" if options.e10s else "non-e10s"
 
         # printing total number of tests
-        if options.browserChrome:
+        if options.flavor == 'browser':
             print "TEST-INFO | checking window state"
             print "Browser Chrome Test Summary"
             print "\tPassed: %s" % self.countpass
             print "\tFailed: %s" % self.countfail
             print "\tTodo: %s" % self.counttodo
             print "\tMode: %s" % e10s_mode
             print "*** End BrowserChrome Test Results ***"
         else:
@@ -2381,19 +2377,19 @@ class MochitestDesktop(MochitestBase):
                 timeout = options.timeout + 30
             elif options.debugger or not options.autorun:
                 timeout = None
             else:
                 timeout = 330.0  # default JS harness timeout is 300 seconds
 
             # detect shutdown leaks for m-bc runs
             detectShutdownLeaks = mozinfo.info[
-                "debug"] and options.browserChrome
-
-            self.start_script_args.append(self.getTestFlavor(options))
+                "debug"] and options.flavor == 'browser'
+
+            self.start_script_args.append(self.normflavor(options.flavor))
             marionette_args = {
                 'symbols_path': options.symbolsPath,
                 'socket_timeout': options.marionette_socket_timeout,
                 'port_timeout': options.marionette_port_timeout,
             }
 
             if options.marionette:
                 host, port = options.marionette.split(':')
@@ -2672,23 +2668,17 @@ def run_test_harness(parser, options):
     logger_options = {
         key: value for key, value in vars(options).iteritems()
         if key.startswith('log') or key == 'valgrind'}
 
     runner = MochitestDesktop(logger_options)
 
     options.runByDir = False
 
-    if runner.getTestFlavor(options) == 'mochitest':
-        options.runByDir = True
-
-    if runner.getTestFlavor(options) == 'browser-chrome':
-        options.runByDir = True
-
-    if runner.getTestFlavor(options) == 'chrome':
+    if options.flavor in ('plain', 'browser', 'chrome'):
         options.runByDir = True
 
     if mozinfo.info.get('buildapp') == 'mulet':
         options.runByDir = False
 
     result = runner.runTests(options)
 
     if runner.mozLogs:
--- a/testing/mochitest/runtestsb2g.py
+++ b/testing/mochitest/runtestsb2g.py
@@ -152,17 +152,17 @@ class MochitestB2G(MochitestBase):
 
             self.remote_log = posixpath.join(self.app_ctx.remote_test_root,
                                              'log', 'mochitest.log')
             if not self.app_ctx.dm.dirExists(
                 posixpath.dirname(
                     self.remote_log)):
                 self.app_ctx.dm.mkDirs(self.remote_log)
 
-            if options.chrome:
+            if options.flavor == 'chrome':
                 # Update chrome manifest file in profile with correct path.
                 self.writeChromeManifest(options)
 
             self.leak_report_file = posixpath.join(
                 self.app_ctx.remote_test_root,
                 'log',
                 'runtests_leaks.log')
 
@@ -195,17 +195,17 @@ class MochitestB2G(MochitestBase):
             # prevent us from trying to pass in an instance of TestManifest via url param.
             manifestFile = options.manifestFile
             options.manifestFile = None
             self.buildURLOptions(options, {'MOZ_HIDE_RESULTS_TABLE': '1'})
             options.manifestFile = manifestFile
 
             self.start_script_args.append(not options.emulator)
             self.start_script_args.append(options.wifi)
-            self.start_script_args.append(options.chrome)
+            self.start_script_args.append(options.flavor == 'chrome')
 
             self.runner.start(outputTimeout=timeout)
 
             self.marionette.wait_for_port()
             self.marionette.start_session()
             self.marionette.set_context(self.marionette.CONTEXT_CHROME)
 
             # Disable offline status management (bug 777145), otherwise the network
@@ -227,17 +227,17 @@ class MochitestB2G(MochitestBase):
                     .getService(Components.interfaces.mozIJSSubScriptLoader);
                   loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserver.jsm",
                     testUtils);
                   testUtils.specialPowersObserver = new testUtils.SpecialPowersObserver();
                   testUtils.specialPowersObserver.init();
                 }
                 """)
 
-            if options.chrome:
+            if options.flavor == 'chrome':
                 self.app_ctx.dm.removeDir(self.remote_chrome_test_dir)
                 self.app_ctx.dm.mkDir(self.remote_chrome_test_dir)
                 local = MochitestBase.getChromeTestDir(self, options)
                 local = os.path.join(local, "chrome")
                 remote = self.remote_chrome_test_dir
                 self.log.info(
                     "pushing %s to %s on device..." %
                     (local, remote))
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -187,17 +187,17 @@ class MochiRemote(MochitestDesktop):
         options.profilePath = self.remoteProfile
         return manifest
 
     def addChromeToProfile(self, options):
         manifest = MochitestDesktop.addChromeToProfile(self, options)
 
         # Support Firefox (browser), B2G (shell), SeaMonkey (navigator), and Webapp
         # Runtime (webapp).
-        if options.chrome:
+        if options.flavor == 'chrome':
             # append overlay to chrome.manifest
             chrome = "overlay chrome://browser/content/browser.xul chrome://mochikit/content/browser-test-overlay.xul"
             path = os.path.join(options.profilePath, 'extensions', 'staged',
                                 'mochikit@mozilla.org', 'chrome.manifest')
             with open(path, "a") as f:
                 f.write(chrome)
         return manifest
 
@@ -221,17 +221,17 @@ class MochiRemote(MochitestDesktop):
         options.profilePath = self.remoteProfile
         options.logFile = self.localLog
         return retVal
 
     def getChromeTestDir(self, options):
         local = super(MochiRemote, self).getChromeTestDir(options)
         local = os.path.join(local, "chrome")
         remote = self.remoteChromeTestDir
-        if options.chrome:
+        if options.flavor == 'chrome':
             self.log.info("pushing %s to %s on device..." % (local, remote))
             self._dm.pushDir(local, remote)
         return remote
 
     def getLogFilePath(self, logFile):
         return logFile
 
     def printDeviceInfo(self, printLogcat=False):
@@ -301,17 +301,17 @@ def run_test_harness(parser, options):
     auto = RemoteAutomation(None, "fennec", processArgs=process_args)
 
     if options is None:
         raise ValueError("Invalid options specified, use --help for a list of valid options")
 
     options.runByDir = False
     # roboextender is used by mochitest-chrome tests like test_java_addons.html,
     # but not by any plain mochitests
-    if not options.chrome:
+    if options.flavor != 'chrome':
         options.extensionsToExclude.append('roboextender@mozilla.org')
 
     dm = options.dm
     auto.setDeviceManager(dm)
     mochitest = MochiRemote(auto, dm, options)
 
     log = mochitest.log
     message_logger.logger = log
--- a/testing/mozharness/configs/android/androidarm.py
+++ b/testing/mozharness/configs/android/androidarm.py
@@ -320,17 +320,17 @@ config = {
             "extra_args": ["--total-chunks=16", "--this-chunk=15"],
         },
         "mochitest-16": {
             "category": "mochitest",
             "extra_args": ["--total-chunks=16", "--this-chunk=16"],
         },
         "mochitest-chrome": {
             "category": "mochitest",
-            "extra_args": ["--chrome"],
+            "extra_args": ["--flavor=chrome"],
         },
         "mochitest-media-1": {
             "category": "mochitest-media",
             "extra_args": ["--this-chunk=1"],
         },
         "mochitest-media-2": {
             "category": "mochitest-media",
             "extra_args": ["--this-chunk=2"],
--- a/testing/mozharness/configs/android/androidarm_4_3.py
+++ b/testing/mozharness/configs/android/androidarm_4_3.py
@@ -118,17 +118,17 @@ config = {
                 "--certificate-path=%(certs_path)s",
                 "--symbols-path=%(symbols_path)s",
                 "--quiet",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--extra-profile-file=fonts",
                 "--extra-profile-file=hyphenation",
                 "--screenshot-on-fail",
-                "--chrome",
+                "--flavor=chrome",
             ],
         },
         "mochitest-plain-gpu": {
             "run_filename": "runtestsremote.py",
             "testsdir": "mochitest",
             "options": [
                 "--dm_trans=adb",
                 "--app=%(app)s",
--- a/testing/mozharness/configs/b2g/emulator_automation_config.py
+++ b/testing/mozharness/configs/b2g/emulator_automation_config.py
@@ -135,17 +135,17 @@ config = {
                 "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s",
                 "--utility-path=%(utility_path)s",
                 "--symbols-path=%(symbols_path)s",
                 "--busybox=%(busybox)s",
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--quiet",
-                "--chrome",
+                "--flavor=chrome",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
             ],
             "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
--- a/testing/mozharness/configs/b2g/taskcluster_emulator_automation.py
+++ b/testing/mozharness/configs/b2g/taskcluster_emulator_automation.py
@@ -123,17 +123,17 @@ config = {
                 "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s",
                 "--utility-path=%(utility_path)s",
                 "--symbols-path=%(symbols_path)s",
                 "--busybox=%(busybox)s",
                 "--total-chunks=%(total_chunks)s",
                 "--this-chunk=%(this_chunk)s",
                 "--quiet",
-                "--chrome",
+                "--flavor=chrome",
                 "--log-raw=%(raw_log_file)s",
                 "--log-errorsummary=%(error_summary_file)s",
                 "--certificate-path=%(certificate_path)s",
                 "--screenshot-on-fail",
             ],
             "tests": ["%(test_path)s"],
             "run_filename": "runtestsb2g.py",
             "testsdir": "mochitest"
@@ -187,9 +187,9 @@ config = {
                 "--emulator=%(emulator)s",
                 "--homedir=%(homedir)s"
             ],
             "run_filename": "runtests.py",
             "testsdir": "marionette/harness/marionette"
         }
     },
     "vcs_output_timeout": 1760,
-}
\ No newline at end of file
+}
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -196,34 +196,34 @@ config = {
                            "--valgrind-supp-files=" + VALGRIND_SUPP_ARCH +
                                "," + VALGRIND_SUPP_CROSS_ARCH,
                            "--timeout=900", "--max-timeouts=50"],
         "plain": [],
         "plain-gpu": ["--subsuite=gpu"],
         "plain-clipboard": ["--subsuite=clipboard"],
         "plain-chunked": ["--chunk-by-dir=4"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--chrome"],
-        "chrome-gpu": ["--chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--browser-chrome"],
-        "browser-chrome-gpu": ["--browser-chrome", "--subsuite=gpu"],
-        "browser-chrome-clipboard": ["--browser-chrome", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--browser-chrome", "--chunk-by-runtime"],
-        "browser-chrome-addons": ["--browser-chrome", "--chunk-by-runtime", "--tag=addons"],
-        "browser-chrome-coverage": ["--browser-chrome", "--chunk-by-runtime", "--timeout=1200"],
-        "browser-chrome-screenshots": ["--browser-chrome", "--subsuite=screenshots"],
+        "chrome": ["--flavor=chrome"],
+        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "browser-chrome": ["--flavor=browser"],
+        "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
+        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
+        "browser-chrome-coverage": ["--flavor=browser", "--chunk-by-runtime", "--timeout=1200"],
+        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
         "mochitest-gl": ["--subsuite=webgl"],
-        "mochitest-devtools-chrome": ["--browser-chrome", "--subsuite=devtools"],
-        "mochitest-devtools-chrome-chunked": ["--browser-chrome", "--subsuite=devtools", "--chunk-by-runtime"],
-        "jetpack-package": ["--jetpack-package"],
-        "jetpack-package-clipboard": ["--jetpack-package", "--subsuite=clipboard"],
-        "jetpack-addon": ["--jetpack-addon"],
-        "a11y": ["--a11y"],
+        "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
+        "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
+        "jetpack-package": ["--flavor=jetpack-package"],
+        "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
+        "jetpack-addon": ["--flavor=jetpack-addon"],
+        "a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "reftest": {
             "options": ["--suite=reftest"],
             "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
         "crashtest": {
--- a/testing/mozharness/configs/unittests/mac_unittest.py
+++ b/testing/mozharness/configs/unittests/mac_unittest.py
@@ -144,33 +144,33 @@ config = {
     },
     # local mochi suites
     "all_mochitest_suites": {
         "plain": [],
         "plain-gpu": ["--subsuite=gpu"],
         "plain-clipboard": ["--subsuite=clipboard"],
         "plain-chunked": ["--chunk-by-dir=4"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--chrome"],
-        "chrome-gpu": ["--chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--browser-chrome"],
-        "browser-chrome-gpu": ["--browser-chrome", "--subsuite=gpu"],
-        "browser-chrome-clipboard": ["--browser-chrome", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--browser-chrome", "--chunk-by-runtime"],
-        "browser-chrome-addons": ["--browser-chrome", "--chunk-by-runtime", "--tag=addons"],
-        "browser-chrome-screenshots": ["--browser-chrome", "--subsuite=screenshots"],
+        "chrome": ["--flavor=chrome"],
+        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "browser-chrome": ["--flavor=browser"],
+        "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
+        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
+        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
         "mochitest-gl": ["--subsuite=webgl"],
-        "mochitest-devtools-chrome": ["--browser-chrome", "--subsuite=devtools"],
-        "mochitest-devtools-chrome-chunked": ["--browser-chrome", "--subsuite=devtools", "--chunk-by-runtime"],
-        "jetpack-package": ["--jetpack-package"],
-        "jetpack-package-clipboard": ["--jetpack-package", "--subsuite=clipboard"],
-        "jetpack-addon": ["--jetpack-addon"],
-        "a11y": ["--a11y"],
+        "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
+        "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
+        "jetpack-package": ["--flavor=jetpack-package"],
+        "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
+        "jetpack-addon": ["--flavor=jetpack-addon"],
+        "a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "reftest": {
             'options': ["--suite=reftest"],
             'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
         "crashtest": {
--- a/testing/mozharness/configs/unittests/win_unittest.py
+++ b/testing/mozharness/configs/unittests/win_unittest.py
@@ -158,34 +158,34 @@ config = {
     # local mochi suites
     "all_mochitest_suites":
     {
         "plain": [],
         "plain-gpu": ["--subsuite=gpu"],
         "plain-clipboard": ["--subsuite=clipboard"],
         "plain-chunked": ["--chunk-by-dir=4"],
         "mochitest-media": ["--subsuite=media"],
-        "chrome": ["--chrome"],
-        "chrome-gpu": ["--chrome", "--subsuite=gpu"],
-        "chrome-clipboard": ["--chrome", "--subsuite=clipboard"],
-        "chrome-chunked": ["--chrome", "--chunk-by-dir=4"],
-        "browser-chrome": ["--browser-chrome"],
-        "browser-chrome-gpu": ["--browser-chrome", "--subsuite=gpu"],
-        "browser-chrome-clipboard": ["--browser-chrome", "--subsuite=clipboard"],
-        "browser-chrome-chunked": ["--browser-chrome", "--chunk-by-runtime"],
-        "browser-chrome-addons": ["--browser-chrome", "--chunk-by-runtime", "--tag=addons"],
-        "browser-chrome-screenshots": ["--browser-chrome", "--subsuite=screenshots"],
+        "chrome": ["--flavor=chrome"],
+        "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
+        "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
+        "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
+        "browser-chrome": ["--flavor=browser"],
+        "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
+        "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
+        "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
+        "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
+        "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
         "mochitest-gl": ["--subsuite=webgl"],
-        "mochitest-devtools-chrome": ["--browser-chrome", "--subsuite=devtools"],
-        "mochitest-devtools-chrome-chunked": ["--browser-chrome", "--subsuite=devtools", "--chunk-by-runtime"],
-        "mochitest-metro-chrome": ["--browser-chrome", "--metro-immersive"],
-        "jetpack-package": ["--jetpack-package"],
-        "jetpack-package-clipboard": ["--jetpack-package", "--subsuite=clipboard"],
-        "jetpack-addon": ["--jetpack-addon"],
-        "a11y": ["--a11y"],
+        "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
+        "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
+        "mochitest-metro-chrome": ["--flavor=browser", "--metro-immersive"],
+        "jetpack-package": ["--flavor=jetpack-package"],
+        "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
+        "jetpack-addon": ["--flavor=jetpack-addon"],
+        "a11y": ["--flavor=a11y"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "reftest": {
             'options': ["--suite=reftest"],
             'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
         "crashtest": {