Bug 1070041 - Move more Mozharness job definition parameters into the tree. r=ahal
authorArmen Zambrano Gasparnian <armenzg@mozilla.com>
Tue, 16 Dec 2014 11:23:17 -0500
changeset 220196 f89cbdabd57cccb2c5b922a2d45a034145c5de4f
parent 220195 9d0ed89e7c5820309c55f12b8a81122298934666
child 220197 7c9dc772c39d3af202711e87b86f8cb79d95832c
push id10457
push userryanvm@gmail.com
push dateThu, 18 Dec 2014 01:54:25 +0000
treeherderfx-team@0e441ff66c5e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1070041
milestone37.0a1
Bug 1070041 - Move more Mozharness job definition parameters into the tree. r=ahal * We are unifying the in-tree mozharness configs to follow the same structure * We are moving run_filename into the tree * We are also staring the move to manage the testdir value which is already in use for Android test jobs
testing/config/mozharness/android_panda_config.py
testing/config/mozharness/b2g_desktop_config.py
testing/config/mozharness/b2g_emulator_config.py
testing/config/mozharness/linux_config.py
testing/config/mozharness/mac_config.py
testing/config/mozharness/marionette.py
testing/config/mozharness/windows_config.py
--- a/testing/config/mozharness/android_panda_config.py
+++ b/testing/config/mozharness/android_panda_config.py
@@ -1,88 +1,137 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 config = {
-    "mochitest_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin", "--certificate-path=certs",
-        "--app=%(app_name)s", "--console-level=INFO",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--run-only-tests=android.json", "--symbols-path=%(symbols_path)s",
-        "--quiet", "--log-raw=%(raw_log_file)s"
-    ],
-    "reftest_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin",
-        "--app=%(app_name)s",
-        "--ignore-window-size", "--bootstrap",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--symbols-path=%(symbols_path)s",
-        "reftest/tests/layout/reftests/reftest.list"
-    ],
-    "crashtest_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin",
-        "--app=%(app_name)s",
-        "--ignore-window-size", "--bootstrap",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--symbols-path=%(symbols_path)s",
-        "reftest/tests/testing/crashtest/crashtests.list"
-    ],
-    "jsreftest_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin",
-        "--app=%(app_name)s",
-        "--ignore-window-size", "--bootstrap",
-        "--extra-profile-file=jsreftest/tests/user.js", "jsreftest/tests/jstests.list",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--symbols-path=%(symbols_path)s"
-    ],
-    "robocop_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin",
-        "--certificate-path=certs",
-        "--app=%(app_name)s", "--console-level=INFO",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--symbols-path=%(symbols_path)s",
-        "--robocop=mochitest/robocop.ini"
-    ],
-    "xpcshell_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--manifest=xpcshell/tests/xpcshell.ini",
-        "--build-info-json=xpcshell/mozinfo.json",
-        "--testing-modules-dir=modules",
-        "--local-lib-dir=../fennec",
-        "--apk=../%(apk_name)s",
-        "--no-logfiles",
-        "--symbols-path=%(symbols_path)s",
-        "--log-raw=%(raw_log_file)s",
-    ],
-    "jittest_options": [
-        "bin/js",
-        "--remote",
-        "-j", "1",
-        "--deviceTransport=sut",
-        "--deviceIP=%(device_ip)s",
-        "--localLib=../tests/bin",
-        "--no-slow",
-        "--no-progress",
-        "--tinderbox",
-        "--tbpl"
-     ],
-     "cppunittest_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=tests/bin",
-        "--dm_trans=SUT",
-        "--deviceIP=%(device_ip)s",
-        "--localBinDir=../tests/bin",
-        "--apk=%(apk_path)s",
-        "--skip-manifest=../tests/cppunittests/android_cppunittest_manifest.txt"
-     ],
-}
+    "suite_definitions": {
+        "cppunittest": {
+            "options": [
+                "--symbols-path=%(symbols_path)s",
+                "--xre-path=tests/bin",
+                "--dm_trans=SUT",
+                "--deviceIP=%(device_ip)s",
+                "--localBinDir=../tests/bin",
+                "--apk=%(apk_path)s",
+                "--skip-manifest=../tests/cppunittests/android_cppunittest_manifest.txt"
+            ],
+            "run_filename": "remotecppunittests.py",
+            "testsdir": "cppunittests"
+        },
+        "crashtest": {
+            "options": [
+                "--deviceIP=%(device_ip)s",
+                "--xre-path=../hostutils/xre",
+                "--utility-path=../hostutils/bin",
+                "--app=%(app_name)s",
+                "--ignore-window-size",
+                "--bootstrap",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--symbols-path=%(symbols_path)s",
+                "reftest/tests/testing/crashtest/crashtests.list"
+            ],
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest"
+        },
+        "jittest": {
+            "options": [
+                "bin/js",
+                "--remote",
+                "-j",
+                "1",
+                "--deviceTransport=sut",
+                "--deviceIP=%(device_ip)s",
+                "--localLib=../tests/bin",
+                "--no-slow",
+                "--no-progress",
+                "--tinderbox",
+                "--tbpl"
+            ],
+            "run_filename": "jit_test.py",
+            "testsdir": "jit-test/jit-test"
+        },
+        "jsreftest": {
+            "options": [
+                "--deviceIP=%(device_ip)s",
+                "--xre-path=../hostutils/xre",
+                "--utility-path=../hostutils/bin",
+                "--app=%(app_name)s",
+                "--ignore-window-size",
+                "--bootstrap",
+                "--extra-profile-file=jsreftest/tests/user.js",
+                "jsreftest/tests/jstests.list",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--symbols-path=%(symbols_path)s"
+            ],
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest"
+        },
+        "mochitest": {
+            "options": [
+                "--deviceIP=%(device_ip)s",
+                "--xre-path=../hostutils/xre",
+                "--utility-path=../hostutils/bin",
+                "--certificate-path=certs",
+                "--app=%(app_name)s",
+                "--console-level=INFO",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--run-only-tests=android.json",
+                "--symbols-path=%(symbols_path)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "runtestsremote.py",
+            "testsdir": "mochitest"
+        },
+        "reftest": {
+            "options": [
+                "--deviceIP=%(device_ip)s",
+                "--xre-path=../hostutils/xre",
+                "--utility-path=../hostutils/bin",
+                "--app=%(app_name)s",
+                "--ignore-window-size",
+                "--bootstrap",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--symbols-path=%(symbols_path)s",
+                "reftest/tests/layout/reftests/reftest.list"
+            ],
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest"
+        },
+        "robocop": {
+            "options": [
+                "--deviceIP=%(device_ip)s",
+                "--xre-path=../hostutils/xre",
+                "--utility-path=../hostutils/bin",
+                "--certificate-path=certs",
+                "--app=%(app_name)s",
+                "--console-level=INFO",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--symbols-path=%(symbols_path)s",
+                "--robocop=mochitest/robocop.ini"
+            ],
+            "run_filename": "runtestsremote.py",
+            "testsdir": "mochitest"
+        },
+        "xpcshell": {
+            "options": [
+                "--deviceIP=%(device_ip)s",
+                "--xre-path=../hostutils/xre",
+                "--manifest=xpcshell/tests/xpcshell.ini",
+                "--build-info-json=xpcshell/mozinfo.json",
+                "--testing-modules-dir=modules",
+                "--local-lib-dir=../fennec",
+                "--apk=../%(apk_name)s",
+                "--no-logfiles",
+                "--symbols-path=%(symbols_path)s",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "remotexpcshelltests.py",
+            "testsdir": "xpcshell"
+        }
+    }
+}
\ No newline at end of file
--- a/testing/config/mozharness/b2g_desktop_config.py
+++ b/testing/config/mozharness/b2g_desktop_config.py
@@ -1,21 +1,41 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 config = {
-    "mochitest_options": [
-        "--console-level=INFO", "%(test_manifest)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "--profile=%(gaia_profile)s", "--app=%(application)s", "--desktop",
-        "--utility-path=%(utility_path)s", "--certificate-path=%(cert_path)s",
-        "--symbols-path=%(symbols_path)s", "--browser-arg=%(browser_arg)s",
-        "--quiet", "--log-raw=%(raw_log_file)s"
-    ],
-
-    "reftest_options": [
-        "--desktop", "--profile=%(gaia_profile)s", "--appname=%(application)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "--browser-arg=%(browser_arg)s", "--symbols-path=%(symbols_path)s",
-        "%(test_manifest)s"
-    ]
-}
+    "suite_definitions": {
+        "mochitest": {
+            "options": [
+                "--console-level=INFO",
+                "%(test_manifest)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s",
+                "--profile=%(gaia_profile)s",
+                "--app=%(application)s",
+                "--desktop",
+                "--utility-path=%(utility_path)s",
+                "--certificate-path=%(cert_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--browser-arg=%(browser_arg)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "runtestsb2g.py",
+            "testsdir": "mochitest"
+        },
+        "reftest": {
+            "options": [
+                "--desktop",
+                "--profile=%(gaia_profile)s",
+                "--appname=%(application)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s",
+                "--browser-arg=%(browser_arg)s",
+                "--symbols-path=%(symbols_path)s",
+                "%(test_manifest)s"
+            ],
+            "run_filename": "runreftestsb2g.py",
+            "testsdir": "reftest"
+        }
+    }
+}
\ No newline at end of file
--- a/testing/config/mozharness/b2g_emulator_config.py
+++ b/testing/config/mozharness/b2g_emulator_config.py
@@ -1,62 +1,121 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 config = {
-    "jsreftest_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
-        "--emulator-res=800x1000", "--logdir=%(logcat_dir)s",
-        "--remote-webserver=%(remote_webserver)s", "--ignore-window-size",
-        "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "--extra-profile-file=jsreftest/tests/user.js",
-        "jsreftest/tests/jstests.list",
-    ],
-
-    "mochitest_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--console-level=INFO",
-        "--emulator=%(emulator)s", "--logdir=%(logcat_dir)s",
-        "--remote-webserver=%(remote_webserver)s", "%(test_manifest)s",
-        "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "--quiet", "--log-raw=%(raw_log_file)s",
-        "--certificate-path=%(certificate_path)s",
-        "--test-path=%(test_path)s",
-    ],
-
-    "reftest_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
-        "--emulator-res=800x1000", "--logdir=%(logcat_dir)s",
-        "--remote-webserver=%(remote_webserver)s", "--ignore-window-size",
-        "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s", "--enable-oop",
-        "tests/layout/reftests/reftest.list",
-    ],
-
-    "crashtest_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
-        "--emulator-res=800x1000", "--logdir=%(logcat_dir)s",
-        "--remote-webserver=%(remote_webserver)s", "--ignore-window-size",
-        "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "tests/testing/crashtest/crashtests.list",
-    ],
-
-    "xpcshell_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
-        "--logdir=%(logcat_dir)s", "--manifest=tests/xpcshell.ini", "--use-device-libs",
-        "--testing-modules-dir=%(modules_dir)s", "--symbols-path=%(symbols_path)s",
-        "--busybox=%(busybox)s", "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "--log-raw=%(raw_log_file)s",
-    ],
-
-    "cppunittest_options": [
-        "--dm_trans=adb",
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=%(xre_path)s",
-        "--addEnv", "LD_LIBRARY_PATH=/vendor/lib:/system/lib:/system/b2g",
-        "--with-b2g-emulator=%(b2gpath)s",
-        "--skip-manifest=b2g_cppunittest_manifest.txt",
-        "."
-    ],
-}
+    "suite_definitions": {
+        "cppunittest": {
+            "options": [
+                "--dm_trans=adb",
+                "--symbols-path=%(symbols_path)s",
+                "--xre-path=%(xre_path)s",
+                "--addEnv",
+                "LD_LIBRARY_PATH=/vendor/lib:/system/lib:/system/b2g",
+                "--with-b2g-emulator=%(b2gpath)s",
+                "--skip-manifest=b2g_cppunittest_manifest.txt",
+                "."
+            ],
+            "run_filename": "remotecppunittests.py",
+            "testsdir": "cppunittests"
+        },
+        "crashtest": {
+            "options": [
+                "--adbpath=%(adbpath)s",
+                "--b2gpath=%(b2gpath)s",
+                "--emulator=%(emulator)s",
+                "--emulator-res=800x1000",
+                "--logdir=%(logcat_dir)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--ignore-window-size",
+                "--xre-path=%(xre_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--busybox=%(busybox)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s",
+                "tests/testing/crashtest/crashtests.list"
+            ],
+            "run_filename": "runreftestb2g.py",
+            "testsdir": "reftest"
+        },
+        "jsreftest": {
+            "options": [
+                "--adbpath=%(adbpath)s",
+                "--b2gpath=%(b2gpath)s",
+                "--emulator=%(emulator)s",
+                "--emulator-res=800x1000",
+                "--logdir=%(logcat_dir)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--ignore-window-size",
+                "--xre-path=%(xre_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--busybox=%(busybox)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s",
+                "--extra-profile-file=jsreftest/tests/user.js",
+                "jsreftest/tests/jstests.list"
+            ],
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest"
+        },
+        "mochitest": {
+            "options": [
+                "--adbpath=%(adbpath)s",
+                "--b2gpath=%(b2gpath)s",
+                "--console-level=INFO",
+                "--emulator=%(emulator)s",
+                "--logdir=%(logcat_dir)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "%(test_manifest)s",
+                "--xre-path=%(xre_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--busybox=%(busybox)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--certificate-path=%(certificate_path)s",
+                "--test-path=%(test_path)s"
+            ],
+            "run_filename": "runtestsb2g.py",
+            "testsdir": "mochitest"
+        },
+        "reftest": {
+            "options": [
+                "--adbpath=%(adbpath)s",
+                "--b2gpath=%(b2gpath)s",
+                "--emulator=%(emulator)s",
+                "--emulator-res=800x1000",
+                "--logdir=%(logcat_dir)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--ignore-window-size",
+                "--xre-path=%(xre_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--busybox=%(busybox)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s",
+                "--enable-oop",
+                "tests/layout/reftests/reftest.list"
+            ],
+            "run_filename": "runreftestsb2g.py",
+            "testsdir": "reftest"
+        },
+        "xpcshell": {
+            "options": [
+                "--adbpath=%(adbpath)s",
+                "--b2gpath=%(b2gpath)s",
+                "--emulator=%(emulator)s",
+                "--logdir=%(logcat_dir)s",
+                "--manifest=tests/xpcshell.ini",
+                "--use-device-libs",
+                "--testing-modules-dir=%(modules_dir)s",
+                "--symbols-path=%(symbols_path)s",
+                "--busybox=%(busybox)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "runtestsb2g.py",
+            "testsdir": "xpcshell"
+        }
+    }
+}
\ No newline at end of file
--- a/testing/config/mozharness/linux_config.py
+++ b/testing/config/mozharness/linux_config.py
@@ -1,44 +1,88 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 config = {
-    "reftest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
-    ],
-    "mochitest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO", "--setpref=webgl.force-enabled=true",
-        "--quiet", "--log-raw=%(raw_log_file)s",
-        "--use-test-media-devices"
-    ],
-    "webapprt_options": [
-        "--app=%(app_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO", "--testing-modules-dir=tests/modules",
-        "--quiet"
-    ],
-    "xpcshell_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--test-plugin-path=%(test_plugin_path)s",
-        "--log-raw=%(raw_log_file)s",
-    ],
-    "cppunittest_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=%(abs_app_dir)s"
-    ],
-    "jittest_options": [
-        "tests/bin/js",
-        "--no-slow",
-        "--no-progress",
-        "--tinderbox",
-        "--tbpl"
-    ],
-    "mozbase_options": [
-        "-b", "%(binary_path)s"
-    ],
-}
+    "suite_definitions": {
+        "cppunittest": {
+            "options": [
+                "--symbols-path=%(symbols_path)s",
+                "--xre-path=%(abs_app_dir)s"
+            ],
+            "run_filename": "runcppunittests.py",
+            "testsdir": "cppunittests"
+        },
+        "jittest": {
+            "options": [
+                "tests/bin/js",
+                "--no-slow",
+                "--no-progress",
+                "--tinderbox",
+                "--tbpl"
+            ],
+            "run_filename": "jit_test.py",
+            "testsdir": "jit-test/jit-test"
+        },
+        "mochitest": {
+            "options": [
+                "--appname=%(binary_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s",
+                "--certificate-path=tests/certs",
+                "--autorun",
+                "--close-when-done",
+                "--console-level=INFO",
+                "--setpref=webgl.force-enabled=true",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--use-test-media-devices"
+            ],
+            "run_filename": "runtests.py",
+            "testsdir": "mochitest"
+        },
+        "mozbase": {
+            "options": [
+                "-b",
+                "%(binary_path)s"
+            ],
+            "run_filename": "test.py",
+            "testsdir": "mozbase"
+        },
+        "reftest": {
+            "options": [
+                "--appname=%(binary_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s"
+            ],
+            "run_filename": "runreftest.py",
+            "testsdir": "reftest"
+        },
+        "webapprt": {
+            "options": [
+                "--app=%(app_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s",
+                "--certificate-path=tests/certs",
+                "--autorun",
+                "--close-when-done",
+                "--console-level=INFO",
+                "--testing-modules-dir=tests/modules",
+                "--quiet"
+            ],
+            "run_filename": "runtests.py",
+            "testsdir": "mochitest"
+        },
+        "xpcshell": {
+            "options": [
+                "--symbols-path=%(symbols_path)s",
+                "--test-plugin-path=%(test_plugin_path)s",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "runxpcshelltests.py",
+            "testsdir": "xpcshell"
+        }
+    }
+}
\ No newline at end of file
--- a/testing/config/mozharness/mac_config.py
+++ b/testing/config/mozharness/mac_config.py
@@ -1,43 +1,86 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 config = {
-    "reftest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
-    ],
-    "mochitest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO",
-        "--quiet", "--log-raw=%(raw_log_file)s"
-    ],
-    "webapprt_options": [
-        "--app=%(app_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO", "--testing-modules-dir=tests/modules",
-        "--quiet"
-    ],
-    "xpcshell_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--test-plugin-path=%(test_plugin_path)s",
-        "--log-raw=%(raw_log_file)s",
-    ],
-    "cppunittest_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=%(abs_app_dir)s"
-    ],
-    "jittest_options": [
-        "tests/bin/js",
-        "--no-slow",
-        "--no-progress",
-        "--tinderbox",
-        "--tbpl"
-    ],
-    "mozbase_options": [
-        "-b", "%(binary_path)s"
-    ],
-}
+    "suite_definitions": {
+        "cppunittest": {
+            "options": [
+                "--symbols-path=%(symbols_path)s",
+                "--xre-path=%(abs_app_dir)s"
+            ],
+            "run_filename": "runcppunittests.py",
+            "testsdir": "cppunittests"
+        },
+        "jittest": {
+            "options": [
+                "tests/bin/js",
+                "--no-slow",
+                "--no-progress",
+                "--tinderbox",
+                "--tbpl"
+            ],
+            "run_filename": "jit_test.py",
+            "testsdir": "jit-test/jit-test"
+        },
+        "mochitest": {
+            "options": [
+                "--appname=%(binary_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s",
+                "--certificate-path=tests/certs",
+                "--autorun",
+                "--close-when-done",
+                "--console-level=INFO",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "runtests.py",
+            "testsdir": "mochitest"
+        },
+        "mozbase": {
+            "options": [
+                "-b",
+                "%(binary_path)s"
+            ],
+            "run_filename": "test.py",
+            "testsdir": "mozbase"
+        },
+        "reftest": {
+            "options": [
+                "--appname=%(binary_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s"
+            ],
+            "run_filename": "runreftest.py",
+            "testsdir": "reftest"
+        },
+        "webapprt": {
+            "options": [
+                "--app=%(app_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s",
+                "--certificate-path=tests/certs",
+                "--autorun",
+                "--close-when-done",
+                "--console-level=INFO",
+                "--testing-modules-dir=tests/modules",
+                "--quiet"
+            ],
+            "run_filename": "runtests.py",
+            "testsdir": "mochitest"
+        },
+        "xpcshell": {
+            "options": [
+                "--symbols-path=%(symbols_path)s",
+                "--test-plugin-path=%(test_plugin_path)s",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "runxpcshelltests.py",
+            "testsdir": "xpcshell"
+        }
+    }
+}
\ No newline at end of file
--- a/testing/config/mozharness/marionette.py
+++ b/testing/config/mozharness/marionette.py
@@ -1,62 +1,86 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 config = {
-    "marionette_desktop_options": [
-        "--type=%(type)s",
-        "--log-raw=%(raw_log_file)s",
-        "--binary=%(binary)s",
-        "--address=%(address)s",
-        "--symbols-path=%(symbols_path)s",
-    ],
-    "marionette_emulator_options": [
-        "--type=%(type)s",
-        "--log-raw=%(raw_log_file)s",
-        "--logcat-dir=%(logcat_dir)s",
-        "--emulator=%(emulator)s",
-        "--homedir=%(homedir)s",
-        "--symbols-path=%(symbols_path)s",
-    ],
-    "webapi_emulator_options": [
-        "--type=%(type)s",
-        "--log-raw=%(raw_log_file)s",
-        "--symbols-path=%(symbols_path)s",
-        "--logcat-dir=%(logcat_dir)s",
-        "--emulator=%(emulator)s",
-        "--homedir=%(homedir)s",
-    ],
-    # This combination is not currently run.
-    "webapi_desktop_options": [
-    ],
-    "gaiatest_emulator_options": [
-        "--restart",
-        "--timeout=%(timeout)s",
-        "--type=%(type)s",
-        "--testvars=%(testvars)s",
-        "--profile=%(profile)s",
-        "--symbols-path=%(symbols_path)s",
-        "--xml-output=%(xml_output)s",
-        "--html-output=%(html_output)s",
-        "--log-raw=%(raw_log_file)s",
-        "--logcat-dir=%(logcat_dir)s",
-        "--emulator=%(emulator)s",
-        "--homedir=%(homedir)s",
-    ],
-    "gaiatest_desktop_options": [
-        "--restart",
-        "--timeout=%(timeout)s",
-        "--type=%(type)s",
-        "--testvars=%(testvars)s",
-        "--profile=%(profile)s",
-        "--symbols-path=%(symbols_path)s",
-        "--gecko-log=%(gecko_log)s",
-        "--xml-output=%(xml_output)s",
-        "--html-output=%(html_output)s",
-        "--log-raw=%(raw_log_file)s",
-        "--binary=%(binary)s",
-        "--address=%(address)s",
-        "--total-chunks=%(total_chunks)s",
-        "--this-chunk=%(this_chunk)s",
-    ],
-}
+    "suite_definitions": {
+        "gaiatest_desktop": {
+            "options": [
+                "--restart",
+                "--timeout=%(timeout)s",
+                "--type=%(type)s",
+                "--testvars=%(testvars)s",
+                "--profile=%(profile)s",
+                "--symbols-path=%(symbols_path)s",
+                "--gecko-log=%(gecko_log)s",
+                "--xml-output=%(xml_output)s",
+                "--html-output=%(html_output)s",
+                "--log-raw=%(raw_log_file)s",
+                "--binary=%(binary)s",
+                "--address=%(address)s",
+                "--total-chunks=%(total_chunks)s",
+                "--this-chunk=%(this_chunk)s"
+            ],
+            "run_filename": "",
+            "testsdir": ""
+        },
+        "gaiatest_emulator": {
+            "options": [
+                "--restart",
+                "--timeout=%(timeout)s",
+                "--type=%(type)s",
+                "--testvars=%(testvars)s",
+                "--profile=%(profile)s",
+                "--symbols-path=%(symbols_path)s",
+                "--xml-output=%(xml_output)s",
+                "--html-output=%(html_output)s",
+                "--log-raw=%(raw_log_file)s",
+                "--logcat-dir=%(logcat_dir)s",
+                "--emulator=%(emulator)s",
+                "--homedir=%(homedir)s"
+            ],
+            "run_filename": "",
+            "testsdir": ""
+        },
+        "marionette_desktop": {
+            "options": [
+                "--type=%(type)s",
+                "--log-raw=%(raw_log_file)s",
+                "--binary=%(binary)s",
+                "--address=%(address)s",
+                "--symbols-path=%(symbols_path)s"
+            ],
+            "run_filename": "",
+            "testsdir": ""
+        },
+        "marionette_emulator": {
+            "options": [
+                "--type=%(type)s",
+                "--log-raw=%(raw_log_file)s",
+                "--logcat-dir=%(logcat_dir)s",
+                "--emulator=%(emulator)s",
+                "--homedir=%(homedir)s",
+                "--symbols-path=%(symbols_path)s"
+            ],
+            "run_filename": "",
+            "testsdir": ""
+        },
+        "webapi_desktop": {
+            "options": [],
+            "run_filename": "",
+            "testsdir": ""
+        },
+        "webapi_emulator": {
+            "options": [
+                "--type=%(type)s",
+                "--log-raw=%(raw_log_file)s",
+                "--symbols-path=%(symbols_path)s",
+                "--logcat-dir=%(logcat_dir)s",
+                "--emulator=%(emulator)s",
+                "--homedir=%(homedir)s"
+            ],
+            "run_filename": "",
+            "testsdir": ""
+        }
+    }
+}
\ No newline at end of file
--- a/testing/config/mozharness/windows_config.py
+++ b/testing/config/mozharness/windows_config.py
@@ -1,43 +1,86 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 config = {
-    "reftest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
-    ],
-    "mochitest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO",
-        "--quiet", "--log-raw=%(raw_log_file)s"
-    ],
-    "webapprt_options": [
-        "--app=%(app_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO", "--testing-modules-dir=tests/modules",
-        "--quiet"
-    ],
-    "xpcshell_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--test-plugin-path=%(test_plugin_path)s",
-        "--log-raw=%(raw_log_file)s",
-    ],
-    "cppunittest_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=%(abs_app_dir)s"
-    ],
-    "jittest_options": [
-        "tests/bin/js",
-        "--no-slow",
-        "--no-progress",
-        "--tinderbox",
-        "--tbpl"
-    ],
-    "mozbase_options": [
-        "-b", "%(binary_path)s"
-    ],
-}
+    "suite_definitions": {
+        "cppunittest": {
+            "options": [
+                "--symbols-path=%(symbols_path)s",
+                "--xre-path=%(abs_app_dir)s"
+            ],
+            "run_filename": "runcppunittests.py",
+            "testsdir": "cppunittests"
+        },
+        "jittest": {
+            "options": [
+                "tests/bin/js",
+                "--no-slow",
+                "--no-progress",
+                "--tinderbox",
+                "--tbpl"
+            ],
+            "run_filename": "jit_test.py",
+            "testsdir": "jit-test/jit-test"
+        },
+        "mochitest": {
+            "options": [
+                "--appname=%(binary_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s",
+                "--certificate-path=tests/certs",
+                "--autorun",
+                "--close-when-done",
+                "--console-level=INFO",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "runtests.py",
+            "testsdir": "mochitest"
+        },
+        "mozbase": {
+            "options": [
+                "-b",
+                "%(binary_path)s"
+            ],
+            "run_filename": "test.py",
+            "testsdir": "mozbase"
+        },
+        "reftest": {
+            "options": [
+                "--appname=%(binary_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s"
+            ],
+            "run_filename": "runreftest.py",
+            "testsdir": "reftest"
+        },
+        "webapprt": {
+            "options": [
+                "--app=%(app_path)s",
+                "--utility-path=tests/bin",
+                "--extra-profile-file=tests/bin/plugins",
+                "--symbols-path=%(symbols_path)s",
+                "--certificate-path=tests/certs",
+                "--autorun",
+                "--close-when-done",
+                "--console-level=INFO",
+                "--testing-modules-dir=tests/modules",
+                "--quiet"
+            ],
+            "run_filename": "runtests.py",
+            "testsdir": "mochitest"
+        },
+        "xpcshell": {
+            "options": [
+                "--symbols-path=%(symbols_path)s",
+                "--test-plugin-path=%(test_plugin_path)s",
+                "--log-raw=%(raw_log_file)s"
+            ],
+            "run_filename": "runxpcshelltests.py",
+            "testsdir": "xpcshell"
+        }
+    }
+}
\ No newline at end of file