Bug 981030 - Use in-tree configs for test options, r=aki
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 24 Mar 2014 09:29:33 -0400
changeset 2567 7f85fd954b6ec5e0d17b3049d724a804e61216f6
parent 2566 18d6f07a6f6f5d577d387bdd2a2f2183c5a52901
child 2568 68b045e5e0ee561428654b970e7c32bcbc0fee10
push id1876
push userahalberstadt@mozilla.com
push dateMon, 24 Mar 2014 13:32:58 +0000
reviewersaki
bugs981030
Bug 981030 - Use in-tree configs for test options, r=aki
configs/android/android_panda_releng.py
configs/android/androidarm.py
configs/android/androidx86.py
configs/b2g/desktop_automation_config.py
configs/b2g/emulator_automation_config.py
configs/unittests/linux_unittest.py
configs/unittests/mac_unittest.py
configs/unittests/win_unittest.py
mozharness/mozilla/testing/testbase.py
scripts/android_emulator_unittest.py
scripts/android_panda.py
scripts/b2g_desktop_unittest.py
scripts/b2g_emulator_unittest.py
scripts/desktop_unittest.py
--- a/configs/android/android_panda_releng.py
+++ b/configs/android/android_panda_releng.py
@@ -19,96 +19,18 @@ config = {
         "xpcshell": "remotexpcshelltests.py",
         "jittest": "jit_test.py",
         "cppunittest": "remotecppunittests.py"
     },
     "hostutils_url":  "http://bm-remote.build.mozilla.org/tegra/tegra-host-utils.Linux.742597.zip",
     "verify_path":  "/builds/sut_tools/verify.py",
     "install_app_path":  "/builds/sut_tools/installApp.py",
     "logcat_path":  "/builds/sut_tools/logcat.py",
-    "mochitest_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin", "--certificate-path=certs",
-        "--app=%(app_name)s", "--console-level=INFO",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--run-only-tests=android.json", "--symbols-path=%(symbols_path)s"
-    ],
-    "reftest_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin",
-        "--app=%(app_name)s",
-        "--ignore-window-size", "--bootstrap",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--symbols-path=%(symbols_path)s",
-        "reftest/tests/layout/reftests/reftest.list"
-    ],
-    "crashtest_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin",
-        "--app=%(app_name)s",
-        "--enable-privilege", "--ignore-window-size", "--bootstrap",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--symbols-path=%(symbols_path)s",
-        "reftest/tests/testing/crashtest/crashtests.list"
-    ],
-    "jsreftest_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin",
-        "--app=%(app_name)s",
-        "--enable-privilege", "--ignore-window-size", "--bootstrap",
-        "--extra-profile-file=jsreftest/tests/user.js", "jsreftest/tests/jstests.list",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--symbols-path=%(symbols_path)s"
-    ],
-    "robocop_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--utility-path=../hostutils/bin",
-        "--certificate-path=certs",
-        "--app=%(app_name)s", "--console-level=INFO",
-        "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-        "--symbols-path=%(symbols_path)s",
-        "--robocop=mochitest/robocop.ini"
-    ],
-    "xpcshell_options": [
-        "--deviceIP=%(device_ip)s",
-        "--xre-path=../hostutils/xre",
-        "--manifest=xpcshell/tests/xpcshell_android.ini",
-        "--build-info-json=xpcshell/mozinfo.json",
-        "--testing-modules-dir=modules",
-        "--local-lib-dir=../fennec",
-        "--apk=../%(apk_name)s",
-        "--no-logfiles",
-        "--symbols-path=%(symbols_path)s"
-    ],
-    "jittest_options": [
-        "bin/js",
-        "--remote",
-        "-j", "1",
-        "--deviceTransport=sut",
-        "--deviceIP=%(device_ip)s",
-        "--localLib=../tests/bin",
-        "--no-slow",
-        "--no-progress",
-        "--tinderbox",
-        "--tbpl"
-     ],
-     "cppunittest_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=tests/bin",
-        "--dm_trans=SUT",
-        "--deviceIP=%(device_ip)s",
-        "--localBinDir=../tests/bin",
-        "--apk=%(apk_path)s",
-        "--skip-manifest=../tests/cppunittests/android_cppunittest_manifest.txt"
-     ],
+    # test harness options are located in the gecko tree
+    "in_tree_config": "config/mozharness/android_panda_config.py",
     "all_mochitest_suites": {
         "mochitest-1": ["--total-chunks=8", "--this-chunk=1"],
         "mochitest-2": ["--total-chunks=8", "--this-chunk=2"],
         "mochitest-3": ["--total-chunks=8", "--this-chunk=3"],
         "mochitest-4": ["--total-chunks=8", "--this-chunk=4"],
         "mochitest-5": ["--total-chunks=8", "--this-chunk=5"],
         "mochitest-6": ["--total-chunks=8", "--this-chunk=6"],
         "mochitest-7": ["--total-chunks=8", "--this-chunk=7"],
--- a/configs/android/androidarm.py
+++ b/configs/android/androidarm.py
@@ -192,45 +192,16 @@ config = {
                 "--robocop-ids=fennec_ids.txt", "--robocop=robocop.ini"],
         },
         "robocop-4": {
             "category": "mochitest",
             "extra_args": ["--total-chunks", "4", "--this-chunk", "4", "--robocop-path=../..",
                 "--robocop-ids=fennec_ids.txt", "--robocop=robocop.ini"],
         },
     }, # end of "test_definitions"
-    "suite_definitions": {
-        "mochitest": {
-            "run_filename": "runtestsremote.py",
-            "options": ["--autorun", "--close-when-done", "--dm_trans=sut",
-                "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
-                "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
-                "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s",
-                "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-                "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s"
-            ],
-        },
-        "reftest": {
-            "run_filename": "remotereftest.py",
-            "options": [ "--app=%(app)s", "--ignore-window-size",
-                "--bootstrap", "--enable-privilege",
-                "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s",
-                "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s",
-                "--devicePort=%(device_port)s", "--http-port=%(http_port)s",
-                "--ssl-port=%(ssl_port)s", "--httpd-path", "reftest/components",
-                "--symbols-path=%(symbols_path)s",
-            ],
-        },
-        "xpcshell": {
-            "run_filename": "remotexpcshelltests.py",
-            "options": ["--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s",
-                "--xre-path=%(xre_path)s", "--testing-modules-dir=%(modules_dir)s",
-                "--apk=%(installer_path)s", "--no-logfiles",
-                "--symbols-path=%(symbols_path)s",
-            ],
-        },
-    }, # end of "suite_definitions"
+    # test harness options are located in the gecko tree
+    "in_tree_config": "config/mozharness/android_arm_config.py",
     "download_minidump_stackwalk": True,
     "default_blob_upload_servers": [
          "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
 }
--- a/configs/android/androidx86.py
+++ b/configs/android/androidx86.py
@@ -126,45 +126,16 @@ config = {
                 "--robocop-ids=fennec_ids.txt", "--robocop=robocop.ini"],
         },
         "robocop-3": {
             "category": "mochitest",
             "extra_args": ["--total-chunks", "3", "--this-chunk", "3", "--robocop-path=../..",
                 "--robocop-ids=fennec_ids.txt", "--robocop=robocop.ini"],
         },
     }, # end of "test_definitions"
-    "suite_definitions": {
-        "mochitest": {
-            "run_filename": "runtestsremote.py",
-            "options": ["--autorun", "--close-when-done", "--dm_trans=sut",
-                "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
-                "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
-                "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s",
-                "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
-                "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s"
-            ],
-        },
-        "reftest": {
-            "run_filename": "remotereftest.py",
-            "options": [ "--app=%(app)s", "--ignore-window-size",
-                "--bootstrap", "--enable-privilege",
-                "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s",
-                "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s",
-                "--devicePort=%(device_port)s", "--http-port=%(http_port)s",
-                "--ssl-port=%(ssl_port)s", "--httpd-path", "reftest/components",
-                "--symbols-path=%(symbols_path)s",
-            ],
-        },
-        "xpcshell": {
-            "run_filename": "remotexpcshelltests.py",
-            "options": ["--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s",
-                "--xre-path=%(xre_path)s", "--testing-modules-dir=%(modules_dir)s",
-                "--apk=%(installer_path)s", "--no-logfiles",
-                "--symbols-path=%(symbols_path)s",
-            ],
-        },
-    }, # end of "suite_definitions"
+    # test harness options are located in the gecko tree
+    "in_tree_config": "config/mozharness/android_x86_config.py",
     "download_minidump_stackwalk": True,
     "default_blob_upload_servers": [
          "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
 }
--- a/configs/b2g/desktop_automation_config.py
+++ b/configs/b2g/desktop_automation_config.py
@@ -30,27 +30,15 @@ config = {
     ],
     "download_symbols": "ondemand",
     "download_minidump_stackwalk": True,
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
 
-    # test harness options
     "run_file_names": {
         "mochitest": "runtestsb2g.py",
         "reftest": "runreftestb2g.py",
     },
-
-    "mochitest_options": [
-        "--console-level=INFO", "%(test_manifest)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "--profile=%(gaia_profile)s", "--app=%(application)s", "--desktop",
-        "--utility-path=%(utility_path)s", "--certificate-path=%(cert_path)s",
-        "--symbols-path=%(symbols_path)s",
-    ],
-
-    "reftest_options": [
-        "--desktop", "--profile=%(gaia_profile)s", "--appname=%(application)s",
-        "--symbols-path=%(symbols_path)s", "%(test_manifest)s",
-    ]
+    # test harness options are located in the gecko tree
+    "in_tree_config": "config/mozharness/b2g_desktop_config.py",
 }
--- a/configs/b2g/emulator_automation_config.py
+++ b/configs/b2g/emulator_automation_config.py
@@ -32,61 +32,19 @@ config = {
     ],
     "download_symbols": "ondemand",
     "download_minidump_stackwalk": True,
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
 
-    # test harness options
     "run_file_names": {
         "jsreftest": "runreftestb2g.py",
         "mochitest": "runtestsb2g.py",
         "reftest": "runreftestb2g.py",
         "crashtest": "runreftestb2g.py",
         "xpcshell": "runtestsb2g.py"
     },
-
-    "jsreftest_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
-        "--emulator-res=800x1000", "--logcat-dir=%(logcat_dir)s",
-        "--remote-webserver=%(remote_webserver)s", "--ignore-window-size",
-        "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "--extra-profile-file=jsreftest/tests/user.js",
-        "%(test_manifest)s",
-    ],
-
-    "mochitest_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--console-level=INFO",
-        "--emulator=%(emulator)s", "--logcat-dir=%(logcat_dir)s",
-        "--remote-webserver=%(remote_webserver)s", "%(test_manifest)s",
-        "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-    ],
-
-    "reftest_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
-        "--emulator-res=800x1000", "--logcat-dir=%(logcat_dir)s",
-        "--remote-webserver=%(remote_webserver)s", "--ignore-window-size",
-        "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "%(test_manifest)s",
-    ],
-
-    "crashtest_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
-        "--emulator-res=800x1000", "--logcat-dir=%(logcat_dir)s",
-        "--remote-webserver=%(remote_webserver)s", "--ignore-window-size",
-        "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
-        "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "%(test_manifest)s",
-    ],
-
-    "xpcshell_options": [
-        "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
-        "--logcat-dir=%(logcat_dir)s", "--manifest=%(test_manifest)s", "--use-device-libs",
-        "--testing-modules-dir=%(modules_dir)s", "--symbols-path=%(symbols_path)s",
-        "--busybox=%(busybox)s", "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-    ],
+    # test harness options are located in the gecko tree
+    "in_tree_config": "config/mozharness/b2g_emulator_config.py",
     "vcs_output_timeout": 1760,
 }
--- a/configs/unittests/linux_unittest.py
+++ b/configs/unittests/linux_unittest.py
@@ -42,44 +42,18 @@ config = {
     "specific_tests_zip_dirs": {
         "mochitest": ["mochitest/*"],
         "reftest": ["reftest/*", "jsreftest/*"],
         "xpcshell": ["xpcshell/*"],
         "cppunittest": ["cppunittests/*"],
         "jittest": ["jit-test/*"],
         "mozbase": ["mozbase/*"]
     },
-    "reftest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
-    ],
-    "mochitest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO", "--setpref=webgl.force-enabled=true"
-    ],
-    "xpcshell_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--test-plugin-path=%(test_plugin_path)s"
-    ],
-    "cppunittest_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=%(abs_app_dir)s"
-    ],
-    "jittest_options": [
-        "tests/bin/js",
-        "--no-slow",
-        "--no-progress",
-        "--tinderbox",
-        "--tbpl"
-    ],
-    "mozbase_options": [
-        "-b", "%(binary_path)s"
-    ],
+    # test harness options are located in the gecko tree
+    "in_tree_config": "config/mozharness/linux_config.py",
     #local mochi suites
     "all_mochitest_suites": {
         "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
         "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
         "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
         "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
         "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
         "chrome": ["--chrome"],
--- a/configs/unittests/mac_unittest.py
+++ b/configs/unittests/mac_unittest.py
@@ -33,44 +33,18 @@ config = {
     "specific_tests_zip_dirs": {
         "mochitest": ["mochitest/*"],
         "reftest": ["reftest/*", "jsreftest/*"],
         "xpcshell": ["xpcshell/*"],
         "cppunittest": ["cppunittests/*"],
         "jittest": ["jit-test/*"],
         "mozbase": ["mozbase/*"]
     },
-    "reftest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
-    ],
-    "mochitest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO"
-    ],
-    "xpcshell_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--test-plugin-path=%(test_plugin_path)s"
-    ],
-    "cppunittest_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=%(abs_app_dir)s"
-    ],
-    "jittest_options": [
-        "tests/bin/js",
-        "--no-slow",
-        "--no-progress",
-        "--tinderbox",
-        "--tbpl"
-    ],
-    "mozbase_options": [
-        "-b", "%(binary_path)s"
-    ],
+    # test harness options are located in the gecko tree
+    "in_tree_config": "config/mozharness/mac_config.py",
     #local mochi suites
     "all_mochitest_suites": {
         "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
         "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
         "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
         "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
         "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
         "chrome": ["--chrome"],
--- a/configs/unittests/win_unittest.py
+++ b/configs/unittests/win_unittest.py
@@ -42,44 +42,18 @@ config = {
     "specific_tests_zip_dirs": {
         "mochitest": ["mochitest/*"],
         "reftest": ["reftest/*", "jsreftest/*"],
         "xpcshell": ["xpcshell/*"],
         "cppunittest": ["cppunittests/*"],
         "jittest": ["jit-test/*"],
         "mozbase": ["mozbase/*"]
     },
-    "reftest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
-    ],
-    "mochitest_options": [
-        "--appname=%(binary_path)s", "--utility-path=tests/bin",
-        "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
-        "--certificate-path=tests/certs", "--autorun", "--close-when-done",
-        "--console-level=INFO"
-    ],
-    "xpcshell_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--test-plugin-path=%(test_plugin_path)s"
-    ],
-    "cppunittest_options": [
-        "--symbols-path=%(symbols_path)s",
-        "--xre-path=%(abs_app_dir)s"
-    ],
-    "jittest_options": [
-        "tests/bin/js",
-        "--no-slow",
-        "--no-progress",
-        "--tinderbox",
-        "--tbpl"
-    ],
-    "mozbase_options": [
-        "-b", "%(binary_path)s"
-    ],
+    # test harness options are located in the gecko tree
+    "in_tree_config": "config/mozharness/windows_config.py",
     #local mochi suites
     "all_mochitest_suites":
     {
         "plain1": ["--total-chunks=5", "--this-chunk=1", "--chunk-by-dir=4"],
         "plain2": ["--total-chunks=5", "--this-chunk=2", "--chunk-by-dir=4"],
         "plain3": ["--total-chunks=5", "--this-chunk=3", "--chunk-by-dir=4"],
         "plain4": ["--total-chunks=5", "--this-chunk=4", "--chunk-by-dir=4"],
         "plain5": ["--total-chunks=5", "--this-chunk=5", "--chunk-by-dir=4"],
--- a/mozharness/mozilla/testing/testbase.py
+++ b/mozharness/mozilla/testing/testbase.py
@@ -221,20 +221,33 @@ 2. running via buildbot and running the 
         self.run_command(unzip_cmd, cwd=test_install_dir,
                          halt_on_failure=True, success_codes=[0, 11])
 
     def _read_tree_config(self):
         """Reads an in-tree config file"""
         dirs = self.query_abs_dirs()
         test_install_dir = dirs.get('abs_test_install_dir',
                                     os.path.join(dirs['abs_work_dir'], 'tests'))
-        tree_config_path = os.path.join(test_install_dir, 'config', 'mozharness_config.py')
+
+        if 'in_tree_config' in self.config:
+            rel_tree_config_path = self.config['in_tree_config']
+            tree_config_path = os.path.join(test_install_dir, rel_tree_config_path)
 
-        if os.path.isfile(tree_config_path):
-            self.tree_config.update(parse_config_file(tree_config_path))
+            if not os.path.isfile(tree_config_path):
+                self.fatal("The in-tree configuration file '%s' does not exist!" \
+                           "It must be added to '%s'. See bug 981030 for more details." %
+                           (tree_config_path, os.path.join('gecko', 'testing', rel_tree_config_path)))
+
+            try:
+                self.tree_config.update(parse_config_file(tree_config_path))
+            except:
+                msg = "There was a problem parsing the in-tree configuration file '%s'!" % \
+                      os.path.join('gecko', 'testing', rel_tree_config_path)
+                self.exception(message=msg, level=FATAL)
+
             self.dump_config(file_path=os.path.join(dirs['abs_log_dir'], 'treeconfig.json'),
                              config=self.tree_config)
         self.tree_config.lock()
 
     def _download_installer(self):
         file_name = None
         if self.installer_path:
             file_name = self.installer_path
--- a/scripts/android_emulator_unittest.py
+++ b/scripts/android_emulator_unittest.py
@@ -369,22 +369,27 @@ class AndroidEmulatorTest(BlobUploadMixi
     def preflight_install(self):
         # in the base class, this checks for mozinstall, but we don't use it
         pass
 
     def _build_command(self, emulator, suite_name):
         c = self.config
         dirs = self.query_abs_dirs()
         suite_category = self.test_suite_definitions[suite_name]["category"]
+
+        if suite_category not in self.tree_config["suite_definitions"]:
+            self.fatal("Key '%s' not defined in the in-tree config! Please add it to '%s'." \
+                       "See bug 981030 for more details." % (suite_category,
+                       os.path.join('gecko', 'testing', self.config['in_tree_config'])))
         cmd = [
             self.query_python_path('python'),
             '-u',
             os.path.join(
                 dirs["abs_%s_dir" % suite_category],
-                c["suite_definitions"][suite_category]["run_filename"]
+                self.tree_config["suite_definitions"][suite_category]["run_filename"]
             ),
         ]
 
         str_format_values = {
             'app': self._query_package_name(),
             'remote_webserver': c['remote_webserver'],
             'xre_path': os.path.join(dirs['abs_xre_dir'], 'xre'),
             'utility_path':  os.path.join(dirs['abs_xre_dir'], 'bin'),
@@ -394,17 +399,17 @@ class AndroidEmulatorTest(BlobUploadMixi
             'ssl_port': emulator['ssl_port'],
             'certs_path': os.path.join(dirs['abs_work_dir'], 'tests/certs'),
             # TestingMixin._download_and_extract_symbols() will set
             # self.symbols_path when downloading/extracting.
             'symbols_path': self.symbols_path,
             'modules_dir': dirs['abs_modules_dir'],
             'installer_path': self.installer_path,
         }
-        for option in c["suite_definitions"][suite_category]["options"]:
+        for option in self.tree_config["suite_definitions"][suite_category]["options"]:
             cmd.extend([option % str_format_values])
         cmd.extend(self.test_suite_definitions[suite_name]["extra_args"])
 
         return cmd
 
     def _query_adb(self):
         return self.which('adb') or os.getenv('ADB_PATH')
 
--- a/scripts/android_panda.py
+++ b/scripts/android_panda.py
@@ -437,26 +437,26 @@ class PandaTest(TestingMixin, MercurialS
             'hostname': self.mozpool_device,
             'symbols_path': self._query_symbols_url(),
             'http_port': http_port,
             'ssl_port':  ssl_port,
             'app_name':  self.app_name,
             'apk_name':  self.filename_apk,
             'apk_path':  self.apk_path
         }
-        if self.config['%s_options' % suite_category]:
-            for option in self.config['%s_options' % suite_category]:
+        if self.tree_config['%s_options' % suite_category]:
+            for option in self.tree_config['%s_options' % suite_category]:
                 options.append(option % str_format_values)
             abs_base_cmd = base_cmd + options
             return abs_base_cmd
         else:
             self.warning("Suite options for %s could not be determined."
                          "\nIf you meant to have options for this suite, "
                          "please make sure they are specified in your "
-                         "config under %s_options" %
+                         "tree config under %s_options" %
                          (suite_category, suite_category))
 
     ###### helper methods
     def _pre_config_lock(self, rw_config):
         c = self.config
         if not c.get('run_all_suites'):
             return  # configs are valid
         for category in SUITE_CATEGORIES:
--- a/scripts/b2g_desktop_unittest.py
+++ b/scripts/b2g_desktop_unittest.py
@@ -161,18 +161,22 @@ class B2GDesktopTest(BlobUploadMixin, Te
                     manifest_param = '--test-manifest'
                 manifest_param = '%s=%s' % (manifest_param, self.test_manifest)
             else:
                 self.log('Ignoring non-existent manifest \'%s\'.' % self.test_manifest,
                          level=WARNING)
                 manifest_param = ''
             str_format_values['test_manifest'] = manifest_param
 
-        name = '%s_options' % suite
-        options = self.tree_config.get(name, self.config.get(name))
+        suite_options = '%s_options' % suite
+        if suite_options not in self.tree_config:
+            self.fatal("Key '%s' not defined in the in-tree config! Please add it to '%s'." \
+                       "See bug 981030 for more details." % (suite_options,
+                       os.path.join('gecko', 'testing', self.config['in_tree_config'])))
+        options = self.tree_config[suite_options]
         if options:
             for option in options:
                 option = option % str_format_values
                 if not option.endswith('None'):
                     cmd.append(option)
         return cmd
 
     def preflight_run_tests(self):
--- a/scripts/b2g_emulator_unittest.py
+++ b/scripts/b2g_emulator_unittest.py
@@ -275,18 +275,22 @@ class B2GEmulatorTest(TestingMixin, Tool
                     manifest_param = '--test-manifest'
                 manifest_param = '%s=%s' % (manifest_param, self.test_manifest)
             else:
                 self.log('Ignoring non-existent manifest \'%s\'.' % self.test_manifest,
                          level=WARNING)
                 manifest_param = ''
             str_format_values['test_manifest'] = manifest_param
 
-        name = '%s_options' % suite
-        options = self.tree_config.get(name, self.config.get(name))
+        suite_options = '%s_options' % suite
+        if suite_options not in self.tree_config:
+            self.fatal("Key '%s' not defined in the in-tree config! Please add it to '%s'." \
+                       "See bug 981030 for more details." % (suite_options,
+                       os.path.join('gecko', 'testing', self.config['in_tree_config'])))
+        options = self.tree_config[suite_options]
         if options:
             for option in options:
                 option = option % str_format_values
                 if not option.endswith('None'):
                     cmd.append(option)
         return cmd
 
     def _query_adb(self):
--- a/scripts/desktop_unittest.py
+++ b/scripts/desktop_unittest.py
@@ -233,18 +233,22 @@ class DesktopUnittest(TestingMixin, Merc
             # self.symbols_path when downloading/extracting.
             if self.symbols_path:
                 str_format_values['symbols_path'] = self.symbols_path
 
             # set pluginsPath
             abs_app_plugins_dir = os.path.join(abs_app_dir, 'plugins')
             str_format_values['test_plugin_path'] = abs_app_plugins_dir
 
-            name = '%s_options' % suite_category
-            options = list(self.tree_config.get(name, c.get(name)))
+            suite_options = '%s_options' % suite_category
+            if suite_options not in self.tree_config:
+                self.fatal("Key '%s' not defined in the in-tree config! Please add it to '%s'." \
+                           "See bug 981030 for more details." % (suite_options,
+                           os.path.join('gecko', 'testing', self.config['in_tree_config'])))
+            options = list(self.tree_config[suite_options])
             if options:
                 for i, option in enumerate(options):
                     options[i] = option % str_format_values
                 abs_base_cmd = base_cmd + options
                 return abs_base_cmd
             else:
                 self.warning("Suite options for %s could not be determined."
                              "\nIf you meant to have options for this suite, "