Bug 1238469 - Part 2: Refactor b2g mulet tests to remove the usage of custom mozharness configs; r=ahal draft
authorEdgar Chen <echen@mozilla.com>
Tue, 19 Jan 2016 18:52:06 +0800
changeset 322647 d75cb0f355776770df2f6d250d7f55828959c52b
parent 322646 d7fd4bdd24cb97015803ba7d6481da8e39bb58ef
child 322648 994c43c57c33f4576aa0ea86ad6810623d154e1c
push id9645
push userechen@mozilla.com
push dateTue, 19 Jan 2016 10:55:29 +0000
reviewersahal
bugs1238469
milestone46.0a1
Bug 1238469 - Part 2: Refactor b2g mulet tests to remove the usage of custom mozharness configs; r=ahal
testing/mozharness/configs/b2g/desktop_automation_config.py
testing/mozharness/configs/b2g/gaia_integration_config.py
testing/mozharness/configs/b2g/gaia_unit_production_config.py
testing/mozharness/configs/b2g/generic_config.py
testing/mozharness/configs/b2g/mulet_config.py
testing/mozharness/configs/b2g/taskcluster_gaia_integration.py
testing/mozharness/configs/b2g/taskcluster_gaia_unit_production.py
testing/mozharness/configs/b2g/taskcluster_mulet_automation.py
testing/taskcluster/tasks/tests/b2g_gaia_unit_oop.yml
testing/taskcluster/tasks/tests/mulet_build_test.yml
testing/taskcluster/tasks/tests/mulet_build_unit.yml
testing/taskcluster/tasks/tests/mulet_gaia_js_integration_tests.yml
testing/taskcluster/tasks/tests/mulet_gaia_unit.yml
testing/taskcluster/tasks/tests/mulet_gaia_unit_oop.yml
testing/taskcluster/tasks/tests/mulet_linter.yml
testing/taskcluster/tasks/tests/mulet_reftests.yml
deleted file mode 100644
--- a/testing/mozharness/configs/b2g/desktop_automation_config.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# This is a template config file for b2g desktop unittest production.
-import os
-
-config = {
-    # mozharness options
-    "application": "b2g",
-    "tooltool_cache": "/builds/tooltool_cache",
-
-    "exes": {
-        'python': '/tools/buildbot/bin/python',
-        'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
-        'tooltool.py': "/tools/tooltool.py",
-    },
-
-    "find_links": [
-        "http://pypi.pvt.build.mozilla.org/pub",
-        "http://pypi.pub.build.mozilla.org/pub",
-    ],
-    "pip_index": False,
-
-    "buildbot_json_path": "buildprops.json",
-
-    "default_actions": [
-        'clobber',
-        'read-buildbot-config',
-        'download-and-extract',
-        'create-virtualenv',
-        'install',
-        'run-tests',
-    ],
-    "download_symbols": "ondemand",
-    "download_minidump_stackwalk": True,
-    "default_blob_upload_servers": [
-        "https://blobupload.elasticbeanstalk.com",
-    ],
-    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
-
-    # testsuite options
-    "run_file_names": {
-        "mochitest": "runtestsb2g.py",
-        "reftest": "runreftestb2g.py",
-    },
-    "suite_definitions": {
-        "mochitest": {
-            "options": [
-                "--total-chunks=%(total_chunks)s",
-                "--this-chunk=%(this_chunk)s",
-                "--profile=%(gaia_profile)s",
-                "--app=%(application)s",
-                "--desktop",
-                "--utility-path=%(utility_path)s",
-                "--certificate-path=%(cert_path)s",
-                "--symbols-path=%(symbols_path)s",
-                "--browser-arg=%(browser_arg)s",
-                "--quiet",
-                "--log-raw=%(raw_log_file)s",
-                "--log-errorsummary=%(error_summary_file)s",
-                "--screenshot-on-fail",
-            ],
-            "run_filename": "runtestsb2g.py",
-            "testsdir": "mochitest"
-        },
-        "reftest": {
-            "options": [
-                "--desktop",
-                "--profile=%(gaia_profile)s",
-                "--appname=%(application)s",
-                "--total-chunks=%(total_chunks)s",
-                "--this-chunk=%(this_chunk)s",
-                "--browser-arg=%(browser_arg)s",
-                "--symbols-path=%(symbols_path)s",
-                "%(test_manifest)s"
-            ],
-            "run_filename": "runreftestsb2g.py",
-            "testsdir": "reftest"
-        }
-    },
-}
deleted file mode 100644
--- a/testing/mozharness/configs/b2g/generic_config.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# This is a config with generic releng related values 
-import os
-
-config = {
-    "exes": {
-        'python': '/tools/buildbot/bin/python',
-        'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
-        'tooltool.py': "/tools/tooltool.py",
-    },
-
-    "find_links": [
-        "http://pypi.pvt.build.mozilla.org/pub",
-        "http://pypi.pub.build.mozilla.org/pub",
-    ],
-    "pip_index": False,
-
-    "buildbot_json_path": "buildprops.json",
-
-    "download_symbols": "ondemand",
-    "download_minidump_stackwalk": True,
-    "default_blob_upload_servers": [
-        "https://blobupload.elasticbeanstalk.com",
-    ],
-    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
-}
rename from testing/mozharness/configs/b2g/gaia_integration_config.py
rename to testing/mozharness/configs/b2g/taskcluster_gaia_integration.py
--- a/testing/mozharness/configs/b2g/gaia_integration_config.py
+++ b/testing/mozharness/configs/b2g/taskcluster_gaia_integration.py
@@ -1,40 +1,28 @@
 # This is a template config file for b2g emulator unittest testing
 import platform
 
 HG_SHARE_BASE_DIR = "/builds/hg-shared"
 
-if platform.system().lower() == 'darwin':
-    xre_url = "https://api.pub.build.mozilla.org/tooltool/sha512/4d8d7a37d90c34a2a2fda3066a8fe85c189b183d05389cb957fc6fed31f10a6924e50c1b84488ff61c015293803f58a3aed5d4819374d04c8e0ee2b9e3997278"
-else:
-    xre_url = "https://api.pub.build.mozilla.org/tooltool/sha512/dc9503b21c87b5a469118746f99e4f41d73888972ce735fa10a80f6d218086da0e3da525d9a4cd8e4ea497ec199fef720e4a525873d77a1af304ac505e076462"
-
 config = {
     # mozharness script options
-    "xre_url": xre_url,
-
     "vcs_share_base": HG_SHARE_BASE_DIR,
-    "exes": {
-        'python': '/tools/buildbot/bin/python',
-        'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
-        'tooltool.py': "/tools/tooltool.py",
-    },
 
     "find_links": [
         "http://pypi.pvt.build.mozilla.org/pub",
         "http://pypi.pub.build.mozilla.org/pub",
     ],
     "pip_index": False,
 
-    "buildbot_json_path": "buildprops.json",
+    "download_symbols": "ondemand",
+    # We bake this directly into the tester image now...
+    "download_minidump_stackwalk": False,
 
     "default_actions": [
         'clobber',
-        'read-buildbot-config',
-        'pull',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-tests',
     ],
     "vcs_output_timeout": 1760,
 }
rename from testing/mozharness/configs/b2g/gaia_unit_production_config.py
rename to testing/mozharness/configs/b2g/taskcluster_gaia_unit_production.py
--- a/testing/mozharness/configs/b2g/gaia_unit_production_config.py
+++ b/testing/mozharness/configs/b2g/taskcluster_gaia_unit_production.py
@@ -1,45 +1,32 @@
 # This is a template config file for b2g emulator unittest testing
 import platform
 import os
 
 HG_SHARE_BASE_DIR = "/builds/hg-shared"
 
-if platform.system().lower() == 'darwin':
-    xre_url = "https://api.pub.build.mozilla.org/tooltool/sha512/4d8d7a37d90c34a2a2fda3066a8fe85c189b183d05389cb957fc6fed31f10a6924e50c1b84488ff61c015293803f58a3aed5d4819374d04c8e0ee2b9e3997278"
-else:
-    xre_url = "https://api.pub.build.mozilla.org/tooltool/sha512/dc9503b21c87b5a469118746f99e4f41d73888972ce735fa10a80f6d218086da0e3da525d9a4cd8e4ea497ec199fef720e4a525873d77a1af304ac505e076462"
-
 config = {
     # mozharness script options
-    "xre_url": xre_url,
-
     "vcs_share_base": HG_SHARE_BASE_DIR,
-    "exes": {
-        'python': '/tools/buildbot/bin/python',
-        'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
-        'tooltool.py': "/tools/tooltool.py",
-    },
 
     "find_links": [
         "http://pypi.pvt.build.mozilla.org/pub",
         "http://pypi.pub.build.mozilla.org/pub",
     ],
     "pip_index": False,
 
-    "buildbot_json_path": "buildprops.json",
-
     "default_actions": [
         'clobber',
-        'read-buildbot-config',
-        'pull',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-tests',
     ],
+    "download_symbols": "ondemand",
+    # We bake this directly into the tester image now...
+    "download_minidump_stackwalk": False,
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "vcs_output_timeout": 1760,
 }
rename from testing/mozharness/configs/b2g/mulet_config.py
rename to testing/mozharness/configs/b2g/taskcluster_mulet_automation.py
--- a/testing/mozharness/configs/b2g/mulet_config.py
+++ b/testing/mozharness/configs/b2g/taskcluster_mulet_automation.py
@@ -1,19 +1,33 @@
-# Call --cfg b2g/generic_config.py before this config
+# This is a template config file for mulet unittest testing
+import os
+
 config = {
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+
+    "download_symbols": "ondemand",
+    "download_minidump_stackwalk": False,
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+
     "default_actions": [
         'clobber',
-        'read-buildbot-config',
-        'pull',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-tests',
     ],
+
     # testsuite options
     "suite_definitions": {
         "reftest": {
             "options": [
                 "--mulet",
                 "--profile=%(gaia_profile)s",
                 "--appname=%(application)s",
                 "--total-chunks=%(total_chunks)s",
--- a/testing/taskcluster/tasks/tests/b2g_gaia_unit_oop.yml
+++ b/testing/taskcluster/tasks/tests/b2g_gaia_unit_oop.yml
@@ -7,23 +7,18 @@ task:
     description: Gaia Unit Test OOP
 
   payload:
     command:
       - entrypoint
       - ./bin/pull_gaia.sh &&
       - >
         python ./mozharness/scripts/gaia_unit.py
-        --no-read-buildbot-config
-        --config-file ./mozharness/configs/b2g/gaia_unit_production_config.py
-        --config-file ./mozharness_configs/gaia_integration_override.py
-        --config-file ./mozharness_configs/remove_executables.py
-        --no-pull
+        --config-file ./mozharness/configs/b2g/taskcluster_gaia_unit_production.py
         --browser-arg -oop
-        --download-symbols ondemand
         --installer-url {{build_url}}
         --test-url {{tests_url}}
         --gaia-repo https://hg.mozilla.org/integration/gaia-central
         --gaia-dir /home/worker
         --xre-url https://api.pub.build.mozilla.org/tooltool/sha512/cefa8c00db04969d3a50e2a5509bd4ea1dc17d256a651a9518cb28dad72e87a1dbbcd3c88ef770be0edf0ab73d2d73925140df93618ffb7fab81b789d312f547
     artifacts:
       'public/build':
         type: directory
--- a/testing/taskcluster/tasks/tests/mulet_build_test.yml
+++ b/testing/taskcluster/tasks/tests/mulet_build_test.yml
@@ -8,24 +8,19 @@ task:
 
   payload:
     command:
       - entrypoint # entrypoint ensures we are running in xvfb
       - ./bin/pull_gaia.sh &&
       - >
         python ./mozharness/scripts/gaia_build_integration.py
         --application firefox
-        --no-read-buildbot-config
-        --config-file ./mozharness/configs/b2g/gaia_integration_config.py
-        --config-file ./mozharness_configs/gaia_integration_override.py
-        --config-file ./mozharness_configs/remove_executables.py
+        --config-file ./mozharness/configs/b2g/taskcluster_gaia_integration.py
         --installer-url {{build_url}}
-        --no-pull
         --test-packages-url {{test_packages_url}}
-        --download-symbols ondemand
         --gaia-repo https://hg.mozilla.org/integration/gaia-central
         --gaia-dir /home/worker
         --xre-url https://api.pub.build.mozilla.org/tooltool/sha512/cefa8c00db04969d3a50e2a5509bd4ea1dc17d256a651a9518cb28dad72e87a1dbbcd3c88ef770be0edf0ab73d2d73925140df93618ffb7fab81b789d312f547
     artifacts:
       'public/build':
         type: directory
         path: '/home/worker/artifacts/'
         expires: '{{#from_now}}1 year{{/from_now}}'
--- a/testing/taskcluster/tasks/tests/mulet_build_unit.yml
+++ b/testing/taskcluster/tasks/tests/mulet_build_unit.yml
@@ -8,22 +8,17 @@ task:
 
   payload:
     command:
       - entrypoint
       - ./bin/pull_gaia.sh &&
       - >
         python ./mozharness/scripts/gaia_build_unit.py
         --application firefox
-        --no-read-buildbot-config
-        --config-file ./mozharness/configs/b2g/gaia_integration_config.py
-        --config-file ./mozharness_configs/gaia_integration_override.py
-        --config-file ./mozharness_configs/remove_executables.py
-        --no-pull
-        --download-symbols ondemand
+        --config-file ./mozharness/configs/b2g/taskcluster_gaia_integration.py
         --installer-url {{build_url}}
         --test-packages-url {{test_packages_url}}
         --gaia-repo https://hg.mozilla.org/integration/gaia-central
         --gaia-dir /home/worker
     artifacts:
       'public/build':
         type: directory
         path: '/home/worker/artifacts/'
--- a/testing/taskcluster/tasks/tests/mulet_gaia_js_integration_tests.yml
+++ b/testing/taskcluster/tasks/tests/mulet_gaia_js_integration_tests.yml
@@ -9,24 +9,19 @@ task:
 
   payload:
     command:
       - entrypoint # entrypoint ensures we are running in xvfb
       - ./bin/pull_gaia.sh &&
       - >
         python ./mozharness/scripts/gaia_integration.py
         --application firefox
-        --no-read-buildbot-config
-        --config-file b2g/gaia_integration_config.py
-        --config-file ./mozharness_configs/gaia_integration_override.py
-        --config-file ./mozharness_configs/remove_executables.py
-        --no-pull
+        --config-file b2g/taskcluster_gaia_integration.py
         --installer-url {{build_url}}
         --test-packages-url {{test_packages_url}}
-        --download-symbols ondemand
         --total-chunk {{total_chunks}}
         --this-chunk {{chunk}}
         --gaia-repo {{gaia_head_repository}}
         --gaia-dir /home/worker
     artifacts:
       'public/build':
         type: directory
         path: '/home/worker/artifacts/'
--- a/testing/taskcluster/tasks/tests/mulet_gaia_unit.yml
+++ b/testing/taskcluster/tasks/tests/mulet_gaia_unit.yml
@@ -8,24 +8,19 @@ task:
 
   payload:
     command:
       - entrypoint
       - ./bin/pull_gaia.sh &&
       - >
         python ./mozharness/scripts/gaia_unit.py
         --application firefox
-        --no-read-buildbot-config
-        --config-file b2g/gaia_unit_production_config.py
-        --config-file ./mozharness_configs/gaia_integration_override.py
-        --config-file ./mozharness_configs/remove_executables.py
-        --no-pull
+        --config-file b2g/taskcluster_gaia_unit_production.py
         --installer-url {{build_url}}
         --test-packages-url {{test_packages_url}}
-        --download-symbols ondemand
         --gaia-repo {{gaia_head_repository}}
         --gaia-dir /home/worker
         --xre-url https://api.pub.build.mozilla.org/tooltool/sha512/cefa8c00db04969d3a50e2a5509bd4ea1dc17d256a651a9518cb28dad72e87a1dbbcd3c88ef770be0edf0ab73d2d73925140df93618ffb7fab81b789d312f547
         --total-chunk={{total_chunks}}
         --this-chunk={{chunk}}
     artifacts:
       'public/build':
         type: directory
--- a/testing/taskcluster/tasks/tests/mulet_gaia_unit_oop.yml
+++ b/testing/taskcluster/tasks/tests/mulet_gaia_unit_oop.yml
@@ -8,25 +8,20 @@ task:
 
   payload:
     command:
       - entrypoint
       - ./bin/pull_gaia.sh &&
       - >
         python ./mozharness/scripts/gaia_unit.py
         --application firefox
-        --no-read-buildbot-config
-        --config-file b2g/gaia_unit_production_config.py
-        --config-file ./mozharness_configs/gaia_integration_override.py
-        --config-file ./mozharness_configs/remove_executables.py
-        --no-pull
+        --config-file b2g/taskcluster_gaia_unit_production.py
         --browser-arg -oop
         --installer-url {{build_url}}
         --test-packages-url {{test_packages_url}}
-        --download-symbols ondemand
         --gaia-repo {{gaia_head_repository}}
         --gaia-dir /home/worker
         --xre-url https://api.pub.build.mozilla.org/tooltool/sha512/cefa8c00db04969d3a50e2a5509bd4ea1dc17d256a651a9518cb28dad72e87a1dbbcd3c88ef770be0edf0ab73d2d73925140df93618ffb7fab81b789d312f547
         --total-chunk={{total_chunks}}
         --this-chunk={{chunk}}
     artifacts:
       'public/build':
         type: directory
--- a/testing/taskcluster/tasks/tests/mulet_linter.yml
+++ b/testing/taskcluster/tasks/tests/mulet_linter.yml
@@ -8,21 +8,17 @@ task:
 
   payload:
     command:
       - entrypoint
       - ./bin/pull_gaia.sh &&
       - >
         python ./mozharness/scripts/gaia_linter.py
         --application firefox
-        --no-read-buildbot-config
-        --config-file ./mozharness/configs/b2g/gaia_integration_config.py
-        --config-file ./mozharness_configs/gaia_integration_override.py
-        --download-symbols ondemand
-        --no-pull
+        --config-file ./mozharness/configs/b2g/taskcluster_gaia_integration.py
         --installer-url {{build_url}}
         --test-packages-url {{test_packages_url}}
         --gaia-repo https://hg.mozilla.org/integration/gaia-central
         --gaia-dir /home/worker
     artifacts:
       'public/build':
         type: directory
         path: '/home/worker/artifacts/'
--- a/testing/taskcluster/tasks/tests/mulet_reftests.yml
+++ b/testing/taskcluster/tasks/tests/mulet_reftests.yml
@@ -8,24 +8,20 @@ task:
 
   payload:
     env:
       MOZ_DISABLE_NONLOCAL_CONNECTIONS: 0
     command:
       - entrypoint
       - ./bin/pull_gaia.sh &&
       - >
-        python ./mozharness/scripts/mulet_unittest.py --no-read-buildbot-config
-        --config-file ./mozharness/configs/b2g/generic_config.py
-        --config-file ./mozharness/configs/b2g/mulet_config.py
-        --config-file ./mozharness_configs/remove_executables.py
-        --no-pull
+        python ./mozharness/scripts/mulet_unittest.py
+        --config-file ./mozharness/configs/b2g/taskcluster_mulet_automation.py
         --installer-url {{build_url}}
         --test-packages-url {{test_packages_url}}
-        --download-symbols ondemand
         --test-suite reftest
         --test-manifest tests/layout/reftests/reftest.list
         --this-chunk {{chunk}}
         --total-chunk {{total_chunks}}
         --gaia-repo https://hg.mozilla.org/integration/gaia-central
         --gaia-dir /home/worker/gaia
     artifacts:
       'public/build':