Bug 1294820 - Add support for running wpt from one-click loaner mach environment, r=ahal
authorJames Graham <james@hoppipolla.co.uk>
Mon, 19 Jun 2017 19:20:41 +0100
changeset 416189 21c963bcc2ce1b09b3d77565d5263af1391c544e
parent 416188 3d1f8cbf8836df558dc1dda85a29b994bbd9b2e5
child 416190 ea14e1c52a01a940a71ec9ad7d36f75730d41a80
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1294820
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1294820 - Add support for running wpt from one-click loaner mach environment, r=ahal Allow running |mach wpt| on one click loaners in order to run web-platform-tests tests. This implementation is just like the one for other testsuites using thee packaged tests rather than the checkout that we get with wpt, at least on Linux. That's also where the tests run from so it seems reasonable for now. Moving to the checkout in the future could remove some of the logic here by using a fake mozbuild environment so that the testsuite itself doesn't have to implement anything much. MozReview-Commit-ID: CaewrdjJ2ef
testing/mozharness/scripts/web_platform_tests.py
testing/tools/mach_test_package_bootstrap.py
testing/web-platform/mach_commands.py
testing/web-platform/mach_commands_base.py
testing/web-platform/mach_test_package_commands.py
testing/web-platform/moz.build
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -202,21 +202,22 @@ class WebPlatformTest(TestingMixin, Merc
                                       str_format_values=str_format_values))
         cmd.extend(self.query_tests_args(try_tests,
                                          str_format_values=str_format_values))
 
         return cmd
 
     def download_and_extract(self):
         super(WebPlatformTest, self).download_and_extract(
-            extract_dirs=["bin/*",
+            extract_dirs=["mach",
+                          "bin/*",
                           "config/*",
                           "mozbase/*",
                           "marionette/*",
-                          "tools/wptserve/*",
+                          "tools/*",
                           "web-platform/*"],
             suite_categories=["web-platform"])
 
     def run_tests(self):
         dirs = self.query_abs_dirs()
         cmd = self._query_cmd()
 
         parser = StructuredOutputParser(config=self.config,
--- a/testing/tools/mach_test_package_bootstrap.py
+++ b/testing/tools/mach_test_package_bootstrap.py
@@ -32,25 +32,28 @@ SEARCH_PATHS = [
     'mozbase/mozrunner',
     'mozbase/mozscreenshot',
     'mozbase/mozsystemmonitor',
     'mozbase/moztest',
     'mozbase/mozversion',
     'reftest',
     'tools/mach',
     'tools/wptserve',
+    'web-platform',
+    'web-platform/tests/tools/wptrunner',
     'xpcshell',
 ]
 
 # Individual files providing mach commands.
 MACH_MODULES = [
     'marionette/mach_test_package_commands.py',
     'mochitest/mach_test_package_commands.py',
     'reftest/mach_test_package_commands.py',
     'tools/mach/mach/commands/commandinfo.py',
+    'web-platform/mach_test_package_commands.py',
     'xpcshell/mach_test_package_commands.py',
 ]
 
 
 CATEGORIES = {
     'testing': {
         'short': 'Testing',
         'long': 'Run tests.',
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -15,25 +15,23 @@ from mozbuild.base import (
     MozbuildObject,
 )
 
 from mach.decorators import (
     CommandProvider,
     Command,
 )
 
-# This should probably be consolidated with similar classes in other test
-# runners.
-class InvalidTestPathError(Exception):
-    """Exception raised when the test path is not valid."""
+from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
+
 
-class WebPlatformTestsRunner(MozbuildObject):
-    """Run web platform tests."""
+class WebPlatformTestsRunnerSetup(MozbuildObject):
+    default_log_type = "mach"
 
-    def setup_kwargs_firefox(self, kwargs):
+    def kwargs_firefox(self, kwargs):
         from wptrunner import wptcommandline
 
         build_path = os.path.join(self.topobjdir, 'build')
         if build_path not in sys.path:
             sys.path.append(build_path)
 
         if kwargs["config"] is None:
             kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
@@ -65,17 +63,17 @@ class WebPlatformTestsRunner(MozbuildObj
 
         kwargs["capture_stdio"] = True
 
         if kwargs["webdriver_binary"] is None:
             kwargs["webdriver_binary"] = self.get_binary_path("geckodriver", validate_exists=False)
 
         kwargs = wptcommandline.check_args(kwargs)
 
-    def setup_kwargs_wptrun(self, kwargs):
+    def kwargs_wptrun(self, kwargs):
         from wptrunner import wptcommandline
         here = os.path.join(self.topsrcdir, 'testing', 'web-platform')
 
         sys.path.insert(0, os.path.join(here, "tests", "tools"))
 
         import wptrun
 
         product = kwargs["product"]
@@ -108,37 +106,16 @@ class WebPlatformTestsRunner(MozbuildObj
         dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json")
 
         if not os.path.exists(dest_manifest) and os.path.exists(src_manifest):
             with open(src_manifest) as src, open(dest_manifest, "w") as dest:
                 dest.write(src.read())
 
         kwargs = wptcommandline.check_args(kwargs)
 
-    def run_tests(self, **kwargs):
-        from wptrunner import wptrunner
-
-        if kwargs["product"] in ["firefox", None]:
-            self.setup_kwargs_firefox(kwargs)
-        elif kwargs["product"] in ("chrome", "edge", "servo"):
-            self.setup_kwargs_wptrun(kwargs)
-        else:
-            raise ValueError("Unknown product %s" % kwargs["product"])
-
-        logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
-        result = wptrunner.run_tests(**kwargs)
-
-        return int(not result)
-
-    def list_test_groups(self, **kwargs):
-        from wptrunner import wptrunner
-
-        self.setup_kwargs(kwargs)
-
-        wptrunner.list_test_groups(**kwargs)
 
 class WebPlatformTestsUpdater(MozbuildObject):
     """Update web platform tests."""
     def run_update(self, **kwargs):
         import update
         from update import updatecommandline
 
         if kwargs["config"] is None:
@@ -302,20 +279,16 @@ class WPTManifestUpdater(MozbuildObject)
     def run_update(self, check_clean=False, rebuild=False, **kwargs):
         import manifestupdate
         from wptrunner import wptlogging
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
         wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
         manifestupdate.update(logger, wpt_dir, check_clean, rebuild)
 
 
-def create_parser_wpt():
-    from wptrunner import wptcommandline
-    return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"])
-
 def create_parser_update():
     from update import updatecommandline
     return updatecommandline.create_parser()
 
 def create_parser_reduce():
     from wptrunner import wptcommandline
     return wptcommandline.create_parser_reduce()
 
@@ -359,22 +332,19 @@ class MachCommands(MachCommandBase):
     def run_web_platform_tests(self, **params):
         self.setup()
 
         if "test_objects" in params:
             for item in params["test_objects"]:
                 params["include"].append(item["name"])
             del params["test_objects"]
 
-        wpt_runner = self._spawn(WebPlatformTestsRunner)
-
-        if params["list_test_groups"]:
-            return wpt_runner.list_test_groups(**params)
-        else:
-            return wpt_runner.run_tests(**params)
+        wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
+        wpt_runner = WebPlatformTestsRunner(wpt_setup)
+        return wpt_runner.run(**params)
 
     @Command("wpt",
              category="testing",
              conditions=[conditions.is_firefox],
              parser=create_parser_wpt)
     def run_wpt(self, **params):
         return self.run_web_platform_tests(**params)
 
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/mach_commands_base.py
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+
+def create_parser_wpt():
+    from wptrunner import wptcommandline
+    return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"])
+
+
+class WebPlatformTestsRunner(object):
+    """Run web platform tests."""
+
+    def __init__(self, setup):
+        self.setup = setup
+
+    def run(self, **kwargs):
+        from wptrunner import wptrunner
+        if kwargs["product"] in ["firefox", None]:
+            self.setup.kwargs_firefox(kwargs)
+        elif kwargs["product"] in ("chrome", "edge", "servo"):
+            self.setup.kwargs_wptrun(kwargs)
+        else:
+            raise ValueError("Unknown product %s" % kwargs["product"])
+        logger = wptrunner.setup_logging(kwargs, {self.setup.default_log_type: sys.stdout})
+        result = wptrunner.start(**kwargs)
+        return int(not result)
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/mach_test_package_commands.py
@@ -0,0 +1,68 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+
+from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
+from mach.decorators import (
+    CommandProvider,
+    Command,
+)
+
+
+class WebPlatformTestsRunnerSetup(object):
+    default_log_type = "tbpl"
+
+    def __init__(self, context):
+        self.context = context
+
+    def kwargs_firefox(self, kwargs):
+        from wptrunner import wptcommandline
+        if kwargs["config"] is None:
+            kwargs["config"] = os.path.join(self.context.package_root, 'web-platform', 'wptrunner.ini')
+        if kwargs["binary"] is None:
+            kwargs["binary"] = self.context.firefox_bin
+        if kwargs["prefs_root"] is None:
+            kwargs["prefs_root"] = os.path.join(self.context.package_root, 'web-platform', "prefs")
+        if kwargs["certutil_binary"] is None:
+            kwargs["certutil_binary"] = os.path.join(self.context.bin_dir, 'certutil')
+        if kwargs["stackfix_dir"] is None:
+            kwargs["stackfix_dir"] = self.context.bin_dir
+        if kwargs["ssl_type"] in (None, "pregenerated"):
+            if kwargs["ca_cert_path"] is None:
+                kwargs["ca_cert_path"] = os.path.join(self.context.package_root, "web-platform", "certs", "cacert.pem")
+            if kwargs["host_key_path"] is None:
+                kwargs["host_key_path"] = os.path.join(self.context.package_root, "web-platform", "certs", "web-platform.test.key")
+            if kwargs["host_cert_path"] is None:
+                kwargs["host_cert_path"] = os.path.join(self.context.package_root, "web-platform", "certs", "web-platform.test.pem")
+        kwargs["capture_stdio"] = True
+
+        if kwargs["webdriver_binary"] is None:
+            kwargs["webdriver_binary"] = os.path.join(self.context.bin_dir, "geckodriver")
+
+        kwargs = wptcommandline.check_args(kwargs)
+
+    def kwargs_wptrun(self, kwargs):
+        raise NotImplementedError
+
+
+@CommandProvider
+class MachCommands(object):
+    def __init__(self, context):
+        self.context = context
+
+    @Command("web-platform-tests",
+             category="testing",
+             parser=create_parser_wpt)
+    def run_web_platform_tests(self, **kwargs):
+        self.context.activate_mozharness_venv()
+        return WebPlatformTestsRunner(WebPlatformTestsRunnerSetup(self.context)).run(**kwargs)
+
+    @Command("wpt",
+             category="testing",
+             parser=create_parser_wpt)
+    def run_wpt(self, **params):
+        return self.run_web_platform_tests(**params)
--- a/testing/web-platform/moz.build
+++ b/testing/web-platform/moz.build
@@ -5,16 +5,18 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 WEB_PLATFORM_TESTS_MANIFESTS += [
     ('meta/MANIFEST.json', 'tests/'),
     ('mozilla/meta/MANIFEST.json', 'mozilla/tests/')
 ]
 
 TEST_HARNESS_FILES['web-platform'] += [
+    'mach_commands_base.py',
+    'mach_test_package_commands.py',
     'outbound/**',
     'runtests.py',
     'wptrunner.ini'
 ]
 
 TEST_HARNESS_FILES['web-platform'].certs = [
     'certs/cacert.pem',
     'certs/web-platform.test.key',