| author | James Graham <james@hoppipolla.co.uk> |
| Tue, 23 May 2017 13:48:29 +0100 | |
| changeset 361567 | ecb85cc1ee3e7e6473ecbe2b450fa2a5a55fc5ff |
| parent 361566 | 1f17d43cec5d18d14c26551358e6139e66cf3b9e |
| child 361568 | 612e32792109e378eeae4580061ec0bab5a903e7 |
| push id | 31933 |
| push user | ryanvm@gmail.com |
| push date | Wed, 31 May 2017 18:33:05 +0000 |
| treeherder | mozilla-central@94906c37940c [default view] [failures only] |
| perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
| reviewers | AutomatedTester |
| bugs | 1367041 |
| milestone | 55.0a1 |
| first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
| last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/.gitignore +++ b/.gitignore @@ -85,16 +85,19 @@ devtools/**/node_modules GTAGS GRTAGS GSYMS GPATH # Git clone directory for updating web-platform-tests testing/web-platform/sync/ +# Third party metadata for web-platform-tests +testing/web-platform/products/ + # Android Gradle artifacts. mobile/android/gradle/.gradle # XCode project cruft embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/project.xcworkspace/xcuserdata embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/xcuserdata # Ignore mozharness execution files
--- a/.hgignore +++ b/.hgignore @@ -93,16 +93,19 @@ GTAGS GRTAGS GSYMS GPATH # Git clone directory for updating web-platform-tests ^testing/web-platform/sync/ +# Third party metadata for web-platform-tests +^testing/web-platform/products/ + # Android Gradle artifacts. ^mobile/android/gradle/.gradle # XCode project cruft ^embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/project.xcworkspace/xcuserdata ^embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/xcuserdata # Ignore mozharness execution files
--- a/testing/web-platform/README.md +++ b/testing/web-platform/README.md @@ -230,50 +230,19 @@ In order to update the manifest it is re web-platform-tests --manifest-update`. This rescans the test directory looking for new, removed, or altered tests. Running Tests In Other Browsers ------------------------------- web-platform-tests is cross browser, and the runner is compatible with multiple browsers. Therefore it's possible to check the behaviour of -tests in other browsers. This is somewhat more involved than running -them in Firefox since extra dependencies may be required. For example -to test in Chrome: - -1. Download the chromedriver binary and place it somewhere sensible - e.g. `~/bin` +tests in other browsers. By default Chrome, Edge and Servo are +supported. In order to run the tests in these browsers use the +`--product` argument to wptrunner: -2. In your gecko source tree activate the virtualenv created by mach, - since this has most dependencies already installed. This is typically - in objdir/_virtualenv and is activated via e.g. - - source objdir/_virtualenv/bin/activate - -3. Install the extra requirements: - - cd testing/web-platform/harness - pip install -r requirements_chrome.txt - -4. Edit the config file `testing/web-platform/wptrunner.ini` so that - Chrome support is enabled by changing the section that reads: + mach wpt --product chrome dom/historical.html - [products] - firefox = - - to read - - [products] - firefox = - chrome = - - (alternatively create a new config file elsewhere and use the - `--config` option to `runtests.py` to point wptrunner at this config - file). - -5. Run `runtests.py` using the location of chromedriver as - the binary: - - cd testing/web-platform - python runtests.py --product=chrome --binary=~/bin/chromedriver --log-mach=- - -By default this will use the same test checkout and metadata as are in -the Gecko tree, so it's easy to compare behaviour relative to Firefox. +By default these browsers run without expectation metadata, but it can +be added in the `testing/web-platform/products/<product>` +directory. To run with the same metadata as for Firefox (so that +differences are reported as unexpected results), pass `--meta +testing/web-platform/meta` to the mach command.
--- a/testing/web-platform/mach_commands.py +++ b/testing/web-platform/mach_commands.py @@ -23,17 +23,17 @@ from mach.decorators import ( # This should probably be consolidated with similar classes in other test # runners. class InvalidTestPathError(Exception): """Exception raised when the test path is not valid.""" class WebPlatformTestsRunner(MozbuildObject): """Run web platform tests.""" - def setup_kwargs(self, kwargs): + def setup_kwargs_firefox(self, kwargs): from wptrunner import wptcommandline build_path = os.path.join(self.topobjdir, 'build') if build_path not in sys.path: sys.path.append(build_path) if kwargs["config"] is None: kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini') @@ -62,20 +62,60 @@ class WebPlatformTestsRunner(MozbuildObj if kwargs["host_cert_path"] is None: kwargs["host_cert_path"] = os.path.join(here, "certs", "web-platform.test.pem") kwargs["capture_stdio"] = True kwargs = wptcommandline.check_args(kwargs) + def setup_kwargs_wptrun(self, kwargs): + from wptrunner import wptcommandline + here = os.path.join(self.topsrcdir, 'testing', 'web-platform') + + sys.path.insert(0, os.path.join(here, "tests", "tools")) + + import wptrun + + setup_func = { + "chrome": wptrun.setup_chrome, + "edge": wptrun.setup_edge, + "servo": wptrun.setup_servo, + }[kwargs["product"]] + + setup_func(wptrun.virtualenv.Virtualenv(self.virtualenv_manager.virtualenv_root), + kwargs, + True) + + kwargs["tests_root"] = os.path.join(here, "tests") + + if kwargs["metadata_root"] is None: + metadir = os.path.join(here, "products", kwargs["product"]) + if not os.path.exists(metadir): + os.makedirs(metadir) + kwargs["metadata_root"] = metadir + + src_manifest = os.path.join(here, "meta", "MANIFEST.json") + dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json") + + if not os.path.exists(dest_manifest) and os.path.exists(src_manifest): + with open(src_manifest) as src, open(dest_manifest, "w") as dest: + dest.write(src.read()) + + kwargs = wptcommandline.check_args(kwargs) + def run_tests(self, **kwargs): from wptrunner import wptrunner - self.setup_kwargs(kwargs) + if kwargs["product"] in ["firefox", None]: + self.setup_kwargs_firefox(kwargs) + elif kwargs["product"] in ("chrome", "edge", "servo"): + self.setup_kwargs_wptrun(kwargs) + else: + raise ValueError("Unknown product %s" % kwargs["product"]) logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) result = wptrunner.run_tests(**kwargs) return int(not result) def list_test_groups(self, **kwargs): from wptrunner import wptrunner @@ -253,17 +293,17 @@ class WPTManifestUpdater(MozbuildObject) from wptrunner import wptlogging logger = wptlogging.setup(kwargs, {"mach": sys.stdout}) wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform')) manifestupdate.update(logger, wpt_dir, check_clean) def create_parser_wpt(): from wptrunner import wptcommandline - return wptcommandline.create_parser(["firefox"]) + return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"]) def create_parser_update(): from update import updatecommandline return updatecommandline.create_parser() def create_parser_reduce(): from wptrunner import wptcommandline return wptcommandline.create_parser_reduce()
new file mode 100644 --- /dev/null +++ b/testing/web-platform/products/README @@ -0,0 +1,3 @@ +This directory is for storing metadata files when running +web-platform-tests with non-firefox browsers. By default all +subdirectories here are ignored in VCS. \ No newline at end of file