Bug 1461498 [wpt PR 10998] - [wptrunner] Recover from malformed test manifest , a=testonly
authorMike Pennisi <mike@mikepennisi.com>
Sat, 19 May 2018 20:31:32 +0000
changeset 419081 69acdec7fe3390b4ff0b1dddf95ee369fd7b43f8
parent 419080 2010d8252b0c14190119172a95e56c75f7953155
child 419082 1b0def1ce2fb31395e4baa1f2e81fc82cdf8f146
push id34026
push userapavel@mozilla.com
push dateMon, 21 May 2018 09:47:33 +0000
treeherdermozilla-central@dc1868d255be [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1461498, 10998
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1461498 [wpt PR 10998] - [wptrunner] Recover from malformed test manifest , a=testonly Automatic update from web-platform-tests[wptrunner] Add surface tests for `--list-tests` Ensure that the WPT CLI exits successfully when the `--list-tests` option is specified (including the case where no test manifest is present). -- [wptrunner] Recover from malformed test manifest If the process of updating the test manifest is interrupted unexpectedly, the file written to disk may not contain valid JSON. Update that WPT CLI to recognize this condition and recover by re-generating the manifest file. -- wpt-commits: 4ae7daf23b5688e0a08d67ee71e4822b7c0171d3, a3a5824500bdf2515114c6653253a9d8d172d37a wpt-pr: 10998
testing/web-platform/tests/tools/wpt/tests/test_wpt.py
testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py
--- a/testing/web-platform/tests/tools/wpt/tests/test_wpt.py
+++ b/testing/web-platform/tests/tools/wpt/tests/test_wpt.py
@@ -8,16 +8,19 @@ import tempfile
 import time
 import urllib2
 
 import pytest
 
 from tools.wpt import wpt
 
 
+here = os.path.abspath(os.path.dirname(__file__))
+
+
 def is_port_8000_in_use():
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     try:
         s.bind(("127.0.0.1", 8000))
     except socket.error as e:
         if e.errno == errno.EADDRINUSE:
             return True
         else:
@@ -59,16 +62,78 @@ def test_help():
     # TODO: It seems like there's a bug in argparse that makes this argument order required
     # should try to work around that
     with pytest.raises(SystemExit) as excinfo:
         wpt.main(argv=["--help"])
     assert excinfo.value.code == 0
 
 
 @pytest.mark.slow
+def test_list_tests(manifest_dir):
+    """The `--list-tests` option should not produce an error under normal
+    conditions."""
+
+    with pytest.raises(SystemExit) as excinfo:
+        wpt.main(argv=["run", "--metadata", manifest_dir, "--list-tests",
+                       "--yes", "chrome", "/dom/nodes/Element-tagName.html"])
+    assert excinfo.value.code == 0
+
+
+@pytest.mark.slow
+def test_list_tests_missing_manifest(manifest_dir):
+    """The `--list-tests` option should not produce an error in the absence of
+    a test manifest file."""
+
+    os.remove(os.path.join(manifest_dir, "MANIFEST.json"))
+
+    with pytest.raises(SystemExit) as excinfo:
+        wpt.main(argv=["run",
+                       # This test triggers the creation of a new manifest
+                       # file which is not necessary to ensure successful
+                       # process completion. Specifying the current directory
+                       # as the tests source via the --tests` option
+                       # drastically reduces the time to execute the test.
+                       "--tests", here,
+                       "--metadata", manifest_dir,
+                       "--list-tests",
+                       "--yes",
+                       "firefox", "/dom/nodes/Element-tagName.html"])
+
+    assert excinfo.value.code == 0
+
+
+@pytest.mark.slow
+def test_list_tests_invalid_manifest(manifest_dir):
+    """The `--list-tests` option should not produce an error in the presence of
+    a malformed test manifest file."""
+
+    manifest_filename = os.path.join(manifest_dir, "MANIFEST.json")
+
+    assert os.path.isfile(manifest_filename)
+
+    with open(manifest_filename, "a+") as handle:
+        handle.write("extra text which invalidates the file")
+
+    with pytest.raises(SystemExit) as excinfo:
+        wpt.main(argv=["run",
+                       # This test triggers the creation of a new manifest
+                       # file which is not necessary to ensure successful
+                       # process completion. Specifying the current directory
+                       # as the tests source via the --tests` option
+                       # drastically reduces the time to execute the test.
+                       "--tests", here,
+                       "--metadata", manifest_dir,
+                       "--list-tests",
+                       "--yes",
+                       "firefox", "/dom/nodes/Element-tagName.html"])
+
+    assert excinfo.value.code == 0
+
+
+@pytest.mark.slow
 @pytest.mark.remote_network
 @pytest.mark.xfail(sys.platform == "win32",
                    reason="Tests currently don't work on Windows for path reasons")
 def test_run_firefox(manifest_dir):
     # TODO: It seems like there's a bug in argparse that makes this argument order required
     # should try to work around that
     if is_port_8000_in_use():
         pytest.skip("port 8000 already in use")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py
@@ -411,20 +411,22 @@ class ManifestLoader(object):
             # TODO: make this not github-specific
             download_from_github(manifest_path, tests_path)
 
         if not recreate:
             try:
                 with open(manifest_path) as f:
                     json_data = json.load(f)
             except IOError:
-                #If the existing file doesn't exist just create one from scratch
-                pass
+                self.logger.info("Unable to find test manifest")
+            except ValueError:
+                self.logger.info("Unable to parse test manifest")
 
         if not json_data:
+            self.logger.info("Creating test manifest")
             manifest_file = manifest.Manifest(url_base)
         else:
             try:
                 manifest_file = manifest.Manifest.from_json(tests_path, json_data)
             except manifest.ManifestVersionMismatch:
                 manifest_file = manifest.Manifest(url_base)
 
         manifest_update.update(tests_path, manifest_file, True)