Bug 1620964 [wpt PR 22139] - Fix manifest writing in python3, a=testonly
authorSergio <svillar@igalia.com>
Sat, 14 Mar 2020 11:29:57 +0000
changeset 518858 b2faa06cd912b3d9fdeae3e0ce1d4f03d8f88124
parent 518857 3eb5aa166665294edba623b9ec502587b04f3763
child 518859 e0db51ee811dc6abb6d0897bbbf56a8ed8fdb1f0
push id37217
push userccoroiu@mozilla.com
push dateSun, 15 Mar 2020 21:37:59 +0000
treeherdermozilla-central@f9fc9427476e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1620964, 22139
milestone76.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1620964 [wpt PR 22139] - Fix manifest writing in python3, a=testonly Automatic update from web-platform-tests Fix manifest writing in python3 (#22139) Store more properties as text and convert to str for output. -- wpt-commits: d4cdf4021ab48c66b052fb62d91da1a06c8cbcf7 wpt-pr: 22139
testing/web-platform/tests/tools/manifest/item.py
testing/web-platform/tests/tools/manifest/manifest.py
testing/web-platform/tests/tools/manifest/sourcefile.py
testing/web-platform/tests/tools/manifest/tests/test_sourcefile.py
testing/web-platform/tests/tools/manifest/typedata.py
testing/web-platform/tests/tools/manifest/vcs.py
testing/web-platform/tests/tools/wpt/requirements.txt
testing/web-platform/tests/tools/wpt/tests/test_wpt.py
testing/web-platform/tests/tools/wptrunner/requirements.txt
testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_products.py
--- a/testing/web-platform/tests/tools/manifest/item.py
+++ b/testing/web-platform/tests/tools/manifest/item.py
@@ -200,17 +200,17 @@ class TestharnessTest(URLManifestItem):
         rv = super(TestharnessTest, self).to_json()
         if self.timeout is not None:
             rv[-1]["timeout"] = self.timeout
         if self.testdriver:
             rv[-1]["testdriver"] = self.testdriver
         if self.jsshell:
             rv[-1]["jsshell"] = True
         if self.script_metadata:
-            rv[-1]["script_metadata"] = self.script_metadata
+            rv[-1]["script_metadata"] = [(k.decode('utf8'), v.decode('utf8')) for (k,v) in self.script_metadata]
         return rv
 
 
 class RefTest(URLManifestItem):
     __slots__ = ("references",)
 
     item_type = "reftest"
 
--- a/testing/web-platform/tests/tools/manifest/manifest.py
+++ b/testing/web-platform/tests/tools/manifest/manifest.py
@@ -386,14 +386,14 @@ def load_and_update(tests_root,  # type:
     return manifest
 
 
 def write(manifest, manifest_path):
     # type: (Manifest, bytes) -> None
     dir_name = os.path.dirname(manifest_path)
     if not os.path.exists(dir_name):
         os.makedirs(dir_name)
-    with open(manifest_path, "wb") as f:
+    with open(manifest_path, "w") as f:
         # Use ',' instead of the default ', ' separator to prevent trailing
         # spaces: https://docs.python.org/2/library/json.html#json.dump
         json.dump(manifest.to_json(caller_owns_obj=True), f,
                   sort_keys=True, indent=1, separators=(',', ': '))
         f.write("\n")
--- a/testing/web-platform/tests/tools/manifest/sourcefile.py
+++ b/testing/web-platform/tests/tools/manifest/sourcefile.py
@@ -1,13 +1,13 @@
 import hashlib
 import re
 import os
 from collections import deque
-from six import binary_type, PY3, iteritems
+from six import binary_type, iteritems, text_type
 from six.moves.urllib.parse import urljoin
 from fnmatch import fnmatch
 
 MYPY = False
 if MYPY:
     # MYPY is set to True when run under Mypy.
     from typing import Any
     from typing import AnyStr
@@ -303,21 +303,17 @@ class SourceFile(object):
     @cached_property
     def hash(self):
         # type: () -> Text
         if not self._hash:
             with self.open() as f:
                 content = f.read()
 
             data = b"".join((b"blob ", b"%d" % len(content), b"\0", content))
-            hash_str = hashlib.sha1(data).hexdigest()  # type: str
-            if PY3:
-                self._hash = hash_str.encode("ascii")
-            else:
-                self._hash = hash_str
+            self._hash = text_type(hashlib.sha1(data).hexdigest())
 
         return self._hash
 
     def in_non_test_dir(self):
         # type: () -> bool
         if self.dir_path == "":
             return True
 
--- a/testing/web-platform/tests/tools/manifest/tests/test_sourcefile.py
+++ b/testing/web-platform/tests/tools/manifest/tests/test_sourcefile.py
@@ -827,9 +827,9 @@ def test_reftest_fuzzy_multi(fuzzy, expe
     s = create("foo/test.html", content)
 
     assert s.content_is_ref_node
     assert s.fuzzy == expected
 
 
 def test_hash():
     s = SourceFile("/", "foo", "/", contents=b"Hello, World!")
-    assert b"b45ef6fec89518d314f546fd6c3025367b721684" == s.hash
+    assert "b45ef6fec89518d314f546fd6c3025367b721684" == s.hash
--- a/testing/web-platform/tests/tools/manifest/typedata.py
+++ b/testing/web-platform/tests/tools/manifest/typedata.py
@@ -251,24 +251,37 @@ class TypeData(TypeDataType):
         Note that the returned object may contain references to the internal
         data structures, and is only guaranteed to be valid until the next
         __getitem__, __setitem__, __delitem__ call, so the caller must not
         mutate any part of the returned object graph.
 
         """
         json_rv = self._json_data.copy()
 
+        def safe_sorter(element):
+            # type: (Tuple[str,str]) -> Tuple[str,str]
+            """ key function to sort lists with None values.
+
+            Python3 is more strict typewise. Comparing None and str for example is valid
+            in python2 but throws an exception in python3.
+            """
+            if element and not element[0]:
+                return ("", element[1])
+            else:
+                return element
+
         stack = [(self._data, json_rv, tuple())]  # type: List[Tuple[Dict[Text, Any], Dict[Text, Any], Tuple[Text, ...]]]
         while stack:
             data_node, json_node, par_full_key = stack.pop()
             for k, v in iteritems(data_node):
                 full_key = par_full_key + (k,)
                 if isinstance(v, set):
                     assert k not in json_node
-                    json_node[k] = [self._hashes.get(full_key)] + [t for t in sorted(test.to_json() for test in v)]
+                    json_node[k] = [self._hashes.get(
+                        full_key)] + [t for t in sorted((test.to_json() for test in v), key=safe_sorter)]
                 else:
                     json_node[k] = json_node.get(k, {}).copy()
                     stack.append((v, json_node[k], full_key))
 
         return json_rv
 
 
 class PathHash(PathHashType):
--- a/testing/web-platform/tests/tools/manifest/vcs.py
+++ b/testing/web-platform/tests/tools/manifest/vcs.py
@@ -58,17 +58,17 @@ class GitHasher(object):
         self.git = git(path)
 
     def _local_changes(self):
         # type: () -> Set[Text]
         """get a set of files which have changed between HEAD and working copy"""
         assert self.git is not None
         # note that git runs the command with tests_root as the cwd, which may
         # not be the root of the git repo (e.g., within a browser repo)
-        cmd = [b"diff-index", b"--relative", b"--no-renames", b"--name-only", b"-z", b"HEAD"]
+        cmd = ["diff-index", "--relative", "--no-renames", "--name-only", "-z", "HEAD"]
         data = self.git(*cmd)
         return set(data.split("\0"))
 
     def hash_cache(self):
         # type: () -> Dict[Text, Optional[Text]]
         """
         A dict of rel_path -> current git object id if the working tree matches HEAD else None
         """
--- a/testing/web-platform/tests/tools/wpt/requirements.txt
+++ b/testing/web-platform/tests/tools/wpt/requirements.txt
@@ -1,1 +1,2 @@
 requests==2.23.0
+mozinfo==1.2.1  # https://bugzilla.mozilla.org/show_bug.cgi?id=1621226
--- a/testing/web-platform/tests/tools/wpt/tests/test_wpt.py
+++ b/testing/web-platform/tests/tools/wpt/tests/test_wpt.py
@@ -1,10 +1,11 @@
 import errno
 import os
+import platform
 import shutil
 import socket
 import subprocess
 import sys
 import tempfile
 import time
 
 try:
@@ -38,18 +39,19 @@ def is_port_8000_in_use():
 def get_persistent_manifest_path():
     directory = ("~/meta" if os.environ.get('TRAVIS') == "true"
                  else wpt.localpaths.repo_root)
     return os.path.join(directory, "MANIFEST.json")
 
 
 @pytest.fixture(scope="module", autouse=True)
 def init_manifest():
-    if sys.version_info >= (3,):
-        pytest.xfail(reason="broken on Py3")
+    # See https://github.com/pypa/virtualenv/issues/1710
+    if sys.version_info[0] >= 3 and platform.system() == "Windows":
+        pytest.xfail(reason="virtualenv activation fails in Windows for python3")
     with pytest.raises(SystemExit) as excinfo:
         wpt.main(argv=["manifest", "--no-download",
                        "--path", get_persistent_manifest_path()])
     assert excinfo.value.code == 0
 
 
 @pytest.fixture
 def manifest_dir():
--- a/testing/web-platform/tests/tools/wptrunner/requirements.txt
+++ b/testing/web-platform/tests/tools/wptrunner/requirements.txt
@@ -1,9 +1,9 @@
 html5lib==1.0.1
-mozinfo==1.1.0
+mozinfo==1.2.1  # https://bugzilla.mozilla.org/show_bug.cgi?id=1621226
 mozlog==6.0
 mozdebug==0.2
 # Pillow 7 requires Python 3
 pillow==6.2.2  # pyup: <7.0
 urllib3[secure]==1.25.8
 requests==2.23.0
 six==1.14.0
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_products.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_products.py
@@ -1,10 +1,8 @@
-import sys
-
 from os.path import join, dirname
 
 import mock
 import pytest
 
 from .base import all_products, active_products
 from .. import environment
 from .. import products
@@ -15,21 +13,17 @@ environment.do_delayed_imports(None, tes
 
 @active_products("product")
 def test_load_active_product(product):
     """test we can successfully load the product of the current testenv"""
     products.load_product({}, product)
     # test passes if it doesn't throw
 
 
-@all_products("product", marks={
-    "firefox": pytest.mark.xfail(sys.platform.startswith('linux') and
-                                 sys.version_info >= (3, 8),
-                                 reason="mozinfo doesn't support Linux 3.8+")
-})
+@all_products("product")
 def test_load_all_products(product):
     """test every product either loads or throws ImportError"""
     try:
         products.load_product({}, product)
     except ImportError:
         pass