Bug 1639256 - Coerce all wpt paths to unicode, r=jdm
authorJames Graham <james@hoppipolla.co.uk>
Tue, 19 May 2020 12:23:22 +0100
changeset 531200 b7bfbeddfe2a01669f5a316b9867618e51b3a633
parent 531199 e752bc588a218457572454e2da885a59ff4e3e4f
child 531201 135acd548abffc587408d2d5f512d0595fd7d244
push id37435
push userapavel@mozilla.com
push dateWed, 20 May 2020 15:28:23 +0000
treeherdermozilla-central@5415da14ec9a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjdm
bugs1639256
milestone78.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1639256 - Coerce all wpt paths to unicode, r=jdm Differential Revision: https://phabricator.services.mozilla.com/D76011
testing/web-platform/tests/tools/manifest/manifest.py
testing/web-platform/tests/tools/manifest/sourcefile.py
--- a/testing/web-platform/tests/tools/manifest/manifest.py
+++ b/testing/web-platform/tests/tools/manifest/manifest.py
@@ -2,16 +2,17 @@ import io
 import itertools
 import json
 import os
 from copy import deepcopy
 from multiprocessing import Pool, cpu_count
 from six import (
     PY3,
     binary_type,
+    ensure_text,
     iteritems,
     itervalues,
     string_types,
     text_type,
 )
 
 from . import vcs
 from .item import (ConformanceCheckerTest, ManifestItem, ManualTest, RefTest, SupportFile,
@@ -170,22 +171,24 @@ class Manifest(object):
         # attribute access in the hot loop below
         data = self._data
 
         types = data.type_by_path()
         deleted = set(types)
 
         to_update = []
 
-        for source_file, update in tree:
+        for source_file_or_path, update in tree:
             if not update:
-                assert isinstance(source_file, (binary_type, text_type))
-                deleted.remove(tuple(source_file.split(os.path.sep)))
+                assert isinstance(source_file_or_path, (binary_type, text_type))
+                path = ensure_text(source_file_or_path)
+                deleted.remove(tuple(path.split(os.path.sep)))
             else:
-                assert not isinstance(source_file, bytes)
+                assert not isinstance(source_file_or_path, (binary_type, text_type))
+                source_file = source_file_or_path
                 rel_path_parts = source_file.rel_path_parts
                 assert isinstance(rel_path_parts, tuple)
 
                 is_new = rel_path_parts not in deleted  # type: bool
                 hash_changed = False  # type: bool
 
                 if not is_new:
                     deleted.remove(rel_path_parts)
--- a/testing/web-platform/tests/tools/manifest/sourcefile.py
+++ b/testing/web-platform/tests/tools/manifest/sourcefile.py
@@ -1,13 +1,13 @@
 import hashlib
 import re
 import os
 from collections import deque
-from six import binary_type, iteritems, text_type
+from six import binary_type, ensure_text, iteritems, text_type
 from six.moves.urllib.parse import urljoin
 from fnmatch import fnmatch
 
 MYPY = False
 if MYPY:
     # MYPY is set to True when run under Mypy.
     from typing import Any
     from typing import AnyStr
@@ -185,51 +185,48 @@ class SourceFile(object):
     dir_non_test = {"resources",
                     "support",
                     "tools"}
 
     dir_path_non_test = {("css21", "archive"),
                          ("css", "CSS2", "archive"),
                          ("css", "common")}  # type: Set[Tuple[bytes, ...]]
 
-    def __init__(self, tests_root, rel_path, url_base, hash=None, contents=None):
+    def __init__(self, tests_root, rel_path_str, url_base, hash=None, contents=None):
         # type: (AnyStr, AnyStr, Text, Optional[Text], Optional[bytes]) -> None
         """Object representing a file in a source tree.
 
         :param tests_root: Path to the root of the source tree
-        :param rel_path: File path relative to tests_root
+        :param rel_path_str: File path relative to tests_root
         :param url_base: Base URL used when converting file paths to urls
         :param contents: Byte array of the contents of the file or ``None``.
         """
 
+        rel_path = ensure_text(rel_path_str)
         assert not os.path.isabs(rel_path), rel_path
-
         if os.name == "nt":
             # do slash normalization on Windows
-            if isinstance(rel_path, binary_type):
-                rel_path = rel_path.replace(b"/", b"\\")
-            else:
-                rel_path = rel_path.replace(u"/", u"\\")
+            rel_path = rel_path.replace(u"/", u"\\")
 
         dir_path, filename = os.path.split(rel_path)
         name, ext = os.path.splitext(filename)
 
         type_flag = None
         if "-" in name:
             type_flag = name.rsplit("-", 1)[1].split(".")[0]
 
         meta_flags = name.split(".")[1:]
 
-        self.tests_root = tests_root  # type: Union[bytes, Text]
-        self.rel_path = rel_path  # type: Union[bytes, Text]
-        self.dir_path = dir_path  # type: Union[bytes, Text]
-        self.filename = filename  # type: Union[bytes, Text]
-        self.name = name  # type: Union[bytes, Text]
-        self.ext = ext  # type: Union[bytes, Text]
-        self.type_flag = type_flag  # type: Optional[Union[bytes, Text]]
+        self.tests_root = ensure_text(tests_root)  # type: Text
+        self.rel_path = rel_path  # type: Text
+        self.dir_path = dir_path  # type: Text
+        self.filename = filename  # type: Text
+        self.name = name  # type: Text
+        self.ext = ext  # type: Text
+        self.type_flag = type_flag  # type: Optional[Text]
         self.meta_flags = meta_flags  # type: Union[List[bytes], List[Text]]
         self.url_base = url_base
         self.contents = contents
         self.items_cache = None  # type: Optional[Tuple[Text, List[ManifestItem]]]
         self._hash = hash
 
     def __getstate__(self):
         # type: () -> Dict[str, Any]
@@ -277,17 +274,17 @@ class SourceFile(object):
 
     @cached_property
     def rel_path_parts(self):
         # type: () -> Tuple[Text, ...]
         return tuple(self.rel_path.split(os.path.sep))
 
     @cached_property
     def path(self):
-        # type: () -> Union[bytes, Text]
+        # type: () -> Text
         return os.path.join(self.tests_root, self.rel_path)
 
     @cached_property
     def rel_url(self):
         # type: () -> Text
         assert not os.path.isabs(self.rel_path), self.rel_path
         return self.rel_path.replace(os.sep, "/")