Bug 1494822 [wpt PR 13248] - Revert "Various test fixes for python3 support. (#11769)", a=testonly
authorGeoffrey Sneddon <me@gsnedders.com>
Fri, 05 Oct 2018 14:21:00 +0000
changeset 440060 11e1636ba8c8b755caf68ae3da829680713ecb87
parent 440059 2978f836d767a5df733f210dd303a3af33f1c918
child 440061 8f0661c28e345aea427515929e1a1fadba3312c4
push id34806
push usernerli@mozilla.com
push dateTue, 09 Oct 2018 04:03:56 +0000
treeherdermozilla-central@6a6c984745ef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1494822, 13248, 11769, 13204
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1494822 [wpt PR 13248] - Revert "Various test fixes for python3 support. (#11769)", a=testonly Automatic update from web-platform-testsRevert "Various test fixes for python3 support. (#11769)" This reverts commit 5bd512cdc76f762662915a2d3d9e49b021f14ff6. Fixes #13204. -- wpt-commits: c757432db546a30c1e6ef833df0b597df2dad6bd wpt-pr: 13248
testing/web-platform/tests/tools/wptserve/tests/functional/base.py
testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py
testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py
testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py
testing/web-platform/tests/tools/wptserve/wptserve/pipes.py
testing/web-platform/tests/tools/wptserve/wptserve/request.py
testing/web-platform/tests/tools/wptserve/wptserve/response.py
--- a/testing/web-platform/tests/tools/wptserve/tests/functional/base.py
+++ b/testing/web-platform/tests/tools/wptserve/tests/functional/base.py
@@ -70,17 +70,17 @@ class TestUsingServer(unittest.TestCase)
 
         for name, value in iteritems(headers):
             req.add_header(name, value)
 
         if body is not None:
             req.add_data(body)
 
         if auth is not None:
-            req.add_header("Authorization", b"Basic %s" % base64.b64encode(("%s:%s" % auth).encode("utf-8")))
+            req.add_header("Authorization", "Basic %s" % base64.b64encode('%s:%s' % auth))
 
         return urlopen(req)
 
 
 @pytest.mark.skipif(not wptserve.utils.http2_compatible(), reason="h2 server only works in python 2.7.15")
 class TestUsingH2Server:
     def setup_method(self, test_method):
         self.server = wptserve.server.WebTestHttpd(host="localhost",
--- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py
+++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py
@@ -83,26 +83,29 @@ class TestFileHandler(TestUsingServer):
             self.request("/document.txt", headers={"Range":"bytes=11-10"})
         self.assertEqual(cm.exception.code, 416)
 
         expected = open(os.path.join(doc_root, "document.txt"), 'rb').read()
         with self.assertRaises(HTTPError) as cm:
             self.request("/document.txt", headers={"Range":"bytes=%i-%i" % (len(expected), len(expected) + 10)})
         self.assertEqual(cm.exception.code, 416)
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_config(self):
         resp = self.request("/sub.sub.txt")
         expected = b"localhost localhost %i" % self.server.port
         assert resp.read().rstrip() == expected
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_headers(self):
         resp = self.request("/sub_headers.sub.txt", headers={"X-Test": "PASS"})
         expected = b"PASS"
         assert resp.read().rstrip() == expected
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_params(self):
         resp = self.request("/sub_params.sub.txt", query="test=PASS")
         expected = b"PASS"
         assert resp.read().rstrip() == expected
 
 
 class TestFunctionHandler(TestUsingServer):
     def test_string_rv(self):
--- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py
+++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py
@@ -52,76 +52,84 @@ class TestSlice(TestUsingServer):
         self.assertEqual(resp.read(), expected[1:])
 
     def test_no_lower(self):
         resp = self.request("/document.txt", query="pipe=slice(null,10)")
         expected = open(os.path.join(doc_root, "document.txt"), 'rb').read()
         self.assertEqual(resp.read(), expected[:10])
 
 class TestSub(TestUsingServer):
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_config(self):
         resp = self.request("/sub.txt", query="pipe=sub")
-        expected = b"localhost localhost %i" % self.server.port
+        expected = "localhost localhost %i" % self.server.port
         self.assertEqual(resp.read().rstrip(), expected)
 
     @pytest.mark.xfail(sys.platform == "win32",
                        reason="https://github.com/web-platform-tests/wpt/issues/12949")
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_file_hash(self):
         resp = self.request("/sub_file_hash.sub.txt")
-        expected = b"""
+        expected = """
 md5: JmI1W8fMHfSfCarYOSxJcw==
 sha1: nqpWqEw4IW8NjD6R375gtrQvtTo=
 sha224: RqQ6fMmta6n9TuA/vgTZK2EqmidqnrwBAmQLRQ==
 sha256: G6Ljg1uPejQxqFmvFOcV/loqnjPTW5GSOePOfM/u0jw=
 sha384: lkXHChh1BXHN5nT5BYhi1x67E1CyYbPKRKoF2LTm5GivuEFpVVYtvEBHtPr74N9E
-sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICzJwGFonfXwg=="""
+sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICz
+JwGFonfXwg=="""
         self.assertEqual(resp.read().rstrip(), expected.strip())
 
     def test_sub_file_hash_unrecognized(self):
         with self.assertRaises(urllib.error.HTTPError):
             self.request("/sub_file_hash_unrecognized.sub.txt")
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_headers(self):
         resp = self.request("/sub_headers.txt", query="pipe=sub", headers={"X-Test": "PASS"})
-        expected = b"PASS"
+        expected = "PASS"
         self.assertEqual(resp.read().rstrip(), expected)
 
     @pytest.mark.xfail(sys.platform == "win32",
                        reason="https://github.com/web-platform-tests/wpt/issues/12949")
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_location(self):
         resp = self.request("/sub_location.sub.txt?query_string")
         expected = """
 host: localhost:{0}
 hostname: localhost
 path: /sub_location.sub.txt
 pathname: /sub_location.sub.txt
 port: {0}
 query: ?query_string
 scheme: http
-server: http://localhost:{0}""".format(self.server.port).encode("ascii")
+server: http://localhost:{0}""".format(self.server.port)
         self.assertEqual(resp.read().rstrip(), expected.strip())
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_params(self):
         resp = self.request("/sub_params.txt", query="test=PASS&pipe=sub")
-        expected = b"PASS"
+        expected = "PASS"
         self.assertEqual(resp.read().rstrip(), expected)
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_url_base(self):
         resp = self.request("/sub_url_base.sub.txt")
-        self.assertEqual(resp.read().rstrip(), b"Before / After")
+        self.assertEqual(resp.read().rstrip(), "Before / After")
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_uuid(self):
         resp = self.request("/sub_uuid.sub.txt")
-        self.assertRegexpMatches(resp.read().rstrip(), b"Before [a-f0-9-]+ After")
+        self.assertRegexpMatches(resp.read().rstrip(), r"Before [a-f0-9-]+ After")
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_var(self):
         resp = self.request("/sub_var.sub.txt")
         port = self.server.port
-        print(port, type(port))
-        expected = b"localhost %d A %d B localhost C" % (port, port)
+        expected = "localhost %s A %s B localhost C" % (port, port)
         self.assertEqual(resp.read().rstrip(), expected)
 
 class TestTrickle(TestUsingServer):
     def test_trickle(self):
         #Actually testing that the response trickles in is not that easy
         t0 = time.time()
         resp = self.request("/document.txt", query="pipe=trickle(1:d2:5:d1:r2)")
         t1 = time.time()
@@ -131,76 +139,84 @@ class TestTrickle(TestUsingServer):
 
     def test_headers(self):
         resp = self.request("/document.txt", query="pipe=trickle(d0.01)")
         self.assertEqual(resp.info()["Cache-Control"], "no-cache, no-store, must-revalidate")
         self.assertEqual(resp.info()["Pragma"], "no-cache")
         self.assertEqual(resp.info()["Expires"], "0")
 
 class TestPipesWithVariousHandlers(TestUsingServer):
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_with_python_file_handler(self):
         resp = self.request("/test_string.py", query="pipe=slice(null,2)")
-        self.assertEqual(resp.read(), b"PA")
+        self.assertEqual(resp.read(), "PA")
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_with_python_func_handler(self):
         @wptserve.handlers.handler
         def handler(request, response):
             return "PASS"
         route = ("GET", "/test/test_pipes_1/", handler)
         self.server.router.register(*route)
         resp = self.request(route[1], query="pipe=slice(null,2)")
-        self.assertEqual(resp.read(), b"PA")
+        self.assertEqual(resp.read(), "PA")
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_with_python_func_handler_using_response_writer(self):
         @wptserve.handlers.handler
         def handler(request, response):
             response.writer.write_content("PASS")
         route = ("GET", "/test/test_pipes_1/", handler)
         self.server.router.register(*route)
         resp = self.request(route[1], query="pipe=slice(null,2)")
         # slice has not been applied to the response, because response.writer was used.
-        self.assertEqual(resp.read(), b"PASS")
+        self.assertEqual(resp.read(), "PASS")
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_header_pipe_with_python_func_using_response_writer(self):
         @wptserve.handlers.handler
         def handler(request, response):
             response.writer.write_content("CONTENT")
         route = ("GET", "/test/test_pipes_1/", handler)
         self.server.router.register(*route)
         resp = self.request(route[1], query="pipe=header(X-TEST,FAIL)")
         # header pipe was ignored, because response.writer was used.
         self.assertFalse(resp.info().get("X-TEST"))
-        self.assertEqual(resp.read(), b"CONTENT")
+        self.assertEqual(resp.read(), "CONTENT")
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_with_json_handler(self):
         @wptserve.handlers.json_handler
         def handler(request, response):
             return json.dumps({'data': 'PASS'})
         route = ("GET", "/test/test_pipes_2/", handler)
         self.server.router.register(*route)
         resp = self.request(route[1], query="pipe=slice(null,2)")
-        self.assertEqual(resp.read(), b'"{')
+        self.assertEqual(resp.read(), '"{')
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_slice_with_as_is_handler(self):
         resp = self.request("/test.asis", query="pipe=slice(null,2)")
         self.assertEqual(202, resp.getcode())
         self.assertEqual("Giraffe", resp.msg)
         self.assertEqual("PASS", resp.info()["X-Test"])
         # slice has not been applied to the response, because response.writer was used.
-        self.assertEqual(b"Content", resp.read())
+        self.assertEqual("Content", resp.read())
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_headers_with_as_is_handler(self):
         resp = self.request("/test.asis", query="pipe=header(X-TEST,FAIL)")
         self.assertEqual(202, resp.getcode())
         self.assertEqual("Giraffe", resp.msg)
         # header pipe was ignored.
         self.assertEqual("PASS", resp.info()["X-TEST"])
-        self.assertEqual(b"Content", resp.read())
+        self.assertEqual("Content", resp.read())
 
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_trickle_with_as_is_handler(self):
         t0 = time.time()
         resp = self.request("/test.asis", query="pipe=trickle(1:d2:5:d1:r2)")
         t1 = time.time()
-        self.assertTrue(b'Content' in resp.read())
+        self.assertTrue('Content' in resp.read())
         self.assertGreater(6, t1-t0)
 
 if __name__ == '__main__':
     unittest.main()
--- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py
+++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py
@@ -1,8 +1,10 @@
+import sys
+
 import pytest
 
 wptserve = pytest.importorskip("wptserve")
 from .base import TestUsingServer
 from wptserve.request import InputFile
 
 
 class TestInputFile(TestUsingServer):
@@ -110,18 +112,19 @@ class TestRequest(TestUsingServer):
 
         route = ("GET", "/test/{match}_*", handler)
         self.server.router.register(*route)
         resp = self.request("/test/some_route")
         self.assertEqual(b"some route", resp.read())
 
 
 class TestAuth(TestUsingServer):
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_auth(self):
         @wptserve.handlers.handler
         def handler(request, response):
             return " ".join((request.auth.username, request.auth.password))
 
         route = ("GET", "/test/test_auth", handler)
         self.server.router.register(*route)
         resp = self.request(route[1], auth=("test", "PASS"))
         self.assertEqual(200, resp.getcode())
-        self.assertEqual([b"test", b"PASS"], resp.read().split(b" "))
+        self.assertEqual(["test", "PASS"], resp.read().split(" "))
--- a/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py
+++ b/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py
@@ -1,11 +1,10 @@
 from cgi import escape
 from collections import deque
-import base64
 import gzip as gzip_module
 import hashlib
 import os
 import re
 import time
 import uuid
 from six.moves import StringIO
 
@@ -269,19 +268,18 @@ def slice(request, response, start, end=
 
     :param start: The starting offset. Follows python semantics including
                   negative numbers.
 
     :param end: The ending offset, again with python semantics and None
                 (spelled "null" in a query string) to indicate the end of
                 the file.
     """
-    content = resolve_content(response)[start:end]
-    response.content = content
-    response.headers.set("Content-Length", len(content))
+    content = resolve_content(response)
+    response.content = content[start:end]
     return response
 
 
 class ReplacementTokenizer(object):
     def arguments(self, token):
         unwrapped = token[1:-1].decode('utf8')
         return ("arguments", re.split(r",\s*", unwrapped) if unwrapped else [])
 
@@ -389,49 +387,50 @@ class SubFunctions(object):
     # are available on all platforms [1]. This ensures that test authors do not
     # unknowingly introduce platform-specific tests.
     #
     # [1] https://docs.python.org/2/library/hashlib.html
     supported_algorithms = ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")
 
     @staticmethod
     def file_hash(request, algorithm, path):
-        assert isinstance(algorithm, text_type)
+        algorithm = algorithm.decode("ascii")
         if algorithm not in SubFunctions.supported_algorithms:
             raise ValueError("Unsupported encryption algorithm: '%s'" % algorithm)
 
         hash_obj = getattr(hashlib, algorithm)()
         absolute_path = os.path.join(request.doc_root, path)
 
         try:
-            with open(absolute_path, "rb") as f:
+            with open(absolute_path) as f:
                 hash_obj.update(f.read())
         except IOError:
             # In this context, an unhandled IOError will be interpreted by the
             # server as an indication that the template file is non-existent.
             # Although the generic "Exception" is less precise, it avoids
             # triggering a potentially-confusing HTTP 404 error in cases where
             # the path to the file to be hashed is invalid.
             raise Exception('Cannot open file for hash computation: "%s"' % absolute_path)
 
-        return base64.b64encode(hash_obj.digest()).strip()
+        return hash_obj.digest().encode('base64').strip()
 
 def template(request, content, escape_type="html"):
     #TODO: There basically isn't any error handling here
     tokenizer = ReplacementTokenizer()
 
     variables = {}
 
     def config_replacement(match):
         content, = match.groups()
 
         tokens = tokenizer.tokenize(content)
         tokens = deque(tokens)
 
         token_type, field = tokens.popleft()
+        field = field.decode("ascii")
 
         if token_type == "var":
             variable = field
             token_type, field = tokens.popleft()
         else:
             variable = None
 
         if token_type != "ident":
@@ -486,21 +485,17 @@ def template(request, content, escape_ty
         if variable is not None:
             variables[variable] = value
 
         escape_func = {"html": lambda x:escape(x, quote=True),
                        "none": lambda x:x}[escape_type]
 
         #Should possibly support escaping for other contexts e.g. script
         #TODO: read the encoding of the response
-        if isinstance(value, binary_type):
-            value = value.decode("utf-8")
-        elif isinstance(value, int):
-            value = text_type(value)
-        return escape_func(value).encode("utf-8")
+        return escape_func(text_type(value)).encode("utf-8")
 
     template_regexp = re.compile(br"{{([^}]*)}}")
     new_content = template_regexp.sub(config_replacement, content)
 
     return new_content
 
 @pipe()
 def gzip(request, response):
--- a/testing/web-platform/tests/tools/wptserve/wptserve/request.py
+++ b/testing/web-platform/tests/tools/wptserve/wptserve/request.py
@@ -1,12 +1,12 @@
 import base64
 import cgi
 from six.moves.http_cookies import BaseCookie
-from six import BytesIO, binary_type, text_type
+from six import BytesIO
 import tempfile
 
 from six.moves.urllib.parse import parse_qsl, urlsplit
 
 from . import stash
 from .utils import HTTPException
 
 missing = object()
@@ -313,18 +313,18 @@ class Request(object):
             self._POST = MultiDict.from_field_storage(fs)
             self.raw_input.seek(pos)
         return self._POST
 
     @property
     def cookies(self):
         if self._cookies is None:
             parser = BaseCookie()
-            cookie_headers = self.headers.get("cookie", u"")
-            parser.load(cookie_headers.encode("utf-8"))
+            cookie_headers = self.headers.get("cookie", "")
+            parser.load(cookie_headers)
             cookies = Cookies()
             for key, value in parser.iteritems():
                 cookies[key] = CookieValue(value)
             self._cookies = cookies
         return self._cookies
 
     @property
     def headers(self):
@@ -350,51 +350,39 @@ class Request(object):
 
 class H2Request(Request):
     def __init__(self, request_handler):
         self.h2_stream_id = request_handler.h2_stream_id
         self.frames = []
         super(H2Request, self).__init__(request_handler)
 
 
-def maybedecode(s):
-    if isinstance(s, text_type):
-        return s
-
-    if isinstance(s, binary_type):
-        return s.decode("ascii")
-
-    raise TypeError("Unexpected value in RequestHeaders: %r" % s)
-
-
 class RequestHeaders(dict):
     """Dictionary-like API for accessing request headers."""
     def __init__(self, items):
         for header in items.keys():
             key = header.lower()
             # get all headers with the same name
             values = items.getallmatchingheaders(header)
             if len(values) > 1:
                 # collect the multiple variations of the current header
                 multiples = []
                 # loop through the values from getallmatchingheaders
                 for value in values:
                     # getallmatchingheaders returns raw header lines, so
                     # split to get name, value
-                    multiples.append(maybedecode(value).split(':', 1)[1].strip())
-                headers = multiples
+                    multiples.append(value.split(':', 1)[1].strip())
+                dict.__setitem__(self, key, multiples)
             else:
-                headers = [maybedecode(items[header])]
-            dict.__setitem__(self, maybedecode(key), headers)
+                dict.__setitem__(self, key, [items[header]])
 
 
     def __getitem__(self, key):
         """Get all headers of a certain (case-insensitive) name. If there is
         more than one, the values are returned comma separated"""
-        key = maybedecode(key)
         values = dict.__getitem__(self, key.lower())
         if len(values) == 1:
             return values[0]
         else:
             return ", ".join(values)
 
     def __setitem__(self, name, value):
         raise Exception
@@ -410,27 +398,25 @@ class RequestHeaders(dict):
         try:
             return self[key]
         except KeyError:
             return default
 
     def get_list(self, key, default=missing):
         """Get all the header values for a particular field name as
         a list"""
-        key = maybedecode(key)
         try:
             return dict.__getitem__(self, key.lower())
         except KeyError:
             if default is not missing:
                 return default
             else:
                 raise
 
     def __contains__(self, key):
-        key = maybedecode(key)
         return dict.__contains__(self, key.lower())
 
     def iteritems(self):
         for item in self:
             yield item, self[item]
 
     def itervalues(self):
         for item in self:
@@ -608,19 +594,17 @@ class Authentication(object):
     def __init__(self, headers):
         self.username = None
         self.password = None
 
         auth_schemes = {"Basic": self.decode_basic}
 
         if "authorization" in headers:
             header = headers.get("authorization")
-            assert isinstance(header, text_type)
             auth_type, data = header.split(" ", 1)
             if auth_type in auth_schemes:
                 self.username, self.password = auth_schemes[auth_type](data)
             else:
                 raise HTTPException(400, "Unsupported authentication scheme %s" % auth_type)
 
     def decode_basic(self, data):
-        assert isinstance(data, text_type)
-        decoded_data = base64.decodestring(data.encode("utf-8"))
-        return decoded_data.decode("utf-8").split(":", 1)
+        decoded_data = base64.decodestring(data)
+        return decoded_data.split(":", 1)
--- a/testing/web-platform/tests/tools/wptserve/wptserve/response.py
+++ b/testing/web-platform/tests/tools/wptserve/wptserve/response.py
@@ -178,20 +178,18 @@ class Response(object):
         and the resulting value (if any) returned.
 
         :param read_file: - boolean controlling the behaviour when content
         is a file handle. When set to False the handle will be returned directly
         allowing the file to be passed to the output in small chunks. When set to
         True, the entire content of the file will be returned as a string facilitating
         non-streaming operations like template substitution.
         """
-        if isinstance(self.content, binary_type):
+        if isinstance(self.content, (binary_type, text_type)):
             yield self.content
-        elif isinstance(self.content, text_type):
-            yield self.content.encode("utf-8")
         elif hasattr(self.content, "read"):
             if read_file:
                 yield self.content.read()
             else:
                 yield self.content
         else:
             for item in self.content:
                 if hasattr(item, "__call__"):