author | Geoffrey Sneddon <me@gsnedders.com> |
Fri, 05 Oct 2018 14:21:00 +0000 | |
changeset 440060 | 11e1636ba8c8b755caf68ae3da829680713ecb87 |
parent 440059 | 2978f836d767a5df733f210dd303a3af33f1c918 |
child 440061 | 8f0661c28e345aea427515929e1a1fadba3312c4 |
push id | 34806 |
push user | nerli@mozilla.com |
push date | Tue, 09 Oct 2018 04:03:56 +0000 |
treeherder | mozilla-central@6a6c984745ef [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | testonly |
bugs | 1494822, 13248, 11769, 13204 |
milestone | 64.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/web-platform/tests/tools/wptserve/tests/functional/base.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/base.py @@ -70,17 +70,17 @@ class TestUsingServer(unittest.TestCase) for name, value in iteritems(headers): req.add_header(name, value) if body is not None: req.add_data(body) if auth is not None: - req.add_header("Authorization", b"Basic %s" % base64.b64encode(("%s:%s" % auth).encode("utf-8"))) + req.add_header("Authorization", "Basic %s" % base64.b64encode('%s:%s' % auth)) return urlopen(req) @pytest.mark.skipif(not wptserve.utils.http2_compatible(), reason="h2 server only works in python 2.7.15") class TestUsingH2Server: def setup_method(self, test_method): self.server = wptserve.server.WebTestHttpd(host="localhost",
--- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py @@ -83,26 +83,29 @@ class TestFileHandler(TestUsingServer): self.request("/document.txt", headers={"Range":"bytes=11-10"}) self.assertEqual(cm.exception.code, 416) expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() with self.assertRaises(HTTPError) as cm: self.request("/document.txt", headers={"Range":"bytes=%i-%i" % (len(expected), len(expected) + 10)}) self.assertEqual(cm.exception.code, 416) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_config(self): resp = self.request("/sub.sub.txt") expected = b"localhost localhost %i" % self.server.port assert resp.read().rstrip() == expected + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_headers(self): resp = self.request("/sub_headers.sub.txt", headers={"X-Test": "PASS"}) expected = b"PASS" assert resp.read().rstrip() == expected + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_params(self): resp = self.request("/sub_params.sub.txt", query="test=PASS") expected = b"PASS" assert resp.read().rstrip() == expected class TestFunctionHandler(TestUsingServer): def test_string_rv(self):
--- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py @@ -52,76 +52,84 @@ class TestSlice(TestUsingServer): self.assertEqual(resp.read(), expected[1:]) def test_no_lower(self): resp = self.request("/document.txt", query="pipe=slice(null,10)") expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() self.assertEqual(resp.read(), expected[:10]) class TestSub(TestUsingServer): + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_config(self): resp = self.request("/sub.txt", query="pipe=sub") - expected = b"localhost localhost %i" % self.server.port + expected = "localhost localhost %i" % self.server.port self.assertEqual(resp.read().rstrip(), expected) @pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_file_hash(self): resp = self.request("/sub_file_hash.sub.txt") - expected = b""" + expected = """ md5: JmI1W8fMHfSfCarYOSxJcw== sha1: nqpWqEw4IW8NjD6R375gtrQvtTo= sha224: RqQ6fMmta6n9TuA/vgTZK2EqmidqnrwBAmQLRQ== sha256: G6Ljg1uPejQxqFmvFOcV/loqnjPTW5GSOePOfM/u0jw= sha384: lkXHChh1BXHN5nT5BYhi1x67E1CyYbPKRKoF2LTm5GivuEFpVVYtvEBHtPr74N9E -sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICzJwGFonfXwg==""" +sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICz +JwGFonfXwg==""" self.assertEqual(resp.read().rstrip(), expected.strip()) def test_sub_file_hash_unrecognized(self): with self.assertRaises(urllib.error.HTTPError): self.request("/sub_file_hash_unrecognized.sub.txt") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_headers(self): resp = self.request("/sub_headers.txt", query="pipe=sub", headers={"X-Test": "PASS"}) - expected = b"PASS" + expected = "PASS" self.assertEqual(resp.read().rstrip(), expected) @pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_location(self): resp = self.request("/sub_location.sub.txt?query_string") expected = """ host: localhost:{0} hostname: localhost path: /sub_location.sub.txt pathname: /sub_location.sub.txt port: {0} query: ?query_string scheme: http -server: http://localhost:{0}""".format(self.server.port).encode("ascii") +server: http://localhost:{0}""".format(self.server.port) self.assertEqual(resp.read().rstrip(), expected.strip()) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_params(self): resp = self.request("/sub_params.txt", query="test=PASS&pipe=sub") - expected = b"PASS" + expected = "PASS" self.assertEqual(resp.read().rstrip(), expected) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_url_base(self): resp = self.request("/sub_url_base.sub.txt") - self.assertEqual(resp.read().rstrip(), b"Before / After") + self.assertEqual(resp.read().rstrip(), "Before / After") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_uuid(self): resp = self.request("/sub_uuid.sub.txt") - self.assertRegexpMatches(resp.read().rstrip(), b"Before [a-f0-9-]+ After") + self.assertRegexpMatches(resp.read().rstrip(), r"Before [a-f0-9-]+ After") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_var(self): resp = self.request("/sub_var.sub.txt") port = self.server.port - print(port, type(port)) - expected = b"localhost %d A %d B localhost C" % (port, port) + expected = "localhost %s A %s B localhost C" % (port, port) self.assertEqual(resp.read().rstrip(), expected) class TestTrickle(TestUsingServer): def test_trickle(self): #Actually testing that the response trickles in is not that easy t0 = time.time() resp = self.request("/document.txt", query="pipe=trickle(1:d2:5:d1:r2)") t1 = time.time() @@ -131,76 +139,84 @@ class TestTrickle(TestUsingServer): def test_headers(self): resp = self.request("/document.txt", query="pipe=trickle(d0.01)") self.assertEqual(resp.info()["Cache-Control"], "no-cache, no-store, must-revalidate") self.assertEqual(resp.info()["Pragma"], "no-cache") self.assertEqual(resp.info()["Expires"], "0") class TestPipesWithVariousHandlers(TestUsingServer): + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_file_handler(self): resp = self.request("/test_string.py", query="pipe=slice(null,2)") - self.assertEqual(resp.read(), b"PA") + self.assertEqual(resp.read(), "PA") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_func_handler(self): @wptserve.handlers.handler def handler(request, response): return "PASS" route = ("GET", "/test/test_pipes_1/", handler) self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") - self.assertEqual(resp.read(), b"PA") + self.assertEqual(resp.read(), "PA") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_func_handler_using_response_writer(self): @wptserve.handlers.handler def handler(request, response): response.writer.write_content("PASS") route = ("GET", "/test/test_pipes_1/", handler) self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") # slice has not been applied to the response, because response.writer was used. - self.assertEqual(resp.read(), b"PASS") + self.assertEqual(resp.read(), "PASS") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_header_pipe_with_python_func_using_response_writer(self): @wptserve.handlers.handler def handler(request, response): response.writer.write_content("CONTENT") route = ("GET", "/test/test_pipes_1/", handler) self.server.router.register(*route) resp = self.request(route[1], query="pipe=header(X-TEST,FAIL)") # header pipe was ignored, because response.writer was used. self.assertFalse(resp.info().get("X-TEST")) - self.assertEqual(resp.read(), b"CONTENT") + self.assertEqual(resp.read(), "CONTENT") + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_json_handler(self): @wptserve.handlers.json_handler def handler(request, response): return json.dumps({'data': 'PASS'}) route = ("GET", "/test/test_pipes_2/", handler) self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") - self.assertEqual(resp.read(), b'"{') + self.assertEqual(resp.read(), '"{') + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_slice_with_as_is_handler(self): resp = self.request("/test.asis", query="pipe=slice(null,2)") self.assertEqual(202, resp.getcode()) self.assertEqual("Giraffe", resp.msg) self.assertEqual("PASS", resp.info()["X-Test"]) # slice has not been applied to the response, because response.writer was used. - self.assertEqual(b"Content", resp.read()) + self.assertEqual("Content", resp.read()) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_headers_with_as_is_handler(self): resp = self.request("/test.asis", query="pipe=header(X-TEST,FAIL)") self.assertEqual(202, resp.getcode()) self.assertEqual("Giraffe", resp.msg) # header pipe was ignored. self.assertEqual("PASS", resp.info()["X-TEST"]) - self.assertEqual(b"Content", resp.read()) + self.assertEqual("Content", resp.read()) + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_trickle_with_as_is_handler(self): t0 = time.time() resp = self.request("/test.asis", query="pipe=trickle(1:d2:5:d1:r2)") t1 = time.time() - self.assertTrue(b'Content' in resp.read()) + self.assertTrue('Content' in resp.read()) self.assertGreater(6, t1-t0) if __name__ == '__main__': unittest.main()
--- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py @@ -1,8 +1,10 @@ +import sys + import pytest wptserve = pytest.importorskip("wptserve") from .base import TestUsingServer from wptserve.request import InputFile class TestInputFile(TestUsingServer): @@ -110,18 +112,19 @@ class TestRequest(TestUsingServer): route = ("GET", "/test/{match}_*", handler) self.server.router.register(*route) resp = self.request("/test/some_route") self.assertEqual(b"some route", resp.read()) class TestAuth(TestUsingServer): + @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_auth(self): @wptserve.handlers.handler def handler(request, response): return " ".join((request.auth.username, request.auth.password)) route = ("GET", "/test/test_auth", handler) self.server.router.register(*route) resp = self.request(route[1], auth=("test", "PASS")) self.assertEqual(200, resp.getcode()) - self.assertEqual([b"test", b"PASS"], resp.read().split(b" ")) + self.assertEqual(["test", "PASS"], resp.read().split(" "))
--- a/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py +++ b/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py @@ -1,11 +1,10 @@ from cgi import escape from collections import deque -import base64 import gzip as gzip_module import hashlib import os import re import time import uuid from six.moves import StringIO @@ -269,19 +268,18 @@ def slice(request, response, start, end= :param start: The starting offset. Follows python semantics including negative numbers. :param end: The ending offset, again with python semantics and None (spelled "null" in a query string) to indicate the end of the file. """ - content = resolve_content(response)[start:end] - response.content = content - response.headers.set("Content-Length", len(content)) + content = resolve_content(response) + response.content = content[start:end] return response class ReplacementTokenizer(object): def arguments(self, token): unwrapped = token[1:-1].decode('utf8') return ("arguments", re.split(r",\s*", unwrapped) if unwrapped else []) @@ -389,49 +387,50 @@ class SubFunctions(object): # are available on all platforms [1]. This ensures that test authors do not # unknowingly introduce platform-specific tests. # # [1] https://docs.python.org/2/library/hashlib.html supported_algorithms = ("md5", "sha1", "sha224", "sha256", "sha384", "sha512") @staticmethod def file_hash(request, algorithm, path): - assert isinstance(algorithm, text_type) + algorithm = algorithm.decode("ascii") if algorithm not in SubFunctions.supported_algorithms: raise ValueError("Unsupported encryption algorithm: '%s'" % algorithm) hash_obj = getattr(hashlib, algorithm)() absolute_path = os.path.join(request.doc_root, path) try: - with open(absolute_path, "rb") as f: + with open(absolute_path) as f: hash_obj.update(f.read()) except IOError: # In this context, an unhandled IOError will be interpreted by the # server as an indication that the template file is non-existent. # Although the generic "Exception" is less precise, it avoids # triggering a potentially-confusing HTTP 404 error in cases where # the path to the file to be hashed is invalid. raise Exception('Cannot open file for hash computation: "%s"' % absolute_path) - return base64.b64encode(hash_obj.digest()).strip() + return hash_obj.digest().encode('base64').strip() def template(request, content, escape_type="html"): #TODO: There basically isn't any error handling here tokenizer = ReplacementTokenizer() variables = {} def config_replacement(match): content, = match.groups() tokens = tokenizer.tokenize(content) tokens = deque(tokens) token_type, field = tokens.popleft() + field = field.decode("ascii") if token_type == "var": variable = field token_type, field = tokens.popleft() else: variable = None if token_type != "ident": @@ -486,21 +485,17 @@ def template(request, content, escape_ty if variable is not None: variables[variable] = value escape_func = {"html": lambda x:escape(x, quote=True), "none": lambda x:x}[escape_type] #Should possibly support escaping for other contexts e.g. script #TODO: read the encoding of the response - if isinstance(value, binary_type): - value = value.decode("utf-8") - elif isinstance(value, int): - value = text_type(value) - return escape_func(value).encode("utf-8") + return escape_func(text_type(value)).encode("utf-8") template_regexp = re.compile(br"{{([^}]*)}}") new_content = template_regexp.sub(config_replacement, content) return new_content @pipe() def gzip(request, response):
--- a/testing/web-platform/tests/tools/wptserve/wptserve/request.py +++ b/testing/web-platform/tests/tools/wptserve/wptserve/request.py @@ -1,12 +1,12 @@ import base64 import cgi from six.moves.http_cookies import BaseCookie -from six import BytesIO, binary_type, text_type +from six import BytesIO import tempfile from six.moves.urllib.parse import parse_qsl, urlsplit from . import stash from .utils import HTTPException missing = object() @@ -313,18 +313,18 @@ class Request(object): self._POST = MultiDict.from_field_storage(fs) self.raw_input.seek(pos) return self._POST @property def cookies(self): if self._cookies is None: parser = BaseCookie() - cookie_headers = self.headers.get("cookie", u"") - parser.load(cookie_headers.encode("utf-8")) + cookie_headers = self.headers.get("cookie", "") + parser.load(cookie_headers) cookies = Cookies() for key, value in parser.iteritems(): cookies[key] = CookieValue(value) self._cookies = cookies return self._cookies @property def headers(self): @@ -350,51 +350,39 @@ class Request(object): class H2Request(Request): def __init__(self, request_handler): self.h2_stream_id = request_handler.h2_stream_id self.frames = [] super(H2Request, self).__init__(request_handler) -def maybedecode(s): - if isinstance(s, text_type): - return s - - if isinstance(s, binary_type): - return s.decode("ascii") - - raise TypeError("Unexpected value in RequestHeaders: %r" % s) - - class RequestHeaders(dict): """Dictionary-like API for accessing request headers.""" def __init__(self, items): for header in items.keys(): key = header.lower() # get all headers with the same name values = items.getallmatchingheaders(header) if len(values) > 1: # collect the multiple variations of the current header multiples = [] # loop through the values from getallmatchingheaders for value in values: # getallmatchingheaders returns raw header lines, so # split to get name, value - multiples.append(maybedecode(value).split(':', 1)[1].strip()) - headers = multiples + multiples.append(value.split(':', 1)[1].strip()) + dict.__setitem__(self, key, multiples) else: - headers = [maybedecode(items[header])] - dict.__setitem__(self, maybedecode(key), headers) + dict.__setitem__(self, key, [items[header]]) def __getitem__(self, key): """Get all headers of a certain (case-insensitive) name. If there is more than one, the values are returned comma separated""" - key = maybedecode(key) values = dict.__getitem__(self, key.lower()) if len(values) == 1: return values[0] else: return ", ".join(values) def __setitem__(self, name, value): raise Exception @@ -410,27 +398,25 @@ class RequestHeaders(dict): try: return self[key] except KeyError: return default def get_list(self, key, default=missing): """Get all the header values for a particular field name as a list""" - key = maybedecode(key) try: return dict.__getitem__(self, key.lower()) except KeyError: if default is not missing: return default else: raise def __contains__(self, key): - key = maybedecode(key) return dict.__contains__(self, key.lower()) def iteritems(self): for item in self: yield item, self[item] def itervalues(self): for item in self: @@ -608,19 +594,17 @@ class Authentication(object): def __init__(self, headers): self.username = None self.password = None auth_schemes = {"Basic": self.decode_basic} if "authorization" in headers: header = headers.get("authorization") - assert isinstance(header, text_type) auth_type, data = header.split(" ", 1) if auth_type in auth_schemes: self.username, self.password = auth_schemes[auth_type](data) else: raise HTTPException(400, "Unsupported authentication scheme %s" % auth_type) def decode_basic(self, data): - assert isinstance(data, text_type) - decoded_data = base64.decodestring(data.encode("utf-8")) - return decoded_data.decode("utf-8").split(":", 1) + decoded_data = base64.decodestring(data) + return decoded_data.split(":", 1)
--- a/testing/web-platform/tests/tools/wptserve/wptserve/response.py +++ b/testing/web-platform/tests/tools/wptserve/wptserve/response.py @@ -178,20 +178,18 @@ class Response(object): and the resulting value (if any) returned. :param read_file: - boolean controlling the behaviour when content is a file handle. When set to False the handle will be returned directly allowing the file to be passed to the output in small chunks. When set to True, the entire content of the file will be returned as a string facilitating non-streaming operations like template substitution. """ - if isinstance(self.content, binary_type): + if isinstance(self.content, (binary_type, text_type)): yield self.content - elif isinstance(self.content, text_type): - yield self.content.encode("utf-8") elif hasattr(self.content, "read"): if read_file: yield self.content.read() else: yield self.content else: for item in self.content: if hasattr(item, "__call__"):