Bug 1464234 [wpt PR 11150] - [testharness.js] Extend test suite with "variants", a=testonly
authorMike Pennisi <mike@mikepennisi.com>
Fri, 06 Jul 2018 13:50:00 +0000
changeset 425819 ae907cfd98e11126bd6134999d8ae828e79eee4c
parent 425818 cbbb092d298d1adec1e96637d339d707230f367a
child 425820 5ffe8b1cf2ea7b3a385aeb5f4122625213075e20
push id34267
push userrgurzau@mozilla.com
push dateWed, 11 Jul 2018 22:05:21 +0000
treeherdermozilla-central@3aca103e4915 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1464234, 11150
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1464234 [wpt PR 11150] - [testharness.js] Extend test suite with "variants", a=testonly Automatic update from web-platform-tests[testharness.js] Extend test suite with "variants" Feature description (from modification to `README.md` file): > `testharness.js` is expected to function consistently in a number of > distinct environments. In order to verify this expectation, each functional > test may be executed under a number of distinct conditions. These conditions > are applied using WPT's "test variants" pattern. The available variants are > defined in the `variants.js` file; this file must be included before > `testharness.js`. Every test must specify at least one variant. Extend test harness for `testharness.js` with additional validation to ensure that each test is correctly formatted. Introduce a minimal UI to help future contributors discover this new functionality and its motivation. -- wpt-commits: 08995880734011eed359c4e79420db29626016ff wpt-pr: 11150
testing/web-platform/tests/resources/test/README.md
testing/web-platform/tests/resources/test/conftest.py
testing/web-platform/tests/resources/test/tests/functional/add_cleanup.html
testing/web-platform/tests/resources/test/tests/functional/add_cleanup_count.html
testing/web-platform/tests/resources/test/tests/functional/add_cleanup_err.html
testing/web-platform/tests/resources/test/tests/functional/add_cleanup_err_multi.html
testing/web-platform/tests/resources/test/tests/functional/api-tests-1.html
testing/web-platform/tests/resources/test/tests/functional/api-tests-2.html
testing/web-platform/tests/resources/test/tests/functional/api-tests-3.html
testing/web-platform/tests/resources/test/tests/functional/force_timeout.html
testing/web-platform/tests/resources/test/tests/functional/generate-callback.html
testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlDictionary/test_partial_interface_of.html
testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_immutable_prototype.html
testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_partial_interface_of.html
testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_primary_interface_of.html
testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_to_json_operation.html
testing/web-platform/tests/resources/test/tests/functional/iframe-callback.html
testing/web-platform/tests/resources/test/tests/functional/iframe-consolidate-errors.html
testing/web-platform/tests/resources/test/tests/functional/iframe-consolidate-tests.html
testing/web-platform/tests/resources/test/tests/functional/iframe-msg.html
testing/web-platform/tests/resources/test/tests/functional/order.html
testing/web-platform/tests/resources/test/tests/functional/promise-async.html
testing/web-platform/tests/resources/test/tests/functional/promise.html
testing/web-platform/tests/resources/test/tests/functional/single-page-test-fail.html
testing/web-platform/tests/resources/test/tests/functional/single-page-test-no-assertions.html
testing/web-platform/tests/resources/test/tests/functional/single-page-test-no-body.html
testing/web-platform/tests/resources/test/tests/functional/single-page-test-pass.html
testing/web-platform/tests/resources/test/tests/functional/uncaught-exception-handle.html
testing/web-platform/tests/resources/test/tests/functional/uncaught-exception-ignore.html
testing/web-platform/tests/resources/test/tests/functional/worker-dedicated.html
testing/web-platform/tests/resources/test/tests/functional/worker-service.html
testing/web-platform/tests/resources/test/tests/functional/worker-shared.html
testing/web-platform/tests/resources/test/variants.js
--- a/testing/web-platform/tests/resources/test/README.md
+++ b/testing/web-platform/tests/resources/test/README.md
@@ -76,8 +76,15 @@ must include a summary of the expected r
           "properties": {},
           "stack": "(implementation-defined)",
           "status_string": "FAIL"
         }
       ],
       "type": "complete"
     }
     </script>
+
+`testharness.js` is expected to function consistently in a number of
+distinct environments. In order to verify this expectation, each functional
+test may be executed under a number of distinct conditions. These conditions
+are applied using WPT's "test variants" pattern. The available variants are
+defined in the `variants.js` file; this file must be included before
+`testharness.js`. Every test must specify at least one variant.
--- a/testing/web-platform/tests/resources/test/conftest.py
+++ b/testing/web-platform/tests/resources/test/conftest.py
@@ -7,16 +7,17 @@ import pytest
 from selenium import webdriver
 
 from wptserver import WPTServer
 
 ENC = 'utf8'
 HERE = os.path.dirname(os.path.abspath(__file__))
 WPT_ROOT = os.path.normpath(os.path.join(HERE, '..', '..'))
 HARNESS = os.path.join(HERE, 'harness.html')
+TEST_TYPES = ('functional', 'unit')
 
 def pytest_addoption(parser):
     parser.addoption("--binary", action="store", default=None, help="path to browser binary")
 
 def pytest_collect_file(path, parent):
     if path.ext.lower() != '.html':
         return
 
@@ -27,43 +28,70 @@ def pytest_collect_file(path, parent):
 
 def pytest_configure(config):
     config.driver = webdriver.Firefox(firefox_binary=config.getoption("--binary"))
     config.server = WPTServer(WPT_ROOT)
     config.server.start()
     config.add_cleanup(config.server.stop)
     config.add_cleanup(config.driver.quit)
 
+def resolve_uri(context, uri):
+    if uri.startswith('/'):
+        base = WPT_ROOT
+        path = uri[1:]
+    else:
+        base = os.path.dirname(context)
+        path = uri
+
+    return os.path.exists(os.path.join(base, path))
+
 class HTMLItem(pytest.Item, pytest.Collector):
     def __init__(self, filename, test_type, parent):
         self.filename = filename
         self.type = test_type
+        self.variants = []
 
-        if test_type not in ('functional', 'unit'):
+        if test_type not in TEST_TYPES:
             raise ValueError('Unrecognized test type: "%s"' % test_type)
 
         with io.open(filename, encoding=ENC) as f:
             markup = f.read()
 
         parsed = html5lib.parse(markup, namespaceHTMLElements=False)
         name = None
+        includes_variants_script = False
         self.expected = None
 
         for element in parsed.getiterator():
             if not name and element.tag == 'title':
                 name = element.text
                 continue
-            if element.attrib.get('id') == 'expected':
-                self.expected = json.loads(unicode(element.text))
+            if element.tag == 'meta' and element.attrib.get('name') == 'variant':
+                self.variants.append(element.attrib.get('content'))
                 continue
+            if element.tag == 'script':
+                if element.attrib.get('id') == 'expected':
+                    self.expected = json.loads(unicode(element.text))
+
+                src = element.attrib.get('src', '')
+
+                if 'variants.js' in src:
+                    includes_variants_script = True
+                    if not resolve_uri(filename, src):
+                        raise ValueError('Could not resolve path "%s" from %s' % (src, filename))
 
         if not name:
             raise ValueError('No name found in file: %s' % filename)
-        elif self.type == 'functional' and not self.expected:
-            raise ValueError('Functional tests must specify expected report data')
+        elif self.type == 'functional':
+            if not self.expected:
+                raise ValueError('Functional tests must specify expected report data')
+            if not includes_variants_script:
+                raise ValueError('No variants script found in file: %s' % filename)
+            if len(self.variants) == 0:
+                raise ValueError('No test variants specified in file %s' % filename)
         elif self.type == 'unit' and self.expected:
             raise ValueError('Unit tests must not specify expected report data')
 
         super(HTMLItem, self).__init__(name, parent)
 
 
     def reportinfo(self):
         return self.fspath, None, self.filename
@@ -90,22 +118,27 @@ class HTMLItem(pytest.Item, pytest.Colle
         summarized = self._summarize(actual)
 
         assert summarized[u'summarized_status'][u'status_string'] == u'OK', summarized[u'summarized_status'][u'message']
         for test in summarized[u'summarized_tests']:
             msg = "%s\n%s" % (test[u'name'], test[u'message'])
             assert test[u'status_string'] == u'PASS', msg
 
     def _run_functional_test(self):
+        for variant in self.variants:
+            self._run_functional_test_variant(variant)
+
+    def _run_functional_test_variant(self, variant):
         driver = self.session.config.driver
         server = self.session.config.server
 
         driver.get(server.url(HARNESS))
 
-        actual = driver.execute_async_script('runTest("%s", "foo", arguments[0])' % server.url(str(self.filename)))
+        test_url = server.url(str(self.filename) + variant)
+        actual = driver.execute_async_script('runTest("%s", "foo", arguments[0])' % test_url)
 
         # Test object ordering is not guaranteed. This weak assertion verifies
         # that the indices are unique and sequential
         indices = [test_obj.get('index') for test_obj in actual['tests']]
         self._assert_sequence(indices)
 
         summarized = self._summarize(actual)
 
--- a/testing/web-platform/tests/resources/test/tests/functional/add_cleanup.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/add_cleanup.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Test#add_cleanup</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <div id="log"></div>
 <script>
 "use strict";
 
--- a/testing/web-platform/tests/resources/test/tests/functional/add_cleanup_count.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/add_cleanup_count.html
@@ -1,12 +1,14 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Test#add_cleanup reported count</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <div id="log"></div>
 
 <script>
 promise_test(function(t) {
--- a/testing/web-platform/tests/resources/test/tests/functional/add_cleanup_err.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/add_cleanup_err.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Test#add_cleanup: error</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <div id="log"></div>
 <script>
 "use strict";
 
--- a/testing/web-platform/tests/resources/test/tests/functional/add_cleanup_err_multi.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/add_cleanup_err_multi.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Test#add_cleanup: multiple functions with one in error</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <div id="log"></div>
 
 <script>
 "use strict";
--- a/testing/web-platform/tests/resources/test/tests/functional/api-tests-1.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/api-tests-1.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Sample HTML5 API Tests</title>
+<script src="../../variants.js"></script>
 <meta name="timeout" content="6000">
 </head>
 <body onload="load_test_attr.done()">
 <h1>Sample HTML5 API Tests</h1>
 <div id="log"></div>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
--- a/testing/web-platform/tests/resources/test/tests/functional/api-tests-2.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/api-tests-2.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Sample HTML5 API Tests</title>
+<script src="../../variants.js"></script>
 </head>
 <body>
 <h1>Sample HTML5 API Tests</h1>
 <p>There should be two results</p>
 <div id="log"></div>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
--- a/testing/web-platform/tests/resources/test/tests/functional/api-tests-3.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/api-tests-3.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Sample HTML5 API Tests</title>
+<script src="../../variants.js"></script>
 </head>
 <script src="/resources/testharness.js"></script>
 
 <body>
 <h1>Sample HTML5 API Tests</h1>
 <div id="log"></div>
 <script>
 setup({explicit_timeout:true});
--- a/testing/web-platform/tests/resources/test/tests/functional/force_timeout.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/force_timeout.html
@@ -1,12 +1,14 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Test#force_timeout</title>
+<script src="../../variants.js"></script>
 </head>
 <body>
 <h1>Test#force_timeout</h1>
 <div id="log"></div>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
 setup({ explicit_timeout: true });
--- a/testing/web-platform/tests/resources/test/tests/functional/generate-callback.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/generate-callback.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Sample for using generate_tests to create a series of tests that share the same callback.</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <script>
 // generate_tests takes an array of arrays that define tests
 // but lets pass it an empty array and verify it does nothing.
 function null_callback() {
--- a/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlDictionary/test_partial_interface_of.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlDictionary/test_partial_interface_of.html
@@ -1,14 +1,17 @@
 <!DOCTYPE HTML>
 <html>
 
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>idlharness: Partial dictionary</title>
+  <script src="/resources/test/variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>
   <script src="/resources/idlharness.js"></script>
 </head>
 
 <body>
   <p>Verify the series of sub-tests that are executed for "partial" dictionary objects.</p>
--- a/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_immutable_prototype.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_immutable_prototype.html
@@ -1,13 +1,16 @@
 <!DOCTYPE HTML>
 <html>
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>idlharness: Immutable prototypes</title>
+  <script src="../../../../variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>
   <script src="/resources/idlharness.js"></script>
 </head>
 <body>
 <script>
 "use strictt";
--- a/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_partial_interface_of.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_partial_interface_of.html
@@ -1,14 +1,17 @@
 <!DOCTYPE HTML>
 <html>
 
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>idlharness: Partail interface</title>
+  <script src="/resources/test/variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>
   <script src="/resources/idlharness.js"></script>
 </head>
 
 <body>
   <p>Verify the series of sub-tests that are executed for "partial" interface objects.</p>
--- a/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_primary_interface_of.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_primary_interface_of.html
@@ -1,13 +1,16 @@
 <!DOCTYPE HTML>
 <html>
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>idlharness: Primary interface</title>
+  <script src="../../../../variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>
   <script src="/resources/idlharness.js"></script>
 </head>
 <body>
 <p>Verify the series of sub-tests that are executed for "tested" interface
 objects but skipped for "untested" interface objects.</p>
--- a/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_to_json_operation.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_to_json_operation.html
@@ -1,13 +1,16 @@
 <!DOCTYPE HTML>
 <html>
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>IdlInterface.prototype.test_to_json_operation()</title>
+  <script src="../../../../variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>
   <script src="/resources/idlharness.js"></script>
   <script src="../../../../idl-helper.js"></script>
 </head>
 <body>
 <script>
--- a/testing/web-platform/tests/resources/test/tests/functional/iframe-callback.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/iframe-callback.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with iframe that notifies containing document via callbacks</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body onload="start_test_in_iframe()">
 <h1>Callbacks From Tests Running In An IFRAME</h1>
 <p>A test is run inside an <tt>iframe</tt> with a same origin document. The
 containing document should receive callbacks as the tests progress inside the
 <tt>iframe</tt>. A single passing test is expected in the summary below.
--- a/testing/web-platform/tests/resources/test/tests/functional/iframe-consolidate-errors.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/iframe-consolidate-errors.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with iframe that consolidates errors via fetch_tests_from_window</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
 var parent_test = async_test("Test executing in parent context");
 </script>
 </head>
 <body onload="parent_test.done()">
 <h1>Fetching Tests From a Child Context</h1>
--- a/testing/web-platform/tests/resources/test/tests/functional/iframe-consolidate-tests.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/iframe-consolidate-tests.html
@@ -1,27 +1,30 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with iframe that consolidates tests via fetch_tests_from_window</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
 var parent_test = async_test("Test executing in parent context");
 </script>
 </head>
 <body onload="parent_test.done()">
 <h1>Fetching Tests From a Child Context</h1>
 <p>This test demonstrates the use of <tt>fetch_tests_from_window</tt> to pull
 tests from an <tt>iframe</tt> into the primary document.</p>
 <p>The test suite will not complete until tests in the child context have finished
 executing</p>
 <div id="log"></div>
 
-<iframe id="childContext" src="promise-async.html" style="display:none"></iframe>
+<iframe id="childContext" src="promise-async.html?keep-promise" style="display:none"></iframe>
 <!-- promise-async.html has async tests with promises -->
 
 <script>
   var childContext = document.getElementById("childContext");
   fetch_tests_from_window(childContext.contentWindow);
 </script>
 <script type="text/json" id="expected">
 {
--- a/testing/web-platform/tests/resources/test/tests/functional/iframe-msg.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/iframe-msg.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with iframe that notifies containing document via cross document messaging</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <h1>Notifications From Tests Running In An IFRAME</h1>
 <p>A test is run inside an <tt>iframe</tt> with a same origin document. The
 containing document should receive messages via <tt>postMessage</tt>/
 <tt>onmessage</tt> as the tests progress inside the <tt>iframe</tt>. A single
--- a/testing/web-platform/tests/resources/test/tests/functional/order.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/order.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Ordering</title>
+<script src="../../variants.js"></script>
 <meta name="timeout" content="6000">
 </head>
 <body>
 <div id="log"></div>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
 test(function() {}, 'second');
--- a/testing/web-platform/tests/resources/test/tests/functional/promise-async.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/promise-async.html
@@ -1,12 +1,14 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Async Tests and Promises</title>
+<script src="../../variants.js"></script>
 </head>
 <body>
 <h1>Async Tests and Promises</h1>
 <p>This test assumes ECMAScript 6 Promise support. Some failures are expected.</p>
 <div id="log"></div>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
--- a/testing/web-platform/tests/resources/test/tests/functional/promise.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/promise.html
@@ -1,12 +1,14 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Promise Tests</title>
+<script src="../../variants.js"></script>
 </head>
 <body>
 <h1>Promise Tests</h1>
 <p>This test demonstrates the use of <tt>promise_test</tt>. Assumes ECMAScript 6
 Promise support. Some failures are expected.</p>
 <div id="log"></div>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
--- a/testing/web-platform/tests/resources/test/tests/functional/single-page-test-fail.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/single-page-test-fail.html
@@ -1,10 +1,13 @@
 <!DOCTYPE HTML>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with file_is_test (should fail)</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
 onload = function() {
   assert_true(false);
   done();
 }
 </script>
--- a/testing/web-platform/tests/resources/test/tests/functional/single-page-test-no-assertions.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/single-page-test-no-assertions.html
@@ -1,10 +1,13 @@
 <!DOCTYPE HTML>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example single page test with no asserts</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
 done();
 </script>
 <script type="text/json" id="expected">
 {
   "summarized_status": {
--- a/testing/web-platform/tests/resources/test/tests/functional/single-page-test-no-body.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/single-page-test-no-body.html
@@ -1,10 +1,13 @@
 <!DOCTYPE HTML>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example single page test with no body</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
 assert_true(true);
 done();
 </script>
 <script type="text/json" id="expected">
 {
--- a/testing/web-platform/tests/resources/test/tests/functional/single-page-test-pass.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/single-page-test-pass.html
@@ -1,10 +1,13 @@
 <!DOCTYPE HTML>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with file_is_test</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
 onload = function() {
   assert_true(true);
   done();
 }
 </script>
--- a/testing/web-platform/tests/resources/test/tests/functional/uncaught-exception-handle.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/uncaught-exception-handle.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Harness Handling Uncaught Exception</title>
+<script src="../../variants.js"></script>
 </head>
 <script src="/resources/testharness.js"></script>
 
 <body>
 <h1>Harness Handling Uncaught Exception</h1>
 <div id="log"></div>
 <script>
 var t = async_test("This should show a harness status of 'Error' and a test status of 'Not Run'");
--- a/testing/web-platform/tests/resources/test/tests/functional/uncaught-exception-ignore.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/uncaught-exception-ignore.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Harness Ignoring Uncaught Exception</title>
+<script src="../../variants.js"></script>
 </head>
 <script src="/resources/testharness.js"></script>
 
 <body>
 <h1>Harness Ignoring Uncaught Exception</h1>
 <div id="log"></div>
 <script>
 setup({allow_uncaught_exception:true});
--- a/testing/web-platform/tests/resources/test/tests/functional/worker-dedicated.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/worker-dedicated.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Dedicated Worker Tests</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <h1>Dedicated Web Worker Tests</h1>
 <p>Demonstrates running <tt>testharness</tt> based tests inside a dedicated web worker.
 <p>The test harness is expected to fail due to an uncaught exception in one worker.</p>
 <div id="log"></div>
--- a/testing/web-platform/tests/resources/test/tests/functional/worker-service.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/worker-service.html
@@ -1,12 +1,14 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Example with a service worker</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <h1>Service Worker Tests</h1>
 <p>Demonstrates running <tt>testharness</tt> based tests inside a service worker.
 <p>The test harness should time out due to one of the tests inside the worker timing out.
 <p>This test assumes that the browser supports <a href="http://www.w3.org/TR/service-workers/">ServiceWorkers</a>.
--- a/testing/web-platform/tests/resources/test/tests/functional/worker-shared.html
+++ b/testing/web-platform/tests/resources/test/tests/functional/worker-shared.html
@@ -1,12 +1,15 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with a shared worker</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>
 <body>
 <h1>Shared Web Worker Tests</h1>
 <p>Demonstrates running <tt>testharness</tt> based tests inside a shared worker.
 <p>The test harness should time out due to one of the tests in the worker timing out.
 <p>This test assumes that the browser supports <a href="http://www.w3.org/TR/workers/#shared-workers-and-the-sharedworker-interface">shared web workers</a>.
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/tests/resources/test/variants.js
@@ -0,0 +1,57 @@
+(function() {
+  'use strict';
+  var variants = {
+    /**
+     * Tests are executed in the absence of the global Promise constructor by
+     * default in order to verify support for the Server browser engine.
+     *
+     * https://github.com/w3c/web-platform-tests/issues/6266
+     */
+    'default': {
+      description: 'Global Promise constructor removed.',
+      apply: function() {
+        delete window.Promise;
+      }
+    },
+    /**
+     * This variant allows for testing functionality that is fundamentally
+     * dependent on Promise support, e.g. the `promise_test` function
+     */
+    'keep-promise': {
+      description: 'No modification of global environment.',
+      apply: function() {}
+    }
+  };
+  var match = window.location.search.match(/\?(.*)$/);
+  var variantName = (match && match[1]) || 'default';
+
+  if (!Object.hasOwnProperty.call(variants, variantName)) {
+    window.location = 'javascript:"Unrecognized variant: ' + variantName + '";';
+    document.close();
+    return;
+  }
+
+  if (typeof test === 'function') {
+    test(function() {
+      assert_unreached('variants.js must be included before testharness.js');
+    });
+  }
+  var variant = variants[variantName];
+
+  var variantNode = document.createElement('div');
+  variantNode.innerHTML = '<p>This testharness.js test was executed with ' +
+    'the variant named, "' + variantName + '". ' + variant.description +
+    '</p><p>Refer to the test harness README file for more information.</p>';
+  function onReady() {
+    if (document.readyState !== 'complete') {
+      return;
+    }
+
+    document.body.insertBefore(variantNode, document.body.firstChild);
+  }
+
+  onReady();
+  document.addEventListener('readystatechange', onReady);
+
+  variant.apply();
+}());