Bug 1644131 [wpt PR 24030] - Add step_wait, step_wait_func, and step_wait_func_done functions, a=testonly
authorjgraham <james@hoppipolla.co.uk>
Mon, 22 Jun 2020 10:39:09 +0000
changeset 536789 232991b4fb0b2f24756620523d4c972d243f4b83
parent 536788 39339bfe278771c6330150ec13d365fdb6e6d807
child 536790 655c8314619fe019c9234f0c3ed66bf7c84c2c09
push id37533
push userdluca@mozilla.com
push dateTue, 23 Jun 2020 21:38:40 +0000
treeherdermozilla-central@d48aa0f0aa0b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1644131, 24030
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1644131 [wpt PR 24030] - Add step_wait, step_wait_func, and step_wait_func_done functions, a=testonly Automatic update from web-platform-tests Add step_wait, step_wait_func, and step_wait_func_done functions (#24030) These allows tests that would normally use set_timeout to wait on a condition function instead. That means faster execution when the timeout is too long and more leeway for cases where the timeout would have been marginal or too short. Co-authored-by: Anne van Kesteren <annevk@annevk.nl> -- wpt-commits: 3016dfa710e2114595431f6506f1c7d5729c2484 wpt-pr: 24030
testing/web-platform/tests/docs/writing-tests/testharness-api.md
testing/web-platform/tests/lint.ignore
testing/web-platform/tests/resources/test/tests/functional/step_wait.html
testing/web-platform/tests/resources/test/tests/functional/step_wait_func.html
testing/web-platform/tests/resources/testharness.js
--- a/testing/web-platform/tests/docs/writing-tests/testharness-api.md
+++ b/testing/web-platform/tests/docs/writing-tests/testharness-api.md
@@ -387,18 +387,57 @@ In general the use of timeouts in tests 
 an observed source of instability in real tests when run on CI
 infrastructure. In particular if a test should fail when something
 doesn't happen, it is good practice to simply let the test run to the
 full timeout rather than trying to guess an appropriate shorter
 timeout to use.
 
 In other cases it may be necessary to use a timeout (e.g., for a test
 that only passes if some event is *not* fired). In this case it is
-*not* permitted to use the standard `setTimeout` function. Instead one
-must use the `step_timeout` function:
+*not* permitted to use the standard `setTimeout` function.
+
+Instead, one of these functions can be used:
+
+* `step_wait` (returns a promise)
+* `step_wait_func` & `step_wait_func_done`
+* As a last resort, `step_timeout`
+
+### `step_wait`, `step_wait_func`, and `step_wait_func_done` ###
+
+These functions are preferred over `step_timeout` as they end when a condition or a timeout is reached, rather than just a timeout. This allows for setting a longer timeout while shortening the runtime of tests on faster machines.
+
+`step_wait(cond, description, timeout=3000, interval=100)` is useful inside `promise_test`, e.g.:
+
+```js
+promise_test(t => {
+  // …
+  await t.step_wait(() => frame.contentDocument === null, "Frame navigated to a cross-origin document");
+  // …
+}, "");
+```
+
+`step_wait_func(cond, func, description, timeout=3000, interval=100)` & `step_wait_func(cond, func, description, timeout=3000, interval=100)` are useful inside `async_test`:
+
+```js
+async_test(t => {
+  const popup = window.open("resources/coop-coep.py?coop=same-origin&coep=&navigate=about:blank");
+  t.add_cleanup(() => popup.close());
+  assert_equals(window, popup.opener);
+
+  popup.onload = t.step_func(() => {
+    assert_true(popup.location.href.endsWith("&navigate=about:blank"));
+    // Use step_wait_func_done as about:blank cannot message back.
+    t.step_wait_func_done(() => popup.location.href === "about:blank");
+  });
+}, "Navigating a popup to about:blank");
+```
+
+### `step_timeout` ###
+
+As a last resort one can use the `step_timeout` function:
 
 ```js
 async_test(function(t) {
   var gotEvent = false;
   document.addEventListener("DOMContentLoaded", t.step_func(function() {
     assert_false(gotEvent, "Unexpected DOMContentLoaded event");
     gotEvent = true;
     t.step_timeout(function() { t.done(); }, 2000);
--- a/testing/web-platform/tests/lint.ignore
+++ b/testing/web-platform/tests/lint.ignore
@@ -310,16 +310,18 @@ SET TIMEOUT: html/webappapis/timers/*
 SET TIMEOUT: orientation-event/resources/orientation-event-helpers.js
 SET TIMEOUT: portals/history/resources/portal-harness.js
 SET TIMEOUT: resources/chromium/*
 SET TIMEOUT: resources/test/tests/functional/add_cleanup.html
 SET TIMEOUT: resources/test/tests/functional/add_cleanup_async.html
 SET TIMEOUT: resources/test/tests/functional/add_cleanup_async_rejection.html
 SET TIMEOUT: resources/test/tests/functional/add_cleanup_async_rejection_after_load.html
 SET TIMEOUT: resources/test/tests/functional/api-tests-1.html
+SET TIMEOUT: resources/test/tests/functional/step_wait.html
+SET TIMEOUT: resources/test/tests/functional/step_wait_func.html
 SET TIMEOUT: resources/test/tests/functional/worker.js
 SET TIMEOUT: resources/test/tests/functional/worker-uncaught-allow.js
 SET TIMEOUT: resources/test/tests/unit/exceptional-cases.html
 SET TIMEOUT: resources/test/tests/unit/exceptional-cases-timeouts.html
 SET TIMEOUT: resources/test/tests/unit/promise_setup.html
 SET TIMEOUT: resources/testharness.js
 
 # setTimeout use in reftests
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/tests/resources/test/tests/functional/step_wait.html
@@ -0,0 +1,59 @@
+<!doctype html>
+<title>Tests for step_wait</title>
+<script src="../../variants.js"></script>
+<meta name="variant" content="?keep-promise">
+<script src=/resources/testharness.js></script>
+<script src=/resources/testharnessreport.js></script>
+<div id=log></div>
+<script>
+promise_test(async t => {
+  let x = 1;
+  Promise.resolve().then(() => ++x);
+  await t.step_wait(() => x === 1);
+  assert_equals(x, 2);
+}, "Basic step_wait() test");
+
+promise_test(async t => {
+  let cond = false;
+  let x = 0;
+  setTimeout(() => cond = true, 100);
+  await t.step_wait(() => {
+    ++x;
+    return cond;
+  });
+  assert_equals(x, 2);
+}, "step_wait() isn't invoked too often");
+
+promise_test(async t => {
+  await t.step_wait(); // Throws
+}, "step_wait() takes an argument");
+</script>
+<script type="text/json" id="expected">
+{
+  "summarized_tests": [
+    {
+      "name": "Basic step_wait() test",
+      "message": null,
+      "properties": {},
+      "status_string": "PASS"
+    },
+    {
+      "name": "step_wait() isn't invoked too often",
+      "message": null,
+      "properties": {},
+      "status_string": "PASS"
+    },
+    {
+      "name": "step_wait() takes an argument",
+      "message": "cond is not a function",
+      "properties": {},
+      "status_string": "FAIL"
+    }
+  ],
+  "type": "complete",
+  "summarized_status": {
+    "status_string": "OK",
+    "message": null
+  }
+}
+</script>
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/tests/resources/test/tests/functional/step_wait_func.html
@@ -0,0 +1,50 @@
+<!doctype html>
+<title>Tests for step_wait_func and step_wait_func_done</title>
+<script src="../../variants.js"></script>
+<script src=/resources/testharness.js></script>
+<script src=/resources/testharnessreport.js></script>
+<div id=log></div>
+<script>
+async_test(t => {
+  let x = 0;
+  let step_x = 0;
+  setTimeout(() => ++x, 100);
+  t.step_wait_func(() => {
+    ++step_x;
+    return x === 1;
+  }, () => {
+    assert_equals(step_x, 2);
+    t.done();
+  });
+}, "Basic step_wait_func() test");
+
+async_test(t => {
+  let x = 0;
+  setTimeout(() => ++x, 100);
+  t.step_wait_func_done(() => true, () => assert_equals(x, 0));
+}, "Basic step_wait_func_done() test");
+
+</script>
+<script type="text/json" id="expected">
+{
+  "summarized_status": {
+    "message": null,
+    "status_string": "OK"
+  },
+  "summarized_tests": [
+    {
+      "properties": {},
+      "message": null,
+      "name": "Basic step_wait_func() test",
+      "status_string": "PASS"
+    },
+    {
+      "properties": {},
+      "message": null,
+      "name": "Basic step_wait_func_done() test",
+      "status_string": "PASS"
+    }
+  ],
+  "type": "complete"
+}
+</script>
--- a/testing/web-platform/tests/resources/testharness.js
+++ b/testing/web-platform/tests/resources/testharness.js
@@ -2030,16 +2030,115 @@ policies and contribution forms [3].
     };
 
     Test.prototype.step_timeout = function(f, timeout) {
         var test_this = this;
         var args = Array.prototype.slice.call(arguments, 2);
         return setTimeout(this.step_func(function() {
             return f.apply(test_this, args);
         }), timeout * tests.timeout_multiplier);
+    };
+
+    Test.prototype.step_wait_func = function(cond, func, description,
+                                             timeout=3000, interval=100) {
+        /**
+         * Poll for a function to return true, and call a callback
+         * function once it does, or assert if a timeout is
+         * reached. This is preferred over a simple step_timeout
+         * whenever possible since it allows the timeout to be longer
+         * to reduce intermittents without compromising test execution
+         * speed when the condition is quickly met.
+         *
+         * @param {Function} cond A function taking no arguments and
+         *                        returning a boolean. The callback is called
+         *                        when this function returns true.
+         * @param {Function} func A function taking no arguments to call once
+         *                        the condition is met.
+         * @param {string} description Error message to add to assert in case of
+         *                             failure.
+         * @param {number} timeout Timeout in ms. This is multiplied by the global
+         *                         timeout_multiplier
+         * @param {number} interval Polling interval in ms
+         *
+         **/
+
+        var timeout_full = timeout * tests.timeout_multiplier;
+        var remaining = Math.ceil(timeout_full / interval);
+        var test_this = this;
+
+        var wait_for_inner = test_this.step_func(() => {
+            if (cond()) {
+                func();
+            } else {
+                if(remaining === 0) {
+                    assert(false, "wait_for", description,
+                           "Timed out waiting on condition");
+                }
+                remaining--;
+                setTimeout(wait_for_inner, interval);
+            }
+        });
+
+        wait_for_inner();
+    };
+
+    Test.prototype.step_wait_func_done = function(cond, func, description,
+                                                  timeout=3000, interval=100) {
+        /**
+         * Poll for a function to return true, and invoke a callback
+         * followed this.done() once it does, or assert if a timeout
+         * is reached. This is preferred over a simple step_timeout
+         * whenever possible since it allows the timeout to be longer
+         * to reduce intermittents without compromising test execution speed
+         * when the condition is quickly met.
+         *
+         * @param {Function} cond A function taking no arguments and
+         *                        returning a boolean. The callback is called
+         *                        when this function returns true.
+         * @param {Function} func A function taking no arguments to call once
+         *                        the condition is met.
+         * @param {string} description Error message to add to assert in case of
+         *                             failure.
+         * @param {number} timeout Timeout in ms. This is multiplied by the global
+         *                         timeout_multiplier
+         * @param {number} interval Polling interval in ms
+         *
+         **/
+
+         this.step_wait_func(cond, () => {
+            if (func) {
+                func();
+            }
+            this.done();
+         }, description, timeout, interval);
+    }
+
+    Test.prototype.step_wait = function(cond, description, timeout=3000, interval=100) {
+        /**
+         * Poll for a function to return true, and resolve a promise
+         * once it does, or assert if a timeout is reached. This is
+         * preferred over a simple step_timeout whenever possible
+         * since it allows the timeout to be longer to reduce
+         * intermittents without compromising test execution speed
+         * when the condition is quickly met.
+         *
+         * @param {Function} cond A function taking no arguments and
+         *                        returning a boolean.
+         * @param {string} description Error message to add to assert in case of
+         *                             failure.
+         * @param {number} timeout Timeout in ms. This is multiplied by the global
+         *                         timeout_multiplier
+         * @param {number} interval Polling interval in ms
+         * @returns {Promise} Promise resolved once cond is met.
+         *
+         **/
+
+        return new Promise(resolve => {
+            this.step_wait_func(cond, resolve, description, timeout, interval);
+        });
     }
 
     /*
      * Private method for registering cleanup functions. `testharness.js`
      * internals should use this method instead of the public `add_cleanup`
      * method in order to hide implementation details from the harness status
      * message in the case errors.
      */