Bug 1592781 use promise_test() to schedule TaskRunner Tasks r=jgraham
authorKarl Tomlinson <karlt+@karlt.net>
Wed, 13 Nov 2019 02:16:07 +0000
changeset 502533 195a6104b5fdd7b3f9d04d1ce8e0e86bdbb29869
parent 502532 fbf7d8f9ba38924f9a71fd5ffed8441ab97bcbf0
child 502534 98ce0aadba7a30c5ca695468bf8561d1c911b038
push id114172
push userdluca@mozilla.com
push dateTue, 19 Nov 2019 11:31:10 +0000
treeherdermozilla-inbound@b5c5ba07d3db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1592781
milestone72.0a1
Bug 1592781 use promise_test() to schedule TaskRunner Tasks r=jgraham promise_test() provides that a subtest is generated for each Task and run even when an exception is thrown while running a previous Task. This change also provides that Tasks that do not complete are reported TIMEOUT, andTasks that are not reached due to TIMEOUT in a previous test are reported NOTRUN. The previous logic in Task.run() to generate a subtest for reporting a PASS after calling _taskFunction() had a flaw that calls to done() during _taskFunction() would synchronously call _testharnessDone() before the subtest was generated. Differential Revision: https://phabricator.services.mozilla.com/D51563
testing/web-platform/meta/webaudio/the-audio-api/the-audiocontext-interface/audiocontext-suspend-resume.html.ini
testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworklet-suspend.https.html.ini
testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-automatic-pull.https.html.ini
testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-channel-count.https.html.ini
testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-disconnected-input.https.html.ini
testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-onerror.https.html.ini
testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletprocessor-options.https.html.ini
testing/web-platform/meta/webaudio/the-audio-api/the-pannernode-interface/panner-automation-position.html.ini
testing/web-platform/tests/webaudio/resources/audit.js
--- a/testing/web-platform/meta/webaudio/the-audio-api/the-audiocontext-interface/audiocontext-suspend-resume.html.ini
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-audiocontext-interface/audiocontext-suspend-resume.html.ini
@@ -6,8 +6,18 @@
   [X p1 instanceof Promise is not true. Got false.]
     expected: FAIL
 
   [Executing "test-suspend"]
     expected: FAIL
 
   [< [test-suspend\] 2 out of 4 assertions were failed.]
     expected: FAIL
+
+  [Executing "test-after-close"]
+    expected: TIMEOUT
+
+  [Audit report]
+    expected: NOTRUN
+
+  [Executing "resume-running-context"]
+    expected: NOTRUN
+
--- a/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworklet-suspend.https.html.ini
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworklet-suspend.https.html.ini
@@ -1,3 +1,12 @@
 [audioworklet-suspend.https.html]
   expected:
     if release_or_beta: ERROR
+  [Executing "load-worklet-and-suspend"]
+    bug: AudioWorklet not enabled on release_or_beta
+    expected:
+      if release_or_beta: TIMEOUT
+
+  [Audit report]
+    expected:
+      if release_or_beta: NOTRUN
+
--- a/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-automatic-pull.https.html.ini
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-automatic-pull.https.html.ini
@@ -1,9 +1,15 @@
 [audioworkletnode-automatic-pull.https.html]
   expected:
     if release_or_beta: OK
     ERROR
   [Executing "setup-worklet"]
     bug: AudioWorklet not enabled on release_or_beta
     expected:
       if release_or_beta: FAIL
+      TIMEOUT
 
+  [Audit report]
+    expected:
+      if release_or_beta: PASS
+      NOTRUN
+
--- a/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-channel-count.https.html.ini
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-channel-count.https.html.ini
@@ -1,16 +1,17 @@
 [audioworkletnode-channel-count.https.html]
   expected:
     if release_or_beta: OK
     ERROR
   [Executing "setup-buffer-and-worklet"]
     bug: AudioWorklet not enabled on release_or_beta
     expected:
       if release_or_beta: FAIL
+      TIMEOUT
 
   [X Second half of Channel #0: Expected 1 for all values but found 128 unexpected values: \n\tIndex\tActual\n\t[0\]\t0\n\t[1\]\t0\n\t[2\]\t0\n\t[3\]\t0\n\t...and 124 more errors.]
     bug: AudioWorklet not enabled on release_or_beta
     expected:
       if release_or_beta: FAIL
 
   [X Second half of Channel #1: Expected 2 for all values but found 128 unexpected values: \n\tIndex\tActual\n\t[0\]\t0\n\t[1\]\t0\n\t[2\]\t0\n\t[3\]\t0\n\t...and 124 more errors.]
     bug: AudioWorklet not enabled on release_or_beta
@@ -27,8 +28,18 @@
     expected:
       if release_or_beta: FAIL
 
   [# AUDIT TASK RUNNER FINISHED: 1 out of 2 tasks were failed.]
     bug: AudioWorklet not enabled on release_or_beta
     expected:
       if release_or_beta: FAIL
 
+  [Executing "verify-rendered-buffer"]
+    expected:
+      if release_or_beta: PASS
+      NOTRUN
+
+  [Audit report]
+    expected:
+      if release_or_beta: PASS
+      NOTRUN
+
--- a/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-disconnected-input.https.html.ini
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-disconnected-input.https.html.ini
@@ -1,9 +1,15 @@
 [audioworkletnode-disconnected-input.https.html]
   expected:
     if release_or_beta: OK
     ERROR
   [Executing "test"]
     bug: AudioWorklet not enabled on release_or_beta
     expected:
       if release_or_beta: FAIL
+      TIMEOUT
 
+  [Audit report]
+    expected:
+      if release_or_beta: PASS
+      NOTRUN
+
--- a/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-onerror.https.html.ini
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletnode-onerror.https.html.ini
@@ -1,5 +1,14 @@
 [audioworkletnode-onerror.https.html]
   bug: AudioWorklet not enabled on release_or_beta
   expected:
     if release_or_beta: ERROR
     TIMEOUT
+  [Audit report]
+    expected: NOTRUN
+
+  [Executing "constructor-error"]
+    expected: TIMEOUT
+
+  [Executing "process-error"]
+    expected: NOTRUN
+
--- a/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletprocessor-options.https.html.ini
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-audioworklet-interface/audioworkletprocessor-options.https.html.ini
@@ -1,14 +1,21 @@
 [audioworkletprocessor-options.https.html]
   expected:
     if release_or_beta: OK
     ERROR
   [Executing "valid-processor-data"]
     bug: AudioWorklet not enabled on release_or_beta
     expected:
       if release_or_beta: FAIL
+      TIMEOUT
 
   [Executing "empty-option"]
     bug: AudioWorklet not enabled on release_or_beta
     expected:
       if release_or_beta: FAIL
+      NOTRUN
 
+  [Audit report]
+    expected:
+      if release_or_beta: PASS
+      NOTRUN
+
--- a/testing/web-platform/meta/webaudio/the-audio-api/the-pannernode-interface/panner-automation-position.html.ini
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-pannernode-interface/panner-automation-position.html.ini
@@ -1,2 +1,29 @@
 [panner-automation-position.html]
   expected: ERROR
+  [Executing "1: 1-channel inverse rolloff: 1"]
+    expected: NOTRUN
+
+  [Executing "3: 1-channel linear rolloff: 1"]
+    expected: NOTRUN
+
+  [Executing "3: 2-channel linear rolloff: 1"]
+    expected: NOTRUN
+
+  [Audit report]
+    expected: NOTRUN
+
+  [Executing "2: 1-channel exponential rolloff: 1.5"]
+    expected: NOTRUN
+
+  [Executing "1: 2-channel inverse rolloff: 1"]
+    expected: NOTRUN
+
+  [Executing "2: 2-channel exponential rolloff: 1.5"]
+    expected: NOTRUN
+
+  [Executing "0: 2-channel inverse rolloff: 1"]
+    expected: NOTRUN
+
+  [Executing "0: 1-channel inverse rolloff: 1"]
+    expected: TIMEOUT
+
--- a/testing/web-platform/tests/webaudio/resources/audit.js
+++ b/testing/web-platform/tests/webaudio/resources/audit.js
@@ -46,22 +46,16 @@ window.Audit = (function() {
   }
 
   function _logFailed(message, detail) {
     test(function() {
       assert_true(false, detail);
     }, message);
   }
 
-  function _logException(message, exception) {
-    test(function() {
-      throw exception;
-    }, message);
-  }
-
   function _throwException(message) {
     throw new Error(message);
   }
 
   // TODO(hongchan): remove this hack after confirming all the tests are
   // finished correctly. (crbug.com/708817)
   const _testharnessDone = window.done;
   window.done = () => {
@@ -1182,38 +1176,25 @@ window.Audit = (function() {
 
       return new Should(this, actual, actualDescription);
     }
 
     // Run this task. |this| task will be passed into the user-supplied test
     // task function.
     run() {
       this._state = TaskState.STARTED;
-
       // Print out the task entry with label and description.
       _logPassed(
           '> [' + this._label + '] ' +
           (this._description ? this._description : ''));
 
-      // Ideally we would just use testharness async_test instead of reinventing
-      // that wheel, but since it's been reinvented...  At least make sure that
-      // an exception while running a task doesn't preclude us running all the
-      // _other_ tasks for the test.
-      let testName = `Executing "${this.label}"`;
-      try {
+      return new Promise(resolve => {
+        this._resolve = resolve;
         this._taskFunction(this, this.should.bind(this));
-        _logPassed(testName);
-      } catch (e) {
-        _logException(testName, e);
-        if (this.state != TaskState.FINISHED) {
-          // We threw before calling done(), so do that manually to run our
-          // other tasks.
-          this.done();
-        }
-      }
+      });
     }
 
     // Update the task success based on the individual assertion/test inside.
     update(subTask) {
       // After one of tests fails within a task, the result is irreversible.
       if (subTask.result === false) {
         this._result = false;
         this._failedAssertions++;
@@ -1233,17 +1214,17 @@ window.Audit = (function() {
             ' assertions)';
         _logPassed(message);
       } else {
         message += this._failedAssertions + ' out of ' + this._totalAssertions +
             ' assertions were failed.'
         _logFailed(message);
       }
 
-      this._taskRunner._runNextTask();
+      this._resolve();
     }
 
     // Runs |subTask| |time| milliseconds later. |setTimeout| is not allowed in
     // WPT linter, so a thin wrapper around the harness's |step_timeout| is
     // used here.
     timeout(subTask, time) {
       async_test((test) => {
         test.step_timeout(() => {
@@ -1266,30 +1247,21 @@ window.Audit = (function() {
   /**
    * @class TaskRunner
    * @description WebAudio testing task runner. Manages tasks.
    */
   class TaskRunner {
     constructor() {
       this._tasks = {};
       this._taskSequence = [];
-      this._currentTaskIndex = -1;
 
       // Configure testharness.js for the async operation.
       setup(new Function(), {explicit_done: true});
     }
 
-    _runNextTask() {
-      if (this._currentTaskIndex < this._taskSequence.length) {
-        this._tasks[this._taskSequence[this._currentTaskIndex++]].run();
-      } else {
-        this._finish();
-      }
-    }
-
     _finish() {
       let numberOfFailures = 0;
       for (let taskIndex in this._taskSequence) {
         let task = this._tasks[this._taskSequence[taskIndex]];
         numberOfFailures += task.result ? 0 : 1;
       }
 
       let prefix = '# AUDIT TASK RUNNER FINISHED: ';
@@ -1297,19 +1269,17 @@ window.Audit = (function() {
         _logFailed(
             prefix + numberOfFailures + ' out of ' + this._taskSequence.length +
             ' tasks were failed.');
       } else {
         _logPassed(
             prefix + this._taskSequence.length + ' tasks ran successfully.');
       }
 
-      // From testharness.js, report back to the test infrastructure that
-      // the task runner completed all the tasks.
-      _testharnessDone();
+      return Promise.resolve();
     }
 
     // |taskLabel| can be either a string or a dictionary. See Task constructor
     // for the detail.
     define(taskLabel, taskFunction) {
       let task = new Task(this, taskLabel, taskFunction);
       if (this._tasks.hasOwnProperty(task.label)) {
         _throwException('Audit.define:: Duplicate task definition.');
@@ -1342,19 +1312,29 @@ window.Audit = (function() {
         }
       }
 
       if (this._taskSequence.length === 0) {
         _throwException('Audit.run:: no task to run.');
         return;
       }
 
-      // Start the first task.
-      this._currentTaskIndex = 0;
-      this._runNextTask();
+      for (let taskIndex in this._taskSequence) {
+        let task = this._tasks[this._taskSequence[taskIndex]];
+        // Some tests assume that tasks run in sequence, which is provided by
+        // promise_test().
+        promise_test(() => task.run(), `Executing "${task.label}"`);
+      }
+
+      // Schedule a summary report on completion.
+      promise_test(() => this._finish(), "Audit report");
+
+      // From testharness.js. The harness now need not wait for more subtests
+      // to be added.
+      _testharnessDone();
     }
   }
 
   /**
    * Load file from a given URL and pass ArrayBuffer to the following promise.
    * @param  {String} fileUrl file URL.
    * @return {Promise}
    *