Bug 1461925 [wpt PR 11028] - Generalize subsetTest() in common/ and use for WebCryptoAPI tests, a=testonly
authorSimon Pieters <zcorpan@gmail.com>
Sat, 19 May 2018 20:34:07 +0000
changeset 419083 f8c7a3a58dbd51c01ef30b024638228aaec58694
parent 419082 1b0def1ce2fb31395e4baa1f2e81fc82cdf8f146
child 419084 5ec35f197df64c2c1901fd05999e837bb1349175
push id34026
push userapavel@mozilla.com
push dateMon, 21 May 2018 09:47:33 +0000
treeherdermozilla-central@dc1868d255be [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1461925, 11028, 10890
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1461925 [wpt PR 11028] - Generalize subsetTest() in common/ and use for WebCryptoAPI tests, a=testonly Automatic update from web-platform-testsGeneralize subsetTest() in common/ and use for WebCryptoAPI tests (#11028) Fixes #10890. -- wpt-commits: e9c44b8cf4d6b3d7c7f2d5d629e83c616ea903e1 wpt-pr: 11028
testing/web-platform/meta/MANIFEST.json
testing/web-platform/tests/WebCryptoAPI/generateKey/successes.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-CBC.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-CTR.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-GCM.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-KW.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_ECDH.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_ECDSA.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_HMAC.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSA-OAEP.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSA-PSS.https.any.js
testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js
testing/web-platform/tests/common/subset-tests.js
testing/web-platform/tests/docs/_writing-tests/testharness.md
--- a/testing/web-platform/meta/MANIFEST.json
+++ b/testing/web-platform/meta/MANIFEST.json
@@ -193257,16 +193257,21 @@
      {}
     ]
    ],
    "common/stringifiers.js.headers": [
     [
      {}
     ]
    ],
+   "common/subset-tests.js": [
+    [
+     {}
+    ]
+   ],
    "common/test-setting-immutable-prototype.js": [
     [
      {}
     ]
    ],
    "common/test-setting-immutable-prototype.js.headers": [
     [
      {}
@@ -405743,57 +405748,57 @@
    "dc18838f5c4f84436c892051a728de7ad83fe854",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/failures_RSASSA-PKCS1-v1_5.https.any.js": [
    "170964b911cd3d1aacaa261de01b013ee3fd745a",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes.js": [
-   "db8091eaf59ea342ae7c8f64e6548c976b5f0b36",
+   "f0f112ddcf43a46cfccc0e7cf29b46208f87bb5f",
    "support"
   ],
   "WebCryptoAPI/generateKey/successes_AES-CBC.https.any.js": [
-   "71f7d2beac74da25795366722aea79fb88c197ab",
+   "49ce28516303c7525f74d29fbbe36b578e9e0d08",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_AES-CTR.https.any.js": [
-   "57f1bff81966f8fc20b7870ed1a5ecf939f1c4d1",
+   "377e8f0ec969f36949c9dc08ae935063d25c08d8",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_AES-GCM.https.any.js": [
-   "faa70a7dda91b0602987ec8e7ef0787c4ed163e3",
+   "a65a71f8f83b017fe9eb8fccbd8da3489ed1f9f0",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_AES-KW.https.any.js": [
-   "23b4567c82e12847f41fb6aa8a6a2cb6bd219a94",
+   "165159903403f67cc27d29f78d46369eb598811c",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_ECDH.https.any.js": [
-   "51f5990d71b6d0008163c5550dfe3ca88fe56356",
+   "525b874eff610c6db8cca7f50ba0c074b8975001",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_ECDSA.https.any.js": [
-   "bfce352ef583a31aaae4a5a303a8b98737cfc22f",
+   "2a63f8d0d12987ca2e4c41e5eb28a53741a091f5",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_HMAC.https.any.js": [
-   "64b1846740147585fc4d497ba8d20a4b5bdef315",
+   "af064fd9cc422c35718d1dcf2f6c861aab7385cc",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_RSA-OAEP.https.any.js": [
-   "ccb63d91473cb7ee6554ad80f0ee2be004e75c91",
+   "f82ce4307fea94a097ccdf470b766f110d876b87",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_RSA-PSS.https.any.js": [
-   "9549e735a6d5d2a570f2b6ec4b09c49fd6a88f51",
+   "8ff33e42d855b7482455ebb3caab5ac311e7cbf5",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js": [
-   "11aff41d669c5a44c5929ac8754d680b675f3c8c",
+   "fbbbef333461a7845ea1b3cba2169435c7f82761",
    "testharness"
   ],
   "WebCryptoAPI/getRandomValues.any.js": [
    "44475110f616c78bc04b93246f015e7a83e78fa7",
    "testharness"
   ],
   "WebCryptoAPI/idlharness.https.html": [
    "a0ac6f063c49d9c14b614dee18b681968eee05ad",
@@ -410590,16 +410595,20 @@
   "common/stringifiers.js": [
    "4c6d28a4b6d982edeb5232fa624936254f2df5f0",
    "support"
   ],
   "common/stringifiers.js.headers": [
    "e3593850f8098d3f3ff82c042deab15f51c66a52",
    "support"
   ],
+  "common/subset-tests.js": [
+   "ee72de5b5f2302d111a0f131eddf5081ad02b789",
+   "support"
+  ],
   "common/test-setting-immutable-prototype.js": [
    "2cafc5c2b867e0fd6f738b1fbeaa503761c400b7",
    "support"
   ],
   "common/test-setting-immutable-prototype.js.headers": [
    "e3593850f8098d3f3ff82c042deab15f51c66a52",
    "support"
   ],
@@ -556383,17 +556392,17 @@
    "2ffb3871fb13e6acd3171205bf80517dca06bcfe",
    "support"
   ],
   "docs/_writing-tests/testharness-api.md": [
    "6e49764657b4aa160f3db1a85f92910bfccae1e2",
    "support"
   ],
   "docs/_writing-tests/testharness.md": [
-   "6d581e1021a2401b34c0c024990b5e85dbef1fa5",
+   "0a0d88c5ddde91514a51d607405c2b7dedee4776",
    "support"
   ],
   "docs/_writing-tests/visual.md": [
    "43ab66e46d0a59851c3ad9d11cc0e2d5dd3ca8c5",
    "support"
   ],
   "docs/assets/_reftest_graph_example.dot": [
    "bc8ebc09e30ff18994b32aa9e5ea43334c276ade",
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes.js
@@ -65,41 +65,23 @@ function run_test(algorithmNames, slowTe
                     assert_goodCryptoKey(result, algorithm, extractable, usages, "secret");
                 }
             }, function(err) {
                 assert_unreached("Threw an unexpected error: " + err.toString());
             });
         }, testTag + ": generateKey" + parameterString(algorithm, extractable, usages));
     }
 
-    // Only test a subset of tests with, e.g., ?1-10 in the URL
-    var subTestStart = 0;
-    var subTestEnd = Infinity;
-    var match;
-    if (location.search) {
-        match = /^\?(\d+)-(\d+|last)$/.exec(location.search);
-        if (match) {
-          subTestStart = match[1];
-          if (match[2] !== "last") {
-              subTestEnd = match[2];
-          }
-        }
-    }
-    var currentSubTest = 0;
-
     // Test all valid sets of parameters for successful
     // key generation.
     testVectors.forEach(function(vector) {
         allNameVariants(vector.name, slowTest).forEach(function(name) {
             allAlgorithmSpecifiersFor(name).forEach(function(algorithm) {
                 allValidUsages(vector.usages, false, vector.mandatoryUsages).forEach(function(usages) {
                     [false, true].forEach(function(extractable) {
-                        currentSubTest++;
-                        if (currentSubTest >= subTestStart && currentSubTest <= subTestEnd) {
-                            testSuccess(algorithm, extractable, usages, vector.resultType, "Success");
-                        }
+                        subsetTest(testSuccess, algorithm, extractable, usages, vector.resultType, "Success");
                     });
                 });
             });
         });
     });
 
 }
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-CBC.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-CBC.https.any.js
@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["AES-CBC"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-CTR.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-CTR.https.any.js
@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["AES-CTR"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-GCM.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-GCM.https.any.js
@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["AES-GCM"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-KW.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-KW.https.any.js
@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["AES-KW"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_ECDH.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_ECDH.https.any.js
@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["ECDH"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_ECDSA.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_ECDSA.https.any.js
@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["ECDSA"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_HMAC.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_HMAC.https.any.js
@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["HMAC"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSA-OAEP.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSA-OAEP.https.any.js
@@ -11,10 +11,11 @@
 // META: variant=?91-100
 // META: variant=?101-110
 // META: variant=?111-120
 // META: variant=?121-130
 // META: variant=?131-140
 // META: variant=?141-150
 // META: variant=?151-last
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["RSA-OAEP"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSA-PSS.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSA-PSS.https.any.js
@@ -1,8 +1,9 @@
 // META: timeout=long
 // META: variant=?1-10
 // META: variant=?11-20
 // META: variant=?21-30
 // META: variant=?31-last
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["RSA-PSS"]);
--- a/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js
+++ b/testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js
@@ -1,8 +1,9 @@
 // META: timeout=long
 // META: variant=?1-10
 // META: variant=?11-20
 // META: variant=?21-30
 // META: variant=?31-last
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["RSASSA-PKCS1-v1_5"]);
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/tests/common/subset-tests.js
@@ -0,0 +1,33 @@
+// Only test a subset of tests with, e.g., ?1-10 in the URL.
+// Can be used together with <meta name="variant" content="...">
+// Sample usage:
+// for (const test of tests) {
+//   subsetTest(async_test, test.fn, test.name);
+// }
+(function() {
+  var subTestStart = 0;
+  var subTestEnd = Infinity;
+  var match;
+  if (location.search) {
+      match = /(?:^\?|&)(\d+)-(\d+|last)(?:&|$)/.exec(location.search);
+      if (match) {
+        subTestStart = parseInt(match[1], 10);
+        if (match[2] !== "last") {
+            subTestEnd = parseInt(match[2], 10);
+        }
+      }
+  }
+  function shouldRunSubTest(currentSubTest) {
+    return currentSubTest >= subTestStart && currentSubTest <= subTestEnd;
+  }
+  var currentSubTest = 0;
+  function subsetTest(testFunc, ...args) {
+    currentSubTest++;
+    if (shouldRunSubTest(currentSubTest)) {
+      return testFunc(...args);
+    }
+    return null;
+  }
+  self.shouldRunSubTest = shouldRunSubTest;
+  self.subsetTest = subsetTest;
+})();
--- a/testing/web-platform/tests/docs/_writing-tests/testharness.md
+++ b/testing/web-platform/tests/docs/_writing-tests/testharness.md
@@ -14,16 +14,54 @@ documented in two sections:
     to the library and a detailed API reference.
 
   * [idlharness.js Documentation][idlharness] — A library for testing
      IDL interfaces using `testharness.js`.
 
 See [server features][] for advanced testing features that are commonly used
 with testharness.js. See also the [general guidelines][] for all test types.
 
+## Variants
+
+A test file can have multiple variants by including `meta` elements,
+for example:
+
+```
+<meta name="variant" content="">
+<meta name="variant" content="?wss">
+```
+
+The test can then do different things based on the URL.
+
+There is a utility script in `/common/subset-tests.js` that works
+well together with variants, where a test that would otherwise have
+too many tests to be useful can be split up in ranges of subtests.
+For example:
+
+```
+<!doctype html>
+<title>Testing variants</title>
+<meta name="variant" content="?1-1000">
+<meta name="variant" content="?1001-2000">
+<meta name="variant" content="?2001-last">
+<script src="/resources/testharness.js">
+<script src="/resources/testharnessreport.js">
+<script src="/common/subset-tests.js">
+<script>
+ const tests = [
+                 { fn: t => { ... }, name: "..." },
+                 ... lots of tests ...
+               ];
+ for (const test of tests) {
+   subsetTest(async_test, test.fn, test.name);
+ }
+</script>
+```
+
+
 ## Auto-generated test boilerplate
 
 While most JavaScript tests require a certain amount of HTML
 boilerplate to include the test library, etc., tests which are
 expressible purely in script (e.g. tests for workers) can have all the
 needed HTML and script boilerplate auto-generated.
 
 ### Standalone window tests
@@ -116,17 +154,17 @@ Use `// META: script=link/to/resource.js
     // META: script=resources/utils.js
 
 can be used to include both the global and a local `utils.js` in a test.
 
 ### Specifying a timeout of long in auto-generated boilerplate tests
 
 Use `// META: timeout=long` at the beginning of the resource.
 
-### Specifying test variants in auto-generated boilerplate tests
+### Specifying test [variants](#variants) in auto-generated boilerplate tests
 
 Use `// META: variant=url-suffix` at the beginning of the resource. For example,
 
     // META: variant=
     // META: variant=?wss
 
 
 [general guidelines]: {{ site.baseurl }}{% link _writing-tests/general-guidelines.md %}