Bug 1441703 - Define all DAMP tests in a single file;r=ochameau draft
authorJulian Descottes <jdescottes@mozilla.com>
Fri, 09 Mar 2018 09:05:20 +0100
changeset 766307 e5342314a99ded5d09c0c2225a8b3de2d7630015
parent 766306 cebfd6265d88a1d5ea5135d200d115f7e5c71c94
push id102271
push userjdescottes@mozilla.com
push dateMon, 12 Mar 2018 15:21:13 +0000
reviewersochameau
bugs1441703
milestone61.0a1
Bug 1441703 - Define all DAMP tests in a single file;r=ochameau MozReview-Commit-ID: 5XLDwdfGyTh
testing/talos/talos/tests/devtools/addon/content/addon-test-frontend.js
testing/talos/talos/tests/devtools/addon/content/damp-tests.js
testing/talos/talos/tests/devtools/addon/content/damp.html
testing/talos/talos/tests/devtools/addon/content/damp.js
--- a/testing/talos/talos/tests/devtools/addon/content/addon-test-frontend.js
+++ b/testing/talos/talos/tests/devtools/addon/content/addon-test-frontend.js
@@ -111,18 +111,18 @@ function triggerStart() {
   $("hide-during-run").style.display = "none";
   $("show-during-run").style.display = "block";
   $("run-results").innerHTML = "";
 
   runTest(config, doneTest);
 }
 
 function deselectAll() {
-  for (var test in defaultConfig.subtests) {
-    $("subtest-" + test).checked = false;
+  for (var test of defaultConfig.subtests) {
+    $("subtest-" + test.name).checked = false;
   }
 }
 
 // E.g. returns "world" for key "hello", "2014" for key "year", and "" for key "dummy":
 // http://localhost/x.html#hello=world&x=12&year=2014
 function getUriHashValue(key) {
   var k = String(key) + "=";
   var uriVars = unescape(document.location.hash).substr(1).split("&");
@@ -138,21 +138,21 @@ function getUriHashValue(key) {
 //        Any errors will express as either javascript errors or not reading the args correctly.
 //        This is not an "official" part of the UI, and when used in talos, will fail early
 //        enough to not cause "weird" issues too late.
 function updateOptionsFromUrl() {
   var uriTests = getUriHashValue("tests");
   var tests = uriTests ? JSON.parse(uriTests) : [];
 
   if (tests.length) {
-    for (var d in defaultConfig.subtests) {
-      $("subtest-" + d).checked = false;
+    for (var test of defaultConfig.subtests) {
+      $("subtest-" + test.name).checked = false;
       for (var t in tests) {
-        if (tests[t] == d) {
-          $("subtest-" + d).checked = true;
+        if (tests[t] == test.name) {
+          $("subtest-" + test.name).checked = true;
         }
       }
     }
   }
 }
 
 function init() {
   updateOptionsFromUrl();
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/devtools/addon/content/damp-tests.js
@@ -0,0 +1,131 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * This is the registry for all DAMP tests. Tests will be run in the order specified by
+ * the DAMP_TESTS array.
+ *
+ * A test is defined with the following properties:
+ * - {String} name: the name of the test (should match the path when possible)
+ * - {String} path: the path to the test file under
+ *   testing/talos/talos/tests/devtools/addon/content/tests/
+ * - {String} description: Test description
+ * - {Boolean} disabled: set to true to skip the test
+ * - {Boolean} cold: set to true to run the test only during the first run of the browser
+ */
+
+window.DAMP_TESTS = [
+  {
+    name: "inspector.cold-open",
+    path: "inspector/cold-open.js",
+    description: "Measure first open toolbox on inspector panel",
+    cold: true
+  },
+  // Run all tests against "simple" document
+  {
+    name: "simple.webconsole",
+    path: "webconsole/simple.js",
+    description: "Measure open/close toolbox on webconsole panel against simple document"
+  }, {
+    name: "simple.inspector",
+    path: "inspector/simple.js",
+    description: "Measure open/close toolbox on inspector panel against simple document"
+  }, {
+    name: "simple.debugger",
+    path: "debugger/simple.js",
+    description: "Measure open/close toolbox on debugger panel against simple document"
+  }, {
+    name: "simple.styleeditor",
+    path: "styleeditor/simple.js",
+    description: "Measure open/close toolbox on style editor panel against simple document"
+  }, {
+    name: "simple.performance",
+    path: "performance/simple.js",
+    description: "Measure open/close toolbox on performance panel against simple document"
+  }, {
+    name: "simple.netmonitor",
+    path: "netmonitor/simple.js",
+    description: "Measure open/close toolbox on network monitor panel against simple document"
+  }, {
+    name: "simple.memory",
+    path: "memory/simple.js",
+    description: "Measure open/close toolbox on memory panel and save/read heap snapshot against simple document"
+  },
+  // Run all tests against "complicated" document
+  {
+    name: "complicated.webconsole",
+    path: "webconsole/complicated.js",
+    description: "Measure open/close toolbox on webconsole panel against complicated document"
+  }, {
+    name: "complicated.inspector",
+    path: "inspector/complicated.js",
+    description: "Measure open/close toolbox on inspector panel against complicated document"
+  }, {
+    name: "complicated.debugger",
+    path: "debugger/complicated.js",
+    description: "Measure open/close toolbox on debugger panel against complicated document"
+  }, {
+    name: "complicated.styleeditor",
+    path: "styleeditor/complicated.js",
+    description: "Measure open/close toolbox on style editor panel against complicated document"
+  }, {
+    name: "complicated.performance",
+    path: "performance/complicated.js",
+    description: "Measure open/close toolbox on performance panel against complicated document"
+  }, {
+    name: "complicated.netmonitor",
+    path: "netmonitor/complicated.js",
+    description: "Measure open/close toolbox on network monitor panel against complicated document"
+  }, {
+    name: "complicated.memory",
+    path: "memory/complicated.js",
+    description: "Measure open/close toolbox on memory panel and save/read heap snapshot against complicated document"
+  },
+  // Run all tests against a document specific to each tool
+  {
+    name: "custom.webconsole",
+    path: "webconsole/custom.js"
+  }, {
+    name: "custom.inspector",
+    path: "inspector/custom.js"
+  }, {
+    name: "custom.debugger",
+    path: "debugger/custom.js"
+  },
+  // Run individual tests covering a very precise tool feature.
+  {
+    name: "console.bulklog",
+    path: "webconsole/bulklog.js",
+    description: "Measure time for a bunch of sync console.log statements to appear"
+  }, {
+    name: "console.streamlog",
+    path: "webconsole/streamlog.js",
+    description: "Measure rAF on page during a stream of console.log statements"
+  }, {
+    name: "console.objectexpand",
+    path: "webconsole/objectexpand.js",
+    description: "Measure time to expand a large object and close the console"
+  }, {
+    name: "console.openwithcache",
+    path: "webconsole/openwithcache.js",
+    description: "Measure time to render last logged messages in console for a page with 100 logged messages"
+  }, {
+    name: "inspector.mutations",
+    path: "inspector/mutations.js",
+    description: "Measure the time to perform childList mutations when inspector is enabled"
+  }, {
+    name: "inspector.layout",
+    path: "inspector/layout.js",
+    description: "Measure the time to open/close toolbox on inspector with layout tab against big document with grid containers"
+  }, {
+    name: "panelsInBackground.reload",
+    path: "toolbox/panels-in-background.js",
+    description: "Measure page reload time when all panels are in background"
+  },
+  // ⚠  Adding new individual tests slows down DAMP execution ⚠
+  // ⚠  Consider contributing to custom.${tool} rather than adding isolated tests ⚠
+  // ⚠  See http://docs.firefox-dev.tools/tests/writing-perf-tests.html ⚠
+];
--- a/testing/talos/talos/tests/devtools/addon/content/damp.html
+++ b/testing/talos/talos/tests/devtools/addon/content/damp.html
@@ -1,87 +1,28 @@
 <html>
   <head>
 <meta charset="UTF-8"/>
 <title>DAMP - Devtools At Maximum Performance</title>
 
+<script src="damp-tests.js"></script>
 <script type="application/x-javascript">
 // Empty subtests interpreted as all subtests, since otherwise meaningless.
 var config = {subtests: [], repeat: 1};
 
 var defaultConfig = {
   repeat: 1,
   rest: 100,
-  subtests: {
-    "cold.inspector": true,
-
-    "simple.webconsole": true,
-    "simple.inspector": true,
-    "simple.debugger": true,
-    "simple.styleeditor": true,
-    "simple.performance": true,
-    "simple.netmonitor": true,
-    "simple.saveAndReadHeapSnapshot": true,
-
-    "complicated.webconsole": true,
-    "complicated.inspector": true,
-    "complicated.debugger": true,
-    "complicated.styleeditor": true,
-    "complicated.performance": true,
-    "complicated.netmonitor": true,
-    "complicated.saveAndReadHeapSnapshot": true,
-
-    "custom.webconsole": true,
-    "custom.inspector": true,
-    "custom.debugger": true,
-
-    "console.bulklog": true,
-    "console.streamlog": true,
-    "console.objectexpand": true,
-    "console.openwithcache": true,
-    "inspector.mutations": true,
-    "inspector.layout": true,
-
-    "panelsInBackground.reload": true,
-  }
-};
-
-var testsInfo = {
-  "cold.inspector": "Measure first open toolbox on inspector panel",
-
-  "simple.webconsole": "Measure open/close toolbox on webconsole panel against simple document",
-  "simple.inspector": "Measure open/close toolbox on inspector panel against simple document",
-  "simple.debugger": "Measure open/close toolbox on debugger panel against simple document",
-  "simple.styleeditor": "Measure open/close toolbox on style editor panel against simple document",
-  "simple.performance": "Measure open/close toolbox on performance panel against simple document",
-  "simple.netmonitor": "Measure open/close toolbox on network monitor panel against simple document",
-  "simple.saveAndReadHeapSnapshot": "Measure open/close toolbox on memory panel and save/read heap snapshot against simple document",
-
-  "complicated.webconsole": "Measure open/close toolbox on webconsole panel against complicated document",
-  "complicated.inspector": "Measure open/close toolbox on inspector panel against complicated document",
-  "complicated.debugger": "Measure open/close toolbox on debugger panel against complicated document",
-  "complicated.styleeditor": "Measure open/close toolbox on style editor panel against complicated document",
-  "complicated.performance": "Measure open/close toolbox on performance panel against complicated document",
-  "complicated.netmonitor": "Measure open/close toolbox on network monitor panel against complicated document",
-  "complicated.saveAndReadHeapSnapshot": "Measure open/close toolbox on memory panel and save/read heap snapshot against complicated document",
-
-  "console.bulklog": "Measure time for a bunch of sync console.log statements to appear",
-  "console.streamlog": "Measure rAF on page during a stream of console.log statements",
-  "console.objectexpand": "Measure time to expand a large object and close the console",
-  "console.openwithcache": "Measure time to render last logged messages in console for a page with 100 logged messages",
-  "inspector.mutations": "Measure the time to perform childList mutations when inspector is enabled",
-  "inspector.layout": "Measure the time to open/close toolbox on inspector with layout tab against big document with grid containers",
-
-  "panelsInBackground.reload": "Measure page reload time when all panels are in background",
+  subtests: window.DAMP_TESTS // from damp-tests.js
 };
 
 function updateConfig() {
   config = {subtests: []};
-  for (var test in defaultConfig.subtests) {
-    if ($("subtest-" + test).checked) { // eslint-disable-line no-undef
+  for (var test of defaultConfig.subtests) {
+    if ($("subtest-" + test.name).checked) { // eslint-disable-line no-undef
       config.subtests.push(test);
     }
   }
 
   var repeat = $("repeat").value; // eslint-disable-line no-undef
   config.repeat = isNaN(repeat) ? 1 : repeat;
 
   // use 1ms rest as a minimum.
@@ -100,23 +41,24 @@ function updateConfig() {
   </ul>
 
 Utilities:
   <a href="pages/simple.html">simple page</a>&nbsp;&nbsp;&nbsp;
   <a href="http://localhost/tests/tp5n/bild.de/www.bild.de/index.html">complicated page</a>&nbsp;&nbsp;&nbsp;
 <br/><br/>
 <b>Configure DAMP</b> (CTRL-F5 to reset to talos defaults) <button type="button" onclick="deselectAll()">Deselect all tests</button><br/>
 <script>
-  for (var test in defaultConfig.subtests) {
-
+  for (let test of defaultConfig.subtests) {
+    let checked = test.disabled ? "unchecked" : "checked";
     // eslint-disable-next-line no-unsanitized/method
-    document.write('<input type="checkbox" id="subtest-' + test + '" ' + (defaultConfig.subtests[test] ? "" : "un") + "checked>"
-                  + test + "</input>"
-                  + '<span style="color:grey">&nbsp;&nbsp;&nbsp;' + testsInfo[test] + "</span>"
-                  + "<br/>");
+    document.write(`<input type="checkbox" id="subtest-${test.name}" ${checked}>
+                      ${test.name}
+                    </input>
+                    <span style="color:grey">&nbsp;&nbsp;&nbsp;${test.description}</span>
+                    <br/>`);
   }
 </script>
   <br/>
   Repeat: <input id="repeat" type="text" size=2 value="1" onchange="updateConfig()"/> times<br/>
   Delay before starting a measured animation: <input id="rest" type="text" size=4 value="10"/> ms<br/>
 
   <button type="button" id="start-test-button" onclick="triggerStart()">Start Devtools At Maximum Performance tests</button>&nbsp;&nbsp;&nbsp;
   <div id="run-results"></div>
--- a/testing/talos/talos/tests/devtools/addon/content/damp.js
+++ b/testing/talos/talos/tests/devtools/addon/content/damp.js
@@ -76,17 +76,16 @@ function awaitBrowserLoaded(browser, inc
 
 /* globals res:true */
 
 function Damp() {
   Services.prefs.setBoolPref("devtools.webconsole.new-frontend-enabled", true);
 }
 
 Damp.prototype = {
-
   async garbageCollect() {
     dump("Garbage collect\n");
 
     // Minimize memory usage
     // mimic miminizeMemoryUsage, by only flushing JS objects via GC.
     // We don't want to flush all the cache like minimizeMemoryUsage,
     // as it slow down next executions almost like a cold start.
 
@@ -179,17 +178,17 @@ Damp.prototype = {
   async testSetup(url) {
     let tab = await this.addTab(url);
     await new Promise(resolve => {
       setTimeout(resolve, this._config.rest);
     });
     return tab;
   },
 
-  async testTeardown() {
+  async testTeardown(url) {
     this.closeCurrentTab();
 
     // Force freeing memory now so that it doesn't happen during the next test
     await this.garbageCollect();
 
     this._runNextTest();
   },
 
@@ -295,85 +294,39 @@ Damp.prototype = {
     this._config = config;
 
     this._win = Services.wm.getMostRecentWindow("navigator:browser");
     this._dampTab = this._win.gBrowser.selectedTab;
     this._win.gBrowser.selectedBrowser.focus(); // Unfocus the URL bar to avoid caret blink
 
     TalosParentProfiler.resume("DAMP - start");
 
-    let tests = {};
+    // Filter tests via `./mach --subtests filter` command line argument
+    let filter = Services.prefs.getCharPref("talos.subtests", "");
+
+    let tests = config.subtests.filter(test => !test.disabled)
+                               .filter(test => test.name.includes(filter));
+
+    if (tests.length === 0) {
+      dump("ERROR: Unable to find any test matching '" + filter + "'\n");
+    }
 
     // Run cold test only once
     let topWindow = getMostRecentBrowserWindow();
-    if (!topWindow.coldRunDAMP) {
-      topWindow.coldRunDAMP = true;
-      tests["cold.inspector"] = "inspector/cold-open.js";
-    }
-
-    tests["panelsInBackground.reload"] = "toolbox/panels-in-background.js";
-
-    // Run all tests against "simple" document
-    tests["simple.inspector"] = "inspector/simple.js";
-    tests["simple.webconsole"] = "webconsole/simple.js";
-    tests["simple.debugger"] = "debugger/simple.js";
-    tests["simple.styleeditor"] = "styleeditor/simple.js";
-    tests["simple.performance"] = "performance/simple.js";
-    tests["simple.netmonitor"] = "netmonitor/simple.js";
-    tests["simple.saveAndReadHeapSnapshot"] = "memory/simple.js";
-
-    // Run all tests against "complicated" document
-    tests["complicated.inspector"] = "inspector/complicated.js";
-    tests["complicated.webconsole"] = "webconsole/complicated.js";
-    tests["complicated.debugger"] = "debugger/complicated.js";
-    tests["complicated.styleeditor"] = "styleeditor/complicated.js";
-    tests["complicated.performance"] = "performance/complicated.js";
-    tests["complicated.netmonitor"] = "netmonitor/complicated.js";
-    tests["complicated.saveAndReadHeapSnapshot"] = "memory/complicated.js";
-
-    // Run all tests against a document specific to each tool
-    tests["custom.inspector"] = "inspector/custom.js";
-    tests["custom.debugger"] = "debugger/custom.js";
-    tests["custom.webconsole"] = "webconsole/custom.js";
-
-    // Run individual tests covering a very precise tool feature.
-    tests["console.bulklog"] = "webconsole/bulklog.js";
-    tests["console.streamlog"] = "webconsole/streamlog.js";
-    tests["console.objectexpand"] = "webconsole/objectexpand.js";
-    tests["console.openwithcache"] = "webconsole/openwithcache.js";
-    tests["inspector.mutations"] = "inspector/mutations.js";
-    tests["inspector.layout"] = "inspector/layout.js";
-    // ⚠  Adding new individual tests slows down DAMP execution ⚠
-    // ⚠  Consider contributing to custom.${tool} rather than adding isolated tests ⚠
-    // ⚠  See http://docs.firefox-dev.tools/tests/writing-perf-tests.html ⚠
-
-    // Filter tests via `./mach --subtests filter` command line argument
-    let filter = Services.prefs.getCharPref("talos.subtests", "");
-    if (filter) {
-      for (let name in tests) {
-        if (!name.includes(filter)) {
-          delete tests[name];
-        }
-      }
-      if (Object.keys(tests).length == 0) {
-        dump("ERROR: Unable to find any test matching '" + filter + "'\n");
-        this._doneInternal();
-        return;
-      }
+    if (topWindow.coldRunDAMPDone) {
+      tests = tests.filter(test => !test.cold);
+    } else {
+      topWindow.coldRunDAMPDone = true;
     }
 
     // Construct the sequence array while filtering tests
     let sequenceArray = [];
-    for (var i in config.subtests) {
-      for (var r = 0; r < config.repeat; r++) {
-        if (!config.subtests[i] || !tests[config.subtests[i]]) {
-          continue;
-        }
-
-        sequenceArray.push(tests[config.subtests[i]]);
+    for (let test of tests) {
+      for (let r = 0; r < config.repeat; r++) {
+        sequenceArray.push(test.path);
       }
     }
 
     // Free memory before running the first test, otherwise we may have a GC
     // related to Firefox startup or DAMP setup during the first test.
     this.garbageCollect().then(() => {
       this._doSequence(sequenceArray, this._doneInternal);
     }).catch(e => {