testing/talos/talos/tests/tart/tart.html
author Morgan Reschenberg <mreschenberg@mozilla.com>
Tue, 18 Jan 2022 23:31:21 +0000
changeset 605085 4c684d5d16fe3a85614e7cb56be8c0029dc98b12
parent 456101 73a91e84dbec4fe4cd6881c0bdb377f7b0137e42
permissions -rw-r--r--
Bug 1748878: Use LayoutDeviceInt types for points, rects, sizes in dev pixels r=Jamie,emilio a=graft Differential Revision: https://phabricator.services.mozilla.com/D135272

<html>
<head>

<meta charset="UTF-8"/>
<link id="tart-icon" rel="icon" href="tart.ico"/>
<title>TART - Tab Animation Regression Test</title>

<script type="application/x-javascript">

function $(id) {
  return document.getElementById(id);
}

// Executes command at the chrome process.
// Limited to one argument (data), which is enough for TART.
// doneCallback will be called once done and, if applicable, with the result as argument.
// Execution might finish quickly (e.g. when setting prefs) or
// take a while (e.g. when triggering the test run)
let _replyId = 1;
function chromeExec(commandName, data, doneCallback) {
  let replyEvent = `tart@mozilla.org:chrome-exec-reply:${_replyId++}`;
  if (doneCallback) {
    addEventListener(replyEvent, e => { doneCallback(e.detail); },
                     {once: true});
  }

  dispatchEvent(
    new CustomEvent("tart@mozilla.org:chrome-exec-event", {
      bubbles: true,
      detail: {
        command: {
          name: commandName,
          data,
        },
        replyEvent,
      },
    })
  );
}

function setASAP() {
  chromeExec("setASAP");
}

function unsetASAP() {
  chromeExec("unsetASAP");
}

function toClipboard(text) {
  chromeExec("toClipboard", text);
}

function runTest(config, doneCallback) {
  chromeExec("runTest", config, doneCallback);
}

// Returns a Promise that resolves when the test extension is loaded.
function waitForLoad() {
  async function tryPing() {
    let pingPromise = new Promise(resolve => chromeExec("ping", null, resolve));
    let timeoutPromise = new Promise((resolve, reject) => setTimeout(reject, 500));

    try {
      await Promise.race([pingPromise, timeoutPromise]);
    } catch (e) {
      return tryPing();
    }
    return null;
  }
  return tryPing();
}


function sum(values) {
  return values.reduce(function(a, b) { return a + b; });
}

function average(values) {
  return values.length ? sum(values) / values.length : 999999999;
}

function stddev(values, avg) {
  if (undefined == avg) avg = average(values);
  if (values.length <= 1) return 0;

  return Math.sqrt(
    values.map(function(v) { return Math.pow(v - avg, 2); })
          .reduce(function(a, b) { return a + b; }) / (values.length - 1));
}

var lastResults = '["[no results collected]"]';

function doneTest(dispResult) {
  $("hide-during-run").style.display = "block";
  $("show-during-run").style.display = "none";
  if (dispResult) {
    // Array of test results, each element has .name and .value (test name and test result).
    // Test result may also be an array of numeric values (all the intervals)

    lastResults = JSON.stringify(dispResult); // for "Copy to clipboard" button

    var stats = {}; // Used for average, stddev when repeat!=1
    var isRepeat = false;

    for (var i in dispResult) {
      var di = dispResult[i];
      var disp = [].concat(di.value).map(function(a) { return " " + (isNaN(a) ? -1 : a.toFixed(1)); }).join("&nbsp;&nbsp;");
      dispResult[i] = String(di.name) + ": " + disp;
      if (di.name.includes(".half") || di.name.includes(".all"))
        dispResult[i] = "<b>" + dispResult[i] + "</b>";
      if (di.name.includes(".raw"))
        dispResult[i] = "<br/>" + dispResult[i]; // Add space before raw results (which are the first result of an animation)

      // stats:
      if (!di.name.includes(".raw")) {
        if (!stats[di.name]) {
          stats[di.name] = [];
        } else {
          isRepeat = true;
        }

        stats[di.name].push(di.value);
      }
    }

    var dispStats = "";
    if (isRepeat) {
      dispStats = "<hr/><b>Aggregated</b>:<br/>";
      for (var s in stats) {
        if (s.includes(".half") )
          dispStats += "<br/>";
        dispStats += s + "&nbsp;&nbsp;&nbsp;&nbsp;Average (" + stats[s].length + "): " + average(stats[s]).toFixed(2) + " stddev: " + stddev(stats[s]).toFixed(2) + "<br/>";
      }

      dispStats += "<hr/><b>Individual animations</b>:<br/>";
    }

    // eslint-disable-next-line no-unsanitized/property
    $("run-results").innerHTML = "<hr/><br/>Results <button onclick='toClipboard(lastResults)'>[ Copy to clipboard as JSON ]</button>:<br/>" + dispStats + dispResult.join("<br/>");

    let testNames = [], testResults = [];
    for (let result of JSON.parse(lastResults)) {
      if (!Array.isArray(result.value)) {
        testNames.push(result.name);
        testResults.push(result.value);
      }
    }
    window.tpRecordTime(testResults.join(","), 0, testNames.join(","));
  }
}

var config = {subtests: [], repeat: 1}; // Empty subtests interpreted as all subtests, since otherwise meaningless.

function triggerStart() {
  updateConfig();
  $("hide-during-run").style.display = "none";
  $("show-during-run").style.display = "block";
  $("run-results").innerHTML = "";

  waitForLoad().then(() => {
    runTest(config, doneTest);
  });
}

var defaultConfig = {
      repeat: 1,
      rest: 500,
      tickle: true,
      controlProfiler: true,  // If true, pause the profiler when not measuring. Else just add markers.
      subtests: {
        simple: true,
        iconDpi1: true,
        iconDpi2: true,
        iconFadeDpi2: true,
        newtabNoPreload: true,
        newtabYesPreload: true,
        simple3open3closeDpiCurrent: false,
        multi: false,
        simpleFadeDpiCurrent: false,
        iconFadeDpiCurrent: false,
        lastTabFadeDpiCurrent: false,
        customize: false,
      },
    };

var simpleInfo = "Measure open/close of a new tab of about:blank";
var iconInfo   = "Measure open/close of a new empty tab with favicon and long title";
var newtabInfo = "Measure open of the standard about:newtab";
var fadeInfo   = "Open a new tab, then measure Fade-out/in";
var dpi1Info   = " (@DPI 1.0)";
var dpi2Info   = " (@DPI 2.0)";
var dpiCurrentInfo = " (@DPI unchanged)";

var testsInfo = {
      simple: simpleInfo + dpi1Info,
      iconDpi1: iconInfo + dpi1Info,
      iconDpi2: iconInfo + dpi2Info,
      iconFadeDpi2: fadeInfo + dpi2Info,
      newtabNoPreload: newtabInfo + " (without preload)",
      newtabYesPreload: newtabInfo + " (with preload)",
      simple3open3closeDpiCurrent: "Measure 3 tab opens and 3 tab closes" + dpiCurrentInfo,
      multi: "Open 6 tabs, then measures open/close of the 7th tab (@DPI 1.0 and 2.0)",
      simpleFadeDpiCurrent: fadeInfo + dpiCurrentInfo,
      iconFadeDpiCurrent: fadeInfo + dpiCurrentInfo,
      lastTabFadeDpiCurrent: "Focus the last tab, then measure Fade-out/in (requires to manually add a tab before testing)",
      customize: "Measure (Australis) Customize-mode enter/exit",
};


function deselectAll() {
  for (var test in defaultConfig.subtests) {
    $("subtest-" + test).checked = false;
  }
}

function updateConfig() {
  config = {subtests: []};
  for (var test in defaultConfig.subtests) {
    if ($("subtest-" + test).checked) {
      config.subtests.push(test);
    }
  }

  var repeat = $("repeat").value;
  config.repeat = isNaN(repeat) ? 1 : repeat;

  var rest = $("rest").value;
  config.rest = isNaN(rest) ? 500 : rest; // 500ms default, use 1ms as minimum
  config.rest = config.rest ? config.rest : 1;

  config.tickle = $("tickle").checked;

  config.controlProfiler = $("controlProfiler").checked;
}

// E.g. returns "world" for key "hello", "2014" for key "year", and "" for key "dummy":
// http://localhost/x.html#hello=world&x=12&year=2014
function getUriHashValue(key) {
  var k = String(key) + "=";
  var uriVars = unescape(document.location.hash).substr(1).split("&");
  for (var i in uriVars) {
    if (uriVars[i].indexOf(k) == 0)
      return uriVars[i].substr(k.length);
  }
  return "";
}

// URL e.g. chrome://tart/content/tart.html#auto&tests=["simple","iconFadeDpiCurrent"]
// Note - there's no error checking for arguments parsing errors.
//        Any errors will express as either javascript errors or not reading the args correctly.
//        This is not an "official" part of the UI, and when used in talos, will fail early
//        enough to not cause "weird" issues too late.
function updateOptionsFromUrl() {
  var uriTests = getUriHashValue("tests");
  var tests = uriTests ? JSON.parse(uriTests) : [];

  if (tests.length) {
    for (var d in defaultConfig.subtests) {
      $("subtest-" + d).checked = false;
      for (var t in tests) {
        if (tests[t] == d) {
          $("subtest-" + d).checked = true;
        }
      }
    }
  }

  var cp = getUriHashValue("controlProfiler");
  if (cp.length)
    $("controlProfiler").checked = (cp == "true");
}

function init() {
  updateOptionsFromUrl();
  if (document.location.hash.indexOf("#auto") == 0) {
    triggerStart();
  }
}

addEventListener("load", init);

</script>
</head>
<body style="font-family:sans-serif;">
<h4>TART - Tab Animation Regression Tests</h4>
<div id="hide-during-run">
   Visit <a href="https://wiki.mozilla.org/Performance_sheriffing/Talos/Tests#TART">talos/TART</a> for detailed info.<br/>
  <ul>
    <li><b>If you just opened the browser</b> - give Firefox few seconds to settle down before testing.</li>
    <li><button onclick="setASAP()">Set ASAP mode</button> <button onclick="unsetASAP()">Restore default</button> (requires restart to take effect). TART runs best (and in talos) with vsync disabled - to measure maximum throughput (ASAP mode). This means the preferences layout.frame_rate = 0 and docshell.event_starvation_delay_hint = 1</li>
    <li>In talos, TART runs with a single tab. If you wish to test a specific animation with several tabs open, you can do so as well by manually opening few tabs before starting the test.</li>
  </ul>

Utilities:
  <a href="blank.icon.html">blank with icon</a>&nbsp;&nbsp;&nbsp;
  <a href="about:config?filter=/newtab|_rate|devP|offmain|docshell.event_starvation_delay_hint|rce-en/">about:config (already filtered with relevant prefs)</a>
<br/><br/>
<b>Configure TART</b> (CTRL-F5 to reset to talos defaults) <button type="button" onclick="deselectAll()">Deselect all tests</button><br/>
<script>
  for (var test in defaultConfig.subtests) {
    // eslint-disable-next-line no-unsanitized/method
    document.write('<input type="checkbox" id="subtest-' + test + '" ' + (defaultConfig.subtests[test] ? "" : "un") + "checked>"
                  + test + "</input>"
                  + '<span style="color:grey">&nbsp;&nbsp;&nbsp;' + testsInfo[test] + "</span>"
                  + "<br/>");
  }
  $("subtest-simple3open3closeDpiCurrent").checked = false; // Disabled by default for talos
</script>
  <br/>
  Repeat: <input id="repeat" type="text" size=2 value="1" onchange="updateConfig()"/> times<br/>
  Delay before starting a measured animation: <input id="rest" type="text" size=4 value="500"/> ms<br/>
  <input id="controlProfiler" type="checkbox" checked>Pause the profiler during animations which are <b>not</b> measured.</input><br/>
  <input id="tickle" type="checkbox" checked>Accurate first recorded frame (tickle the Back button before measurements)</iinput><br/>

  <button type="button" id="start-test-button" onclick="triggerStart()">Start Tab Animation tests</button>&nbsp;&nbsp;&nbsp;
  <div id="run-results"></div>
</div>
<div id="show-during-run" style="display:none">Testing in progress ...</div>
</body>
</html>