Bug 1299118: Measure TTI (or TTFI at the moment) in Raptor TP6 r=rwood
☠☠ backed out by 6a40850883ff ☠ ☠
authorRandell Jesup <rjesup@jesup.org>
Thu, 11 Oct 2018 13:23:38 -0400
changeset 499253 14451eb9a2b8ca1549e4fe1f6ae877fde2cf9f20
parent 499252 e5adc30bdf7f3d5c10b7160836c477c4abf0f17d
child 499254 f73e13de8e712a5188866e4331f0cc6000a568cd
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1299118
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1299118: Measure TTI (or TTFI at the moment) in Raptor TP6 r=rwood
modules/libpref/init/all.js
taskcluster/ci/test/raptor.yml
testing/profiles/raptor/user.js
testing/raptor/raptor/manifest.py
testing/raptor/raptor/tests/raptor-tp6.ini
testing/raptor/webext/raptor/measure.js
testing/raptor/webext/raptor/runner.js
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -177,18 +177,18 @@ pref("dom.performance.enable_scheduler_t
 pref("dom.permissions.revoke.enable", false);
 
 // Enable exposing timeToNonBlankPaint
 pref("dom.performance.time_to_non_blank_paint.enabled", false);
 
 // Enable exposing timeToDOMContentFlushed
 pref("dom.performance.time_to_dom_content_flushed.enabled", false);
 
-// Enable exposing timeToInteractive
-pref("dom.performance.time_to_interactive.enabled", false);
+// Enable exposing timeToFirstInteractive
+pref("dom.performance.time_to_first_interactive.enabled", false);
 
 // Enable requestIdleCallback API
 pref("dom.requestIdleCallback.enabled", true);
 
 // Enable Pointer Lock API
 // This is added for accessibility purpose. When user has no way to exit
 // pointer lock (e.g. no keyboard available), they can use this pref to
 // disable the Pointer Lock API altogether.
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -32,17 +32,17 @@ job-defaults:
                 default:
                     - raptor/linux_config.py
 
 raptor-tp6-firefox:
     description: "Raptor tp6 on Firefox"
     try-name: raptor-tp6-firefox
     treeherder-symbol: Rap(tp6)
     run-on-projects: ['try', 'mozilla-central']
-    max-run-time: 1200
+    max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-tp6
 
 raptor-tp6-chrome:
     description: "Raptor tp6 on Chrome"
     try-name: raptor-tp6-chrome
     treeherder-symbol: Rap-C(tp6)
--- a/testing/profiles/raptor/user.js
+++ b/testing/profiles/raptor/user.js
@@ -1,7 +1,8 @@
 // Preferences file used by the raptor harness
 /* globals user_pref */
 user_pref("dom.performance.time_to_non_blank_paint.enabled", true);
 user_pref("dom.performance.time_to_dom_content_flushed.enabled", true);
+user_pref("dom.performance.time_to_first_interactive.enabled", true);
 
 // required for geckoview logging
 user_pref("geckoview.console.enabled", true);
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -79,16 +79,18 @@ def write_test_settings_json(test_detail
         if "dcf" in test_details['measure']:
             test_settings['raptor-options']['measure']['dcf'] = True
         if "fnbpaint" in test_details['measure']:
             test_settings['raptor-options']['measure']['fnbpaint'] = True
         if "fcp" in test_details['measure']:
             test_settings['raptor-options']['measure']['fcp'] = True
         if "hero" in test_details['measure']:
             test_settings['raptor-options']['measure']['hero'] = test_details['hero'].split()
+        if "ttfi" in test_details['measure']:
+            test_settings['raptor-options']['measure']['ttfi'] = True
     if test_details.get("page_timeout", None) is not None:
         test_settings['raptor-options']['page_timeout'] = int(test_details['page_timeout'])
     test_settings['raptor-options']['unit'] = test_details.get("unit", "ms")
     if test_details.get("lower_is_better", "true") == "false":
         test_settings['raptor-options']['lower_is_better'] = False
     else:
         test_settings['raptor-options']['lower_is_better'] = True
 
--- a/testing/raptor/raptor/tests/raptor-tp6.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6.ini
@@ -9,39 +9,42 @@ type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
 python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
 
 [raptor-tp6-amazon-firefox]
 apps = firefox
 test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
 playback_recordings = amazon.mp
-measure = fnbpaint, hero, dcf
+measure = fnbpaint, hero, dcf, ttfi
 hero = hero1
 
 [raptor-tp6-facebook-firefox]
 apps = firefox
 test_url = https://www.facebook.com
 playback_recordings = facebook.mp
-measure = fnbpaint, hero, dcf
+measure = fnbpaint, hero, dcf, ttfi
 hero = hero1
 
 [raptor-tp6-google-firefox]
 apps = firefox
 # note: use the full url as the first part (without '&cad=h') redirects
 # to the url + '&cad=h'; that redirection causes measure.js content
 # to be loaded into that page also; resulting in 2 fnbpaint values etc.
 test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
 playback_recordings = google-search.mp
-measure = fnbpaint, hero, dcf
+measure = fnbpaint, hero, dcf, ttfi
 hero = hero1
 
 [raptor-tp6-youtube-firefox]
 apps = firefox
 test_url = https://www.youtube.com
 playback_recordings = youtube.mp
 measure = fnbpaint, hero, dcf
 hero = hero1
--- a/testing/raptor/webext/raptor/measure.js
+++ b/testing/raptor/webext/raptor/measure.js
@@ -18,16 +18,22 @@ var heroesToCapture = [];
 var getFNBPaint = false;
 
 // measure firefox domContentFlushed
 // note: this browser pref must be enabled:
 // dom.performance.time_to_dom_content_flushed.enabled = True
 // default only; this is set via control server settings json
 var getDCF = false;
 
+// measure firefox TTFI
+// note: this browser pref must be enabled:
+// dom.performance.time_to_first_interactive.enabled = True
+// default only; this is set via control server settings json
+var getTTFI = false;
+
 // measure google's first-contentful-paint
 // default only; this is set via control server settings json
 var getFCP = false;
 
 // performance.timing measurement used as 'starttime'
 var startMeasure = "fetchStart";
 
 function contentHandler() {
@@ -82,16 +88,24 @@ function setup(settings) {
   if (settings.measure.hero !== undefined) {
     if (settings.measure.hero.length !== 0) {
       getHero = true;
       heroesToCapture = settings.measure.hero;
       console.log("hero elements to measure: " + heroesToCapture);
       measureHero();
     }
   }
+
+  if (settings.measure.ttfi !== undefined) {
+    getTTFI = settings.measure.ttfi;
+    if (getTTFI) {
+      console.log("will be measuring ttfi");
+      measureTTFI();
+    }
+  }
 }
 
 function measureHero() {
   var obs = null;
 
   var heroElementsFound = window.document.querySelectorAll("[elementtiming]");
   console.log("found " + heroElementsFound.length + " hero elements in the page");
 
@@ -174,16 +188,47 @@ function measureDCF() {
       console.log("\dcf is not yet available (0), retry number " + gRetryCounter + "...\n");
       window.setTimeout(measureDCF, 100);
     } else {
       console.log("\nunable to get a value for dcf after " + gRetryCounter + " retries\n");
     }
   }
 }
 
+function measureTTFI() {
+  var x = window.performance.timing.timeToFirstInteractive;
+
+  if (typeof(x) == "undefined") {
+    console.log("ERROR: timeToFirstInteractive is undefined; ensure the pref is enabled");
+    return;
+  }
+  if (x > 0) {
+    console.log("got timeToFirstInteractive: " + x);
+    gRetryCounter = 0;
+    var startTime = perfData.timing.fetchStart;
+    sendResult("ttfi", x - startTime);
+  } else {
+    gRetryCounter += 1;
+    // NOTE: currently the gecko implementation doesn't look at network
+    // requests, so this is closer to TimeToFirstInteractive than
+    // TimeToInteractive.  Also, we use FNBP instead of FCP as the start
+    // point.  TTFI/TTI requires running at least 5 seconds past last
+    // "busy" point, give 25 seconds here (overall the harness times out at
+    // 30 seconds).  Some pages will never get 5 seconds without a busy
+    // period!
+    if (gRetryCounter <= 25*(1000/200)) {
+      console.log("\TTFI is not yet available (0), retry number " + gRetryCounter + "...\n");
+      window.setTimeout(measureTTFI, 200);
+    } else {
+      // unable to get a value for TTFI - filter out later
+      sendResult("ttfi", 0);
+    }
+  }
+}
+
 function measureFirstContentfulPaint() {
   // see https://developer.mozilla.org/en-US/docs/Web/API/PerformancePaintTiming
   var resultType = "fcp";
   var result = 0;
 
   let performanceEntries = perfData.getEntriesByType("paint");
 
   if (performanceEntries.length >= 2) {
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -32,22 +32,24 @@ var testType;
 var pageCycles = 0;
 var pageCycle = 0;
 var testURL;
 var testTabID = 0;
 var getHero = false;
 var getFNBPaint = false;
 var getFCP = false;
 var getDCF = false;
+var getTTFI = false;
 var isHeroPending = false;
 var pendingHeroes = [];
 var settings = {};
 var isFNBPaintPending = false;
 var isFCPPending = false;
 var isDCFPending = false;
+var isTTFIPending = false;
 var isBenchmarkPending = false;
 var pageTimeout = 10000; // default pageload timeout
 
 var results = {"name": "",
                "page": "",
                "type": "",
                "lower_is_better": true,
                "alert_threshold": 2.0,
@@ -101,16 +103,19 @@ function getTestSettings() {
             if (settings.measure.fcp !== undefined) {
               getFCP = settings.measure.fcp;
             }
             if (settings.measure.hero !== undefined) {
               if (settings.measure.hero.length !== 0) {
                 getHero = true;
               }
             }
+            if (settings.measure.ttfi !== undefined) {
+              getTTFI = settings.measure.ttfi;
+            }
           } else {
             console.log("abort: 'measure' key not found in test settings");
             cleanUp();
           }
         }
 
         // write options to storage that our content script needs to know
         if (["firefox", "geckoview"].includes(browserName)) {
@@ -172,17 +177,17 @@ async function testTabUpdated(tab) {
   nextCycle();
 }
 
 function waitForResult() {
   console.log("awaiting results...");
   return new Promise(resolve => {
     function checkForResult() {
       if (testType == "pageload") {
-        if (!isHeroPending && !isFNBPaintPending && !isFCPPending && !isDCFPending) {
+        if (!isHeroPending && !isFNBPaintPending && !isFCPPending && !isDCFPending && !isTTFIPending) {
           cancelTimeoutAlarm("raptor-page-timeout");
           resolve();
         } else {
           setTimeout(checkForResult, 5);
         }
       } else if (testType == "benchmark") {
         if (!isBenchmarkPending) {
           cancelTimeoutAlarm("raptor-page-timeout");
@@ -217,16 +222,18 @@ function nextCycle() {
           pendingHeroes = Array.from(settings.measure.hero);
         }
         if (getFNBPaint)
           isFNBPaintPending = true;
         if (getFCP)
           isFCPPending = true;
         if (getDCF)
           isDCFPending = true;
+        if (getTTFI)
+          isTTFIPending = true;
       } else if (testType == "benchmark") {
         isBenchmarkPending = true;
       }
       // update the test page - browse to our test URL
       ext.tabs.update(testTabID, {url: testURL}, testTabUpdated);
     }, pageCycleDelay);
   } else {
     verifyResults();
@@ -293,16 +300,19 @@ function resultListener(request, sender,
           }
         }
       } else if (request.type == "fnbpaint") {
         results.measurements.fnbpaint.push(request.value);
         isFNBPaintPending = false;
       } else if (request.type == "dcf") {
         results.measurements.dcf.push(request.value);
         isDCFPending = false;
+      } else if (request.type == "ttfi") {
+        results.measurements.ttfi.push(request.value);
+        isTTFIPending = false;
       } else if (request.type == "fcp") {
         results.measurements.fcp.push(request.value);
         isFCPPending = false;
       }
     } else if (testType == "benchmark") {
       // benchmark results received (all results for that complete benchmark run)
       console.log("received results from benchmark");
       results.measurements[request.type].push(request.value);