Bug 1496544 - Don't timeout the pageload / fail raptor test when TTFI is not available; r=jmaher
authorRob Wood <rwood@mozilla.com>
Mon, 15 Oct 2018 17:06:58 +0000
changeset 499771 e18f1acd0782f6333ee892c7ebc9711d9a5e550a
parent 499770 900eabf8fceeec876fe998b1b91d5ddd09bb20f2
child 499772 fb1c936feb43f2571bd35d3fd904ec40ef3148ac
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1496544
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1496544 - Don't timeout the pageload / fail raptor test when TTFI is not available; r=jmaher Differential Revision: https://phabricator.services.mozilla.com/D8590
testing/raptor/raptor/filter.py
testing/raptor/raptor/output.py
testing/raptor/webext/raptor/measure.js
--- a/testing/raptor/raptor/filter.py
+++ b/testing/raptor/raptor/filter.py
@@ -226,16 +226,30 @@ def ignore_min(series):
     """
     ignore minimum data point
     """
     return ignore(series, min)
 
 
 @register_filter
 @define_filter
+def ignore_negative(series):
+    """
+    ignore data points that have a negative value
+    caution: if all data values are < 0, this will return an empty list
+    """
+    if len(series) <= 1:
+        # don't modify short series
+        return series
+    series = series[:]  # do not mutate the original series
+    return list(filter(lambda x: x >= 0, series))
+
+
+@register_filter
+@define_filter
 def v8_subtest(series, name):
     """
        v8 benchmark score - modified for no sub benchmarks.
        * removed Crypto and kept Encrypt/Decrypt standalone
        * removed EarlyBoyer and kept Earley/Boyer standalone
 
        this is not 100% in parity but within .3%
     """
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -76,17 +76,32 @@ class Output(object):
                     new_subtest = {}
                     new_subtest['name'] = test.name + "-" + measurement_name
                     new_subtest['replicates'] = replicates
                     new_subtest['lowerIsBetter'] = test.subtest_lower_is_better
                     new_subtest['alertThreshold'] = float(test.alert_threshold)
                     new_subtest['value'] = 0
                     new_subtest['unit'] = test.subtest_unit
 
+                    # ignore first value due to 1st pageload noise
+                    LOG.info("ignoring the first %s value due to initial pageload noise"
+                             % measurement_name)
                     filtered_values = filter.ignore_first(new_subtest['replicates'], 1)
+
+                    # for pageload tests that measure TTFI: TTFI is not guaranteed to be available
+                    # everytime; the raptor measure.js webext will substitute a '-1' value in the
+                    # cases where TTFI is not available, which is acceptable; however we don't want
+                    # to include those '-1' TTFI values in our final results calculations
+                    if measurement_name == "ttfi":
+                        filtered_values = filter.ignore_negative(filtered_values)
+                        # we've already removed the first pageload value; if there aren't any more
+                        # valid TTFI values available for this pageload just remove it from results
+                        if len(filtered_values) < 1:
+                            continue
+
                     new_subtest['value'] = filter.median(filtered_values)
 
                     vals.append([new_subtest['value'], new_subtest['name']])
                     subtests.append(new_subtest)
 
             elif test.type == "benchmark":
                 if 'speedometer' in test.measurements:
                     subtests, vals = self.parseSpeedometerOutput(test)
--- a/testing/raptor/webext/raptor/measure.js
+++ b/testing/raptor/webext/raptor/measure.js
@@ -210,21 +210,22 @@ function measureTTFI() {
     // NOTE: currently the gecko implementation doesn't look at network
     // requests, so this is closer to TimeToFirstInteractive than
     // TimeToInteractive.  Also, we use FNBP instead of FCP as the start
     // point.  TTFI/TTI requires running at least 5 seconds past last
     // "busy" point, give 25 seconds here (overall the harness times out at
     // 30 seconds).  Some pages will never get 5 seconds without a busy
     // period!
     if (gRetryCounter <= 25 * (1000 / 200)) {
-      console.log("\TTFI is not yet available (0), retry number " + gRetryCounter + "...\n");
+      console.log("TTFI is not yet available (0), retry number " + gRetryCounter + "...\n");
       window.setTimeout(measureTTFI, 200);
     } else {
-      // unable to get a value for TTFI - filter out later
-      sendResult("ttfi", 0);
+      // unable to get a value for TTFI - negative value will be filtered out later
+      console.log("TTFI was not available for this pageload");
+      sendResult("ttfi", -1);
     }
   }
 }
 
 function measureFCP() {
   // see https://developer.mozilla.org/en-US/docs/Web/API/PerformancePaintTiming
   var resultType = "fcp";
   var result = 0;
@@ -251,13 +252,15 @@ function measureFCP() {
     }
   }
 }
 
 function sendResult(_type, _value) {
   // send result back to background runner script
   console.log("sending result back to runner: " + _type + " " + _value);
   chrome.runtime.sendMessage({"type": _type, "value": _value}, function(response) {
-    console.log(response.text);
+    if (response !== undefined) {
+      console.log(response.text);
+    }
   });
 }
 
 window.onload = contentHandler();