Bug 1532671 output what measurements are still pending though when there is a page timeout in raptor r=davehunt
authorFlorin Strugariu <fstrugariu@mozilla.com>
Tue, 12 Mar 2019 12:16:09 +0000
changeset 521516 ee300588de96
parent 521515 a28e95a5730f
child 521517 52fd3c1c9615
push id10867
push userdvarga@mozilla.com
push dateThu, 14 Mar 2019 15:20:45 +0000
treeherdermozilla-beta@abad13547875 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1532671
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1532671 output what measurements are still pending though when there is a page timeout in raptor r=davehunt Differential Revision: https://phabricator.services.mozilla.com/D22107
testing/raptor/raptor/control_server.py
testing/raptor/raptor/raptor.py
testing/raptor/raptor/results.py
testing/raptor/webext/raptor/runner.js
--- a/testing/raptor/raptor/control_server.py
+++ b/testing/raptor/raptor/control_server.py
@@ -70,17 +70,18 @@ def MakeCustomHandlerClass(results_handl
                 self.write_raw_gecko_profile(_test, _pagecycle, _raw_profile)
             elif data['type'] == 'webext_results':
                 LOG.info("received " + data['type'] + ": " + str(data['data']))
                 self.results_handler.add(data['data'])
             elif data['type'] == "webext_raptor-page-timeout":
                 LOG.info("received " + data['type'] + ": " + str(data['data']))
                 # pageload test has timed out; record it as a failure
                 self.results_handler.add_page_timeout(str(data['data'][0]),
-                                                      str(data['data'][1]))
+                                                      str(data['data'][1]),
+                                                      dict(data['data'][2]))
             elif data['data'] == "__raptor_shutdownBrowser":
                 LOG.info("received " + data['type'] + ": " + str(data['data']))
                 # webext is telling us it's done, and time to shutdown the browser
                 self.shutdown_browser()
             elif data['type'] == 'webext_screenshot':
                 LOG.info("received " + data['type'])
                 self.results_handler.add_image(str(data['data'][0]),
                                                str(data['data'][1]),
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -10,17 +10,16 @@ import os
 import posixpath
 import shutil
 import sys
 import tempfile
 import time
 
 import mozcrash
 import mozinfo
-
 from mozdevice import ADBDevice
 from mozlog import commandline, get_default_logger
 from mozprofile import create_profile
 from mozrunner import runners
 
 # need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 paths = [here]
@@ -36,16 +35,17 @@ paths.append(webext_dir)
 
 for path in paths:
     if not os.path.exists(path):
         raise IOError("%s does not exist. " % path)
     sys.path.insert(0, path)
 
 try:
     from mozbuild.base import MozbuildObject
+
     build = MozbuildObject.from_environment(cwd=here)
 except ImportError:
     build = None
 
 from benchmark import Benchmark
 from cmdline import parse_args
 from control_server import RaptorControlServer
 from gen_test_config import gen_test_config
@@ -710,18 +710,21 @@ def main(args=sys.argv[1:]):
         os.sys.exit(1)
 
     # if we have results but one test page timed out (i.e. one tp6 test page didn't load
     # but others did) we still dumped PERFHERDER_DATA for the successfull pages but we
     # want the overall test job to marked as a failure
     pages_that_timed_out = raptor.get_page_timeout_list()
     if len(pages_that_timed_out) > 0:
         for _page in pages_that_timed_out:
-            LOG.critical("TEST-UNEXPECTED-FAIL: test '%s' timed out loading test page: %s"
-                         % (_page['test_name'], _page['url']))
+            LOG.critical("TEST-UNEXPECTED-FAIL: test '%s' timed out loading test page: %s "
+                         "pending metrics: %s"
+                         % (_page['test_name'],
+                            _page['url'],
+                            _page['pending_metrics']))
         os.sys.exit(1)
 
     # when running raptor locally with gecko profiling on, use the view-gecko-profile
     # tool to automatically load the latest gecko profile in profiler.firefox.com
     if args.gecko_profile and args.run_local:
         if os.environ.get('DISABLE_PROFILE_LAUNCH', '0') == '1':
             LOG.info("Not launching profiler.firefox.com because DISABLE_PROFILE_LAUNCH=1")
         else:
--- a/testing/raptor/raptor/results.py
+++ b/testing/raptor/raptor/results.py
@@ -1,20 +1,19 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # class to process, format, and report raptor test results
 # received from the raptor control server
 from __future__ import absolute_import
 
+from mozlog import get_proxy_logger
 from output import Output
 
-from mozlog import get_proxy_logger
-
 LOG = get_proxy_logger(component='results-handler')
 
 
 class RaptorResultsHandler():
     """Handle Raptor test results"""
 
     def __init__(self):
         self.results = []
@@ -30,18 +29,23 @@ class RaptorResultsHandler():
 
     def add_image(self, screenshot, test_name, page_cycle):
         # add to results
         LOG.info("received screenshot")
         self.images.append({'screenshot': screenshot,
                             'test_name': test_name,
                             'page_cycle': page_cycle})
 
-    def add_page_timeout(self, test_name, page_url):
-        self.page_timeout_list.append({'test_name': test_name, 'url': page_url})
+    def add_page_timeout(self, test_name, page_url, pending_metrics):
+
+        pending_metrics = [key for key, value in pending_metrics.items() if value]
+
+        self.page_timeout_list.append({'test_name': test_name,
+                                       'url': page_url,
+                                       'pending_metrics': ", ".join(pending_metrics)})
 
     def add_supporting_data(self, supporting_data):
         ''' Supporting data is additional data gathered outside of the regular
         Raptor test run (i.e. power data). Will arrive in a dict in the format of:
 
         supporting_data = {'type': 'data-type',
                            'test': 'raptor-test-ran-when-data-was-gathered',
                            'unit': 'unit that the values are in',
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -393,17 +393,28 @@ async function nextCycle() {
       }, pageCycleDelay);
     } else {
       verifyResults();
     }
 }
 
 async function timeoutAlarmListener() {
   console.error("raptor-page-timeout on %s" % testURL);
-  postToControlServer("raptor-page-timeout", [testName, testURL]);
+
+  var pendingMetrics = {
+    "hero": isHeroPending,
+    "fnb paint": isFNBPaintPending,
+    "fcp": isFCPPending,
+    "dcf": isDCFPending,
+    "ttfi": isTTFIPending,
+    "load time": isLoadTimePending,
+  };
+
+  postToControlServer("raptor-page-timeout", [testName, testURL, pendingMetrics]);
+
   // take a screen capture
   if (screenCapture) {
     await getScreenCapture();
   }
   // call clean-up to shutdown gracefully
   cleanUp();
 }