Bug 1617551 - Add page load replay confidence as a subtest with alerting enabled r=tarek,perftest-reviewers,sparky
authorFlorin Strugariu <fstrugariu@mozilla.com>
Wed, 25 Mar 2020 14:16:12 +0000
changeset 520363 b172a03bfbb67ff42e3c01ee6238d41cc414aa64
parent 520362 236b90d0f6637e35539ab941c9946ecd3675f293
child 520364 7e1333796925e0abd003b6b76ecbc0d714c219df
push id37249
push userdvarga@mozilla.com
push dateWed, 25 Mar 2020 21:39:06 +0000
treeherdermozilla-central@b3c3f7d0f044 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstarek, perftest-reviewers, sparky
bugs1617551
milestone76.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1617551 - Add page load replay confidence as a subtest with alerting enabled r=tarek,perftest-reviewers,sparky Differential Revision: https://phabricator.services.mozilla.com/D64112
testing/mozbase/mozproxy/mozproxy/backends/base.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py
testing/raptor/raptor/output.py
testing/raptor/raptor/results.py
testing/raptor/raptor/webextension/base.py
--- a/testing/mozbase/mozproxy/mozproxy/backends/base.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/base.py
@@ -25,8 +25,12 @@ class Playback(object):
 
     @abstractmethod
     def start(self):
         pass
 
     @abstractmethod
     def stop(self):
         pass
+
+    @abstractmethod
+    def confidence(self):
+        pass
--- a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
@@ -1,26 +1,26 @@
 """Functions to download, install, setup, and use the mitmproxy playback tool"""
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import absolute_import
 
 import glob
+import json
 import os
+import signal
+import socket
 import subprocess
 import sys
 import time
-import socket
 from subprocess import PIPE
-import signal
 
 import mozinfo
 from mozprocess import ProcessHandler
-
 from mozproxy.backends.base import Playback
 from mozproxy.utils import (
     transform_platform,
     tooltool_download,
     download_file_from_url,
     get_available_port,
     LOG,
 )
@@ -90,16 +90,17 @@ class Mitmproxy(Playback):
         self.port = None
         self.mitmproxy_proc = None
         self.mitmdump_path = None
         self.browser_path = os.path.normpath(config.get("binary"))
         self.policies_dir = None
         self.ignore_mitmdump_exit_failure = config.get(
             "ignore_mitmdump_exit_failure", False
         )
+        self.recording_paths = None
 
         if self.config.get("playback_version") is None:
             LOG.info(
                 "mitmproxy was not provided with a 'playback_version' "
                 "Using default playback version: 4.0.4"
             )
             self.config["playback_version"] = "4.0.4"
 
@@ -234,32 +235,30 @@ class Mitmproxy(Playback):
             LOG.info("Staring Proxy using provided command line!")
             command.extend(self.config["playback_tool_args"])
         elif "playback_files" in self.config:
             script = os.path.join(
                 os.path.dirname(os.path.realpath(__file__)),
                 "scripts",
                 "alternate-server-replay.py",
             )
-            recording_paths = [
-                normalize_path(recording_path)
-                for recording_path in self.config["playback_files"]
-            ]
+            self.recording_paths = [normalize_path(recording_path)
+                                    for recording_path in self.config["playback_files"]]
 
             if self.config["playback_version"] in ["4.0.4", "5.0.1"]:
                 args = [
                     "-v",
                     "--set",
                     "upstream_cert=false",
                     "--set",
                     "upload_dir=" + normalize_path(self.upload_dir),
                     "--set",
                     "websocket=false",
                     "--set",
-                    "server_replay_files={}".format(",".join(recording_paths)),
+                    "server_replay_files={}".format(",".join(self.recording_paths)),
                     "--scripts",
                     normalize_path(script),
                 ]
                 command.extend(args)
             else:
                 raise Exception("Mitmproxy version is unknown!")
 
         else:
@@ -340,16 +339,40 @@ class Mitmproxy(Playback):
         try:
             s.connect((host, port))
             s.shutdown(socket.SHUT_RDWR)
             s.close()
             return True
         except socket.error:
             return False
 
+    def confidence(self):
+        file_name = "mitm_netlocs_%s.json" % os.path.splitext(
+            os.path.basename(
+                self.recording_paths[0]
+            )
+        )[0]
+        path = os.path.normpath(os.path.join(self.upload_dir,
+                                             file_name))
+        if os.path.exists(path):
+            try:
+                LOG.info("Reading confidence values from: %s" % path)
+                with open(path, "r") as f:
+                    data = json.load(f)
+                    return {"confidence": data["confidence"],
+                            "not-replayed": data["not-replayed"],
+                            "replayed": data["replayed"]
+                            }
+            except Exception:
+                LOG.info("Can't read netlocs file!", exc_info=True)
+                return None
+        else:
+            LOG.info("Netlocs file is not available! Cant find %s" % path)
+            return None
+
 
 class MitmproxyDesktop(Mitmproxy):
     def setup(self):
         """
         Installs certificates.
 
         For Firefox we need to install the generated mitmproxy CA cert. For
         Chromium this is not necessary as it will be started with the
--- a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py
@@ -204,21 +204,26 @@ class AlternateServerPlayback:
     def done(self):
         if self._done or not ctx.options.upload_dir:
             return
 
         confidence = float(self._replayed) / float(self._replayed + self._not_replayed)
         stats = {"totals": dict(self.netlocs),
                  "calls": self.calls,
                  "replayed": self._replayed,
-                 "not_replayed": self._not_replayed,
+                 "not-replayed": self._not_replayed,
                  "confidence": int(confidence * 100)}
-
+        file_name = "mitm_netlocs_%s.json" % \
+                    os.path.splitext(
+                        os.path.basename(
+                            ctx.options.server_replay_files[0]
+                        )
+                    )[0]
         path = os.path.normpath(os.path.join(ctx.options.upload_dir,
-                                             "mitm_netlocs.json"))
+                                             file_name))
         try:
             with open(path, "w") as f:
                 f.write(json.dumps(stats, indent=2, sort_keys=True))
         finally:
             self._done = True
 
     def request(self, f):
         if self.flowmap:
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -28,16 +28,17 @@ class PerftestOutput(object):
         - results : list of RaptorTestResult instances
         """
         self.results = results
         self.summarized_results = {}
         self.supporting_data = supporting_data
         self.summarized_supporting_data = []
         self.summarized_screenshots = []
         self.subtest_alert_on = subtest_alert_on
+        self.mozproxy_data = False
         self.browser_name = None
         self.browser_version = None
 
     @abstractmethod
     def summarize(self, test_names):
         raise NotImplementedError()
 
     def set_browser_meta(self, browser_name, browser_version):
@@ -77,16 +78,19 @@ class PerftestOutput(object):
         self.summarized_supporting_data = []
         support_data_by_type = {}
 
         for data_set in self.supporting_data:
 
             data_type = data_set["type"]
             LOG.info("summarizing %s data" % data_type)
 
+            if "mozproxy" in data_type:
+                self.mozproxy_data = True
+
             if data_type not in support_data_by_type:
                 support_data_by_type[data_type] = {
                     "framework": {"name": "raptor"},
                     "suites": [],
                 }
 
             # suite name will be name of the actual raptor test that ran, plus the type of
             # supporting data i.e. 'raptor-speedometer-geckoview-power'
@@ -96,30 +100,46 @@ class PerftestOutput(object):
                 "name": data_set["test"] + "-" + data_set["type"],
                 "type": data_set["type"],
                 "subtests": subtests,
                 "lowerIsBetter": True,
                 "unit": data_set["unit"],
                 "alertThreshold": 2.0,
             }
 
+            # custom setting for mozproxy-replay
+            if data_type == 'mozproxy-replay':
+                suite["lowerIsBetter"] = False
+                suite["unit"] = "%"
+
+            for result in self.results:
+                if result["name"] == data_set["test"]:
+                    suite["extraOptions"] = result["extra_options"]
+
             support_data_by_type[data_type]["suites"].append(suite)
 
             # each supporting data measurement becomes a subtest, with the measurement type
             # used for the subtest name. i.e. 'power-cpu'
             # the overall 'suite' value for supporting data is dependent on
             # the unit of the values, by default the sum of all measurements
             # is taken.
             for measurement_name, value in data_set["values"].iteritems():
                 new_subtest = {}
                 new_subtest["name"] = data_type + "-" + measurement_name
                 new_subtest["value"] = value
                 new_subtest["lowerIsBetter"] = True
                 new_subtest["alertThreshold"] = 2.0
                 new_subtest["unit"] = data_set["unit"]
+
+                if measurement_name == "confidence":
+                    new_subtest["unit"] = "%"
+
+                if measurement_name == "replayed":
+                    new_subtest["lowerIsBetter"] = False
+
                 subtests.append(new_subtest)
                 vals.append([new_subtest["value"], new_subtest["name"]])
 
             if len(subtests) >= 1:
                 suite["value"] = self.construct_summary(
                     vals, testname="supporting_data", unit=data_set["unit"]
                 )
 
@@ -207,17 +227,17 @@ class PerftestOutput(object):
                 self.summarized_results["application"]["version"] = self.browser_version
 
         total_perfdata = 0
         if output_perf_data:
             # if we have supporting data i.e. power, we ONLY want those measurements
             # dumped out. TODO: Bug 1515406 - Add option to output both supplementary
             # data (i.e. power) and the regular Raptor test result
             # Both are already available as separate PERFHERDER_DATA json blobs
-            if len(self.summarized_supporting_data) == 0:
+            if len(self.summarized_supporting_data) == 0 or self.mozproxy_data:
                 LOG.info("PERFHERDER_DATA: %s" % json.dumps(self.summarized_results))
                 total_perfdata = 1
             else:
                 LOG.info(
                     "supporting data measurements exist - only posting those to perfherder"
                 )
 
         json.dump(
@@ -356,29 +376,46 @@ class PerftestOutput(object):
         if testname.startswith("raptor-wasm-godot"):
             # wasm_godot_score: first-interactive mean
             return filters.mean(_filter(vals, "first-interactive"))
 
         if testname.startswith("raptor-youtube-playback"):
             return round(filters.mean(_filter(vals)), 2)
 
         if testname.startswith("supporting_data"):
-            if unit:
-                if unit in ("%",):
-                    return filters.mean(_filter(vals))
-                elif unit in ("W", "MHz"):
-                    # For power in Watts and clock frequencies,
-                    # summarize with the sum of the averages
-                    allavgs = []
-                    for (val, subtest) in vals:
-                        if "avg" in subtest:
-                            allavgs.append(val)
-                    if allavgs:
-                        return sum(allavgs)
-            return sum(_filter(vals))
+            if not unit:
+                return sum(_filter(vals))
+
+            if unit == "%":
+                return filters.mean(_filter(vals))
+
+            if unit == "a.u.":
+                if any("mozproxy-replay" in val[1] for val in vals):
+                    for val in vals:
+                        if val[1].endswith("confidence"):
+                            return val[0]
+                    raise Exception("No confidence measurement found in mozproxy supporting_data")
+
+            if unit in ("W", "MHz"):
+                # For power in Watts and clock frequencies,
+                # summarize with the sum of the averages
+                allavgs = []
+                for val, subtest in vals:
+                    if "avg" in subtest:
+                        allavgs.append(val)
+                if allavgs:
+                    return sum(allavgs)
+
+                raise Exception(
+                    "No average measurements found for supporting data with W, or MHz unit .")
+
+            if unit in ["KB", "mAh"]:
+                return sum(_filter(vals))
+
+            raise NotImplementedError("Unit %s not suported" % unit)
 
         if len(vals) > 1:
             return round(filters.geometric_mean(_filter(vals)), 2)
 
         return round(filters.mean(_filter(vals)), 2)
 
     def parseSpeedometerOutput(self, test):
         # each benchmark 'index' becomes a subtest; each pagecycle / iteration
--- a/testing/raptor/raptor/results.py
+++ b/testing/raptor/raptor/results.py
@@ -120,16 +120,21 @@ class PerftestResultsHandler(object):
                     break
 
         if is_scenario and not is_resource_test():
             # skip perfherder check when a scenario test-type is run without
             # a resource flag
             return None
 
         expected_perfherder = 1
+
+        if output.mozproxy_data:
+            # Check if we have mozproxy data available.
+            expected_perfherder += 1
+
         if is_resource_test():
             # when resource tests are run, no perfherder data is output
             # for the regular raptor tests (i.e. speedometer) so we
             # expect one per resource-type, starting with 0
             expected_perfherder = 0
             if self.power_test:
                 expected_perfherder += 1
             if self.memory_test:
--- a/testing/raptor/raptor/webextension/base.py
+++ b/testing/raptor/raptor/webextension/base.py
@@ -128,16 +128,29 @@ class WebExtension(Perftest):
                 self.control_server._runtime_error["stack"],
             ))
 
     def run_test_teardown(self, test):
         super(WebExtension, self).run_test_teardown(test)
 
         if self.playback is not None:
             self.playback.stop()
+
+            confidence_values = self.playback.confidence()
+            if confidence_values:
+                mozproxy_replay = {
+                    u'type': u'mozproxy-replay',
+                    u'test': test["name"],
+                    u'unit': u'a.u.',
+                    u'values': confidence_values
+                }
+                self.control_server.submit_supporting_data(mozproxy_replay)
+            else:
+                LOG.info("Mozproxy replay confidence data not available!")
+
             self.playback = None
 
         self.remove_raptor_webext()
 
     def set_browser_test_prefs(self, raw_prefs):
         # add test specific preferences
         LOG.info("setting test-specific Firefox preferences")
         self.profile.set_preferences(json.loads(raw_prefs))