Bug 1614565 - [Mozproxy] Add mitmproxy 5.0.1 capability to mozproxy r=tarek,perftest-reviewers,AlexandruIonescu
authorFlorin Strugariu <fstrugariu@mozilla.com>
Mon, 09 Mar 2020 10:37:31 +0000
changeset 517510 74bb87071991d2398596cb04b4f9f4d523d4c33f
parent 517509 7e94dc16ae33f13c8c850e07699565af8bb57fd5
child 517511 7d8b346540db407b23ec4c538af659de47f8bcd8
push id37198
push useropoprus@mozilla.com
push dateMon, 09 Mar 2020 21:52:54 +0000
treeherdermozilla-central@268543e53e1b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstarek, perftest-reviewers, AlexandruIonescu
bugs1614565
milestone75.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1614565 - [Mozproxy] Add mitmproxy 5.0.1 capability to mozproxy r=tarek,perftest-reviewers,AlexandruIonescu Differential Revision: https://phabricator.services.mozilla.com/D62420
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-linux64.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-osx.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-win.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py
--- a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
@@ -79,35 +79,44 @@ POLICIES_CONTENT_OFF = """{
     }
   }
 }"""
 
 
 class Mitmproxy(Playback):
     def __init__(self, config):
         self.config = config
-        self.host = "127.0.0.1" if 'localhost' in self.config["host"] else self.config["host"]
+        self.host = (
+            "127.0.0.1" if "localhost" in self.config["host"] else self.config["host"]
+        )
         self.port = None
         self.mitmproxy_proc = None
         self.mitmdump_path = None
         self.browser_path = os.path.normpath(config.get("binary"))
         self.policies_dir = None
         self.ignore_mitmdump_exit_failure = config.get(
             "ignore_mitmdump_exit_failure", False
         )
 
         if self.config.get("playback_version") is None:
-            LOG.info("mitmproxy was not provided with a 'playback_version' "
-                     "Using default playback version: 4.0.4")
+            LOG.info(
+                "mitmproxy was not provided with a 'playback_version' "
+                "Using default playback version: 4.0.4"
+            )
             self.config["playback_version"] = "4.0.4"
 
         if self.config.get("playback_binary_manifest") is None:
-            LOG.info("mitmproxy was not provided with a 'playback_binary_manifest' "
-                     "Using default playback_binary_manifest")
-            self.config["playback_binary_manifest"] = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+            LOG.info(
+                "mitmproxy was not provided with a 'playback_binary_manifest' "
+                "Using default playback_binary_manifest"
+            )
+            self.config["playback_binary_manifest"] = (
+                "mitmproxy-rel-bin-%s-{platform}.manifest"
+                % self.config["playback_version"]
+            )
 
         # mozproxy_dir is where we will download all mitmproxy required files
         # when running locally it comes from obj_path via mozharness/mach
         if self.config.get("obj_path") is not None:
             self.mozproxy_dir = self.config.get("obj_path")
         else:
             # in production it is ../tasks/task_N/build/, in production that dir
             # is not available as an envvar, however MOZ_UPLOAD_DIR is set as
@@ -122,16 +131,19 @@ class Mitmproxy(Playback):
         LOG.info(
             "mozproxy_dir used for mitmproxy downloads and exe files: %s"
             % self.mozproxy_dir
         )
         # setting up the MOZPROXY_DIR env variable so custom scripts know
         # where to get the data
         os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
 
+        LOG.info("Playback tool: %s" % self.config["playback_tool"])
+        LOG.info("Playback tool version: %s" % self.config["playback_version"])
+
     def start(self):
         # go ahead and download and setup mitmproxy
         self.download()
 
         # mitmproxy must be started before setup, so that the CA cert is available
         self.start_mitmproxy_playback(self.mitmdump_path, self.browser_path)
 
         # In case the setup fails, we want to stop the process before raising.
@@ -146,34 +158,37 @@ class Mitmproxy(Playback):
         if not os.path.exists(self.mozproxy_dir):
             os.makedirs(self.mozproxy_dir)
 
         _manifest = os.path.join(here, self.config["playback_binary_manifest"])
         transformed_manifest = transform_platform(_manifest, self.config["platform"])
 
         # generate the mitmdump_path
         self.mitmdump_path = os.path.normpath(
-                os.path.join(self.mozproxy_dir, "mitmdump-%s" %
-                             self.config["playback_version"],
-                             "mitmdump"))
+            os.path.join(
+                self.mozproxy_dir,
+                "mitmdump-%s" % self.config["playback_version"],
+                "mitmdump",
+            )
+        )
 
         # Check if mitmproxy bin exists
         if os.path.exists(self.mitmdump_path):
             LOG.info("mitmproxy binary already exists. Skipping download")
         else:
             # Download and unpack mitmproxy binary
             download_path = os.path.dirname(self.mitmdump_path)
             LOG.info("create mitmproxy %s dir" % self.config["playback_version"])
             if not os.path.exists(download_path):
                 os.makedirs(download_path)
 
             LOG.info("downloading mitmproxy binary")
             tooltool_download(
-                transformed_manifest, self.config["run_local"],
-                download_path)
+                transformed_manifest, self.config["run_local"], download_path
+            )
 
         if "playback_pageset_manifest" in self.config:
             # we use one pageset for all platforms
             LOG.info("downloading mitmproxy pageset")
             _manifest = self.config["playback_pageset_manifest"]
             transformed_manifest = transform_platform(
                 _manifest, self.config["platform"]
             )
@@ -208,54 +223,65 @@ class Mitmproxy(Playback):
         LOG.info("browser path: %s" % browser_path)
 
         # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
         env = os.environ.copy()
         env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
         command = [mitmdump_path]
 
         # add proxy host and port options
-        command.extend(["--listen-host", self.host,
-                        "--listen-port", str(self.port)])
+        command.extend(["--listen-host", self.host, "--listen-port", str(self.port)])
 
         if "playback_tool_args" in self.config:
             LOG.info("Staring Proxy using provided command line!")
             command.extend(self.config["playback_tool_args"])
         elif "playback_files" in self.config:
             script = os.path.join(
-                os.path.dirname(os.path.realpath(__file__)), "scripts",
-                "alternate-server-replay-{}.py".format(
-                    self.config["playback_version"]))
+                os.path.dirname(os.path.realpath(__file__)),
+                "scripts",
+                "alternate-server-replay.py",
+            )
+            recording_paths = [
+                normalize_path(recording_path)
+                for recording_path in self.config["playback_files"]
+            ]
 
-            recording_paths = [normalize_path(recording_path)
-                               for recording_path in self.config["playback_files"]]
-
-            if self.config["playback_version"] == "4.0.4":
+            if self.config["playback_version"] in ["4.0.4", "5.0.1"]:
                 args = [
                     "-v",
-                    "--set", "upstream_cert=false",
-                    "--set", "upload_dir=" + normalize_path(self.upload_dir),
-                    "--set", "websocket=false",
-                    "--set", "server_replay_files={}".format(",".join(recording_paths)),
-                    "--scripts", normalize_path(script),
+                    "--set",
+                    "upstream_cert=false",
+                    "--set",
+                    "upload_dir=" + normalize_path(self.upload_dir),
+                    "--set",
+                    "websocket=false",
+                    "--set",
+                    "server_replay_files={}".format(",".join(recording_paths)),
+                    "--scripts",
+                    normalize_path(script),
                 ]
                 command.extend(args)
             else:
                 raise Exception("Mitmproxy version is unknown!")
 
         else:
-            raise Exception("Mitmproxy can't start playback! Playback settings missing.")
+            raise Exception(
+                "Mitmproxy can't start playback! Playback settings missing."
+            )
 
         LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
         LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command))
         # to turn off mitmproxy log output, use these params for Popen:
         # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
         self.mitmproxy_proc = ProcessHandler(
-            command, logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
-            env=env, processStderrLine=LOG.error, storeOutput=False
+            command,
+            logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
+            env=env,
+            processStderrLine=LOG.error,
+            storeOutput=False,
         )
         self.mitmproxy_proc.run()
         end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
         ready = False
         while time.time() < end_time:
             ready = self.check_proxy(host=self.host, port=self.port)
             if ready:
                 LOG.info(
@@ -290,19 +316,22 @@ class Mitmproxy(Playback):
 
         if exit_code != 0:
             if exit_code is None:
                 LOG.error("Failed to kill the mitmproxy playback process")
                 return
 
             if mozinfo.os == "win":
                 from mozprocess.winprocess import ERROR_CONTROL_C_EXIT  # noqa
+
                 if exit_code == ERROR_CONTROL_C_EXIT:
-                    LOG.info("Successfully killed the mitmproxy playback process"
-                             " with exit code %d" % exit_code)
+                    LOG.info(
+                        "Successfully killed the mitmproxy playback process"
+                        " with exit code %d" % exit_code
+                    )
                     return
             log_func = LOG.error
             if self.ignore_mitmdump_exit_failure:
                 log_func = LOG.info
             log_func("Mitmproxy exited with error code %d" % exit_code)
         else:
             LOG.info("Successfully killed the mitmproxy playback process")
 
@@ -356,19 +385,18 @@ class MitmproxyDesktop(Mitmproxy):
         if not os.path.exists(self.policies_dir):
             LOG.info("creating folder: %s" % self.policies_dir)
             os.makedirs(self.policies_dir)
         else:
             LOG.info("folder already exists: %s" % self.policies_dir)
 
         self.write_policies_json(
             self.policies_dir,
-            policies_content=POLICIES_CONTENT_ON % {"cert": self.cert_path,
-                                                    "host": self.host,
-                                                    "port": self.port},
+            policies_content=POLICIES_CONTENT_ON
+            % {"cert": self.cert_path, "host": self.host, "port": self.port},
         )
 
         # cannot continue if failed to add CA cert to Firefox, need to check
         if not self.is_mitmproxy_cert_installed():
             LOG.error(
                 "Aborting: failed to install mitmproxy CA cert into Firefox desktop"
             )
             self.stop_mitmproxy_playback()
@@ -391,18 +419,18 @@ class MitmproxyDesktop(Mitmproxy):
     def is_mitmproxy_cert_installed(self):
         """Verify mitmxproy CA cert was added to Firefox"""
         try:
             # read autoconfig file, confirm mitmproxy cert is in there
             contents = self.read_policies_json(self.policies_dir)
             LOG.info("Firefox policies file contents:")
             LOG.info(contents)
             if (
-                    POLICIES_CONTENT_ON
-                    % {"cert": self.cert_path, "host": self.host, "port": self.port}
+                POLICIES_CONTENT_ON
+                % {"cert": self.cert_path, "host": self.host, "port": self.port}
             ) in contents:
                 LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
             else:
 
                 return False
         except Exception as e:
             LOG.info("failed to read Firefox policies file, exeption: %s" % e)
             return False
@@ -437,19 +465,22 @@ class MitmproxyAndroid(Mitmproxy):
         If running in production:
         1. Get the tooltools manifest file for downloading hostutils (contains certutil)
         2. Get the `certutil` tool by downloading hostutils using the tooltool manifest
         """
         if self.config["run_local"]:
             # when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
             self.certutil_path = os.path.join(os.environ["MOZ_HOST_BIN"], "certutil")
             if not (
-                    os.path.isfile(self.certutil_path) and os.access(self.certutil_path, os.X_OK)
+                os.path.isfile(self.certutil_path)
+                and os.access(self.certutil_path, os.X_OK)
             ):
-                raise Exception("Abort: unable to execute certutil: {}".format(self.certutil_path))
+                raise Exception(
+                    "Abort: unable to execute certutil: {}".format(self.certutil_path)
+                )
             self.certutil_path = os.environ["MOZ_HOST_BIN"]
             os.environ["LD_LIBRARY_PATH"] = self.certutil_path
         else:
             # must download certutil inside hostutils via tooltool; use this manifest:
             # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
             # after it will be found here inside the worker/bitbar container:
             # /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64
             LOG.info("downloading certutil binary (hostutils)")
@@ -532,19 +563,17 @@ class MitmproxyAndroid(Mitmproxy):
             "-n",
             "mitmproxy-cert",
             "-t",
             "TC,,",
             "-a",
             "-i",
             local_cert_path,
         ]
-        LOG.info(
-            "importing mitmproxy cert into db using command"
-        )
+        LOG.info("importing mitmproxy cert into db using command")
         self.certutil(args)
 
     def create_cert_db(self, cert_db_location):
         # create cert db if it doesn't already exist; it may exist already
         # if a previous pageload test ran in the same test suite
         args = ["-d", cert_db_location, "-N", "--empty-password"]
 
         LOG.info("creating nss cert database")
@@ -588,17 +617,19 @@ class MitmproxyAndroid(Mitmproxy):
             )
             return True
         return False
 
     def certutil(self, args, raise_exception=True):
         cmd = [self.certutil_path] + list(args)
         LOG.info("Certutil: Running command: %s" % " ".join(cmd))
         try:
-            cmd_proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy())
+            cmd_proc = subprocess.Popen(
+                cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy()
+            )
 
             cmd_output, errs = cmd_proc.communicate()
         except subprocess.SubprocessError:
             LOG.critical("could not run the certutil command")
             raise
 
         if cmd_proc.returncode == 0:
             # Debug purpose only remove if stable
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-linux64.manifest
@@ -0,0 +1,10 @@
+[
+    {
+    "size": 76345093,
+    "visibility": "public",
+    "digest": "4d653c0c74a8677e8e78cd72d109b1b54c75ef57b2e2ce980c5cfd602966c310065cf0e95c35f4fbfb1fe817f062ac6cf9cc129d72d42184b0237fb9b0bde081",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.0.1-linux.tar.gz",
+    "unpack": true
+    }
+]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-osx.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 41341221,
+    "visibility": "public",
+    "digest": "4624bc26638cd7f3ab8c8d2ee8ab44ab81018114714a2b82614883adbfad540de73a97455fb228a3003006b4d03a8a9a597d70f8b27ccf1718f78380f911bdd8",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.0.1-osx.tar.gz",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-win.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 37875625,
+    "visibility": "public",
+    "digest": "d66234c9ca692d03412dd194b3f47c098e8ee1b17b178034fc86e0d8ada4d4f6cd1fbcfb62b7e7016539a878b1274ef83451a0acaca7011efaacb291fa52918d",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.0.1-windows.zip",
+    "unpack": true
+  }
+]
rename from testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py
--- a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py
@@ -80,24 +80,28 @@ def _remote_settings_changed(self, event
 
 
 Http2Layer._handle_remote_settings_changed = _remote_settings_changed
 # END OF PATCHING
 
 
 class AlternateServerPlayback:
     def __init__(self):
+
         ctx.master.addons.remove(ctx.master.addons.get("serverplayback"))
         self.flowmap = {}
         self.configured = False
         self.netlocs = defaultdict(lambda: defaultdict(int))
         self.calls = []
         self._done = False
         self._replayed = 0
         self._not_replayed = 0
+        self.mitm_version = ctx.mitmproxy.version.MITMPROXY
+
+        ctx.log.info("MitmProxy version: %s" % self.mitm_version)
 
     def load(self, loader):
         loader.add_option(
             "server_replay_files",
             typing.Sequence[str],
             [],
             "Replay server responses from a saved file.",
         )
@@ -107,26 +111,40 @@ class AlternateServerPlayback:
         )
 
     def load_flows(self, flows):
         """
             Replay server responses from flows.
         """
         for i in flows:
             if i.type == 'websocket':
+                # Mitmproxy can't replay WebSocket packages.
                 ctx.log.info(
-                    "Request is a WebSocketFlow. Removing from request list as WebSockets"
-                    " are dissabled "
+                    "Recorded response is a WebSocketFlow. Removing from recording list as"
+                    "  WebSockets are disabled"
                 )
             elif i.response:
-                l = self.flowmap.setdefault(self._hash(i), [])
-                l.append(i)
+                # check if recorded request has a response associated
+                if self.mitm_version == "5.0.1":
+                    if i.response.content:
+                        # Mitmproxy 5.0.1 Cannot assemble flow with missing content
+
+                        l = self.flowmap.setdefault(self._hash(i), [])
+                        l.append(i)
+                    else:
+                        ctx.log.info(
+                             "Recorded response %s has no content. Removing from recording list"
+                             % i.request.url
+                        )
+                if self.mitm_version == "4.0.4":
+                    l = self.flowmap.setdefault(self._hash(i), [])
+                    l.append(i)
             else:
                 ctx.log.info(
-                    "Request %s has no response. Removing from request list"
+                    "Recorded request %s has no response. Removing from recording list"
                     % i.request.url
                 )
         ctx.master.addons.trigger("update", [])
 
     def load_files(self, paths):
         if "," in paths[0]:
             paths = paths[0].split(",")
         for path in paths: