Bug 1539983 [mozproxy] Move alternate-server-replay script into mozproxy r=tarek
authorFlorin Strugariu <fstrugariu@mozilla.com>
Wed, 14 Aug 2019 11:49:35 +0000
changeset 487922 9e4ae2566332e5dd39b551087ae4705e7333d2ed
parent 487921 9c93a75bc692dd3efeec21f153e90237c50c4c16
child 487923 39a3421f5e048807b2d0ec91de9e9c374111ffb9
push id113895
push userbtara@mozilla.com
push dateWed, 14 Aug 2019 22:08:04 +0000
treeherdermozilla-inbound@d65454c57706 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstarek
bugs1539983
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1539983 [mozproxy] Move alternate-server-replay script into mozproxy r=tarek Differential Revision: https://phabricator.services.mozilla.com/D41540
.eslintignore
testing/mozbase/mozproxy/mozproxy/backends/mitm.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-linux64.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-osx.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-win.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-linux64.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-osx.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-win.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy_requirements.txt
testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-2.0.2.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py
testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/LICENSE
testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/deterministic.js
testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-linux64.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-osx.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-win.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-linux64.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-osx.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-win.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy_requirements.txt
testing/mozbase/mozproxy/tests/test_proxy.py
testing/raptor/raptor/playback/alternate-server-replay-2.0.2.py
testing/raptor/raptor/playback/alternate-server-replay-4.0.4.py
testing/raptor/raptor/playback/scripts/catapult/LICENSE
testing/raptor/raptor/playback/scripts/catapult/deterministic.js
testing/raptor/raptor/playback/scripts/inject-deterministic.py
testing/raptor/raptor/raptor.py
testing/raptor/test/test_playback.py
--- a/.eslintignore
+++ b/.eslintignore
@@ -290,27 +290,29 @@ testing/marionette/harness
 testing/mochitest/tests/Harness_sanity/**
 testing/mochitest/MochiKit/**
 testing/mochitest/tests/MochiKit-1.4.2/**
 testing/mochitest/tests/SimpleTest/**
 testing/modules/ajv-4.1.1.js
 testing/modules/sinon-7.2.7.js
 # octothorpe used for pref file comment causes parsing error
 testing/mozbase/mozprofile/tests/files/prefs_with_comments.js
+
+# Mozproxy third party
+testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/**
+
 testing/talos/talos/scripts/jszip.min.js
 testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js
 testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js
 testing/talos/talos/tests/devtools/addon/content/pages/**
 testing/talos/talos/tests/dromaeo/**
 testing/talos/talos/tests/v8_7/**
 testing/talos/talos/tests/kraken/**
 # Runing Talos may extract data here, see bug 1435677.
 testing/talos/talos/tests/tp5n/**
-# Raptor third party
-testing/raptor/raptor/playback/scripts/catapult/**
 
 testing/web-platform/**
 testing/xpcshell/moz-http2/**
 testing/xpcshell/node-http2/**
 testing/xpcshell/dns-packet/**
 testing/xpcshell/node-ip/**
 
 
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py
@@ -0,0 +1,8 @@
+# flake8: noqa
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from .mitm import *
rename from testing/mozbase/mozproxy/mozproxy/backends/mitm.py
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
--- a/testing/mozbase/mozproxy/mozproxy/backends/mitm.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
@@ -190,17 +190,61 @@ class Mitmproxy(Playback):
         LOG.info("browser path: %s" % browser_path)
 
         # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
         env = os.environ.copy()
         env["PATH"] = os.path.dirname(browser_path) + os.pathsep + env["PATH"]
         command = [mitmdump_path]
 
         if "playback_tool_args" in self.config:
+            LOG.info("Staring Proxy using provided command line!")
             command.extend(self.config["playback_tool_args"])
+        elif "playback_files" in self.config:
+            script = os.path.join(
+                                os.path.dirname(os.path.realpath(__file__)), "scripts",
+                                "alternate-server-replay-{}.py".format(
+                                    self.config["playback_version"]))
+            recording_paths = self.config["playback_files"]
+            # this part is platform-specific
+            if mozinfo.os == "win":
+                script = script.replace("\\", "\\\\\\")
+                recording_paths = [recording_path.replace("\\", "\\\\\\")
+                                   for recording_path in recording_paths]
+
+            if self.config["playback_version"] == "2.0.2":
+                args = [
+                    "--replay-kill-extra",
+                    "-v",
+                    "--script",
+                    '""{} {}""'.format(script, " ".join(recording_paths)),
+                ]
+
+                if not self.config["playback_upstream_cert"]:
+                    LOG.info("No upstream certificate sniffing")
+                    args.insert(0, "--no-upstream-cert")
+                self.playback.config["playback_tool_args"] = args
+            elif self.config["playback_version"] == "4.0.4":
+                args = [
+                    "-v",
+                    "--set",
+                    "websocket=false",
+                    "--set",
+                    "server_replay_files={}".format(" ".join(recording_paths)),
+                    "--scripts",
+                    script,
+                ]
+                if not self.config["playback_upstream_cert"]:
+                    LOG.info("No upstream certificate sniffing")
+                    args = ["--set", "upstream_cert=false"] + args
+                command.extend(args)
+            else:
+                raise Exception("Mitmproxy version is unknown!")
+
+        else:
+            raise Exception("Mitmproxy can't start playback! Playback settings missing.")
 
         LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
         LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command))
         # to turn off mitmproxy log output, use these params for Popen:
         # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
         self.mitmproxy_proc = ProcessHandler(
             command, logfile=os.path.join(self.upload_dir, "mitmproxy.log"), env=env
         )
rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-linux64.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-linux64.manifest
rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-osx.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-osx.manifest
rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-win.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-win.manifest
rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-linux64.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-linux64.manifest
rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-osx.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-osx.manifest
rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-win.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-win.manifest
rename from testing/mozbase/mozproxy/mozproxy/backends/mitmproxy_requirements.txt
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy_requirements.txt
rename from testing/raptor/raptor/playback/alternate-server-replay-2.0.2.py
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-2.0.2.py
rename from testing/raptor/raptor/playback/alternate-server-replay-4.0.4.py
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py
--- a/testing/raptor/raptor/playback/alternate-server-replay-4.0.4.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py
@@ -1,14 +1,15 @@
 # This file was copied from  mitmproxy/mitmproxy/addons/serverplayback.py release tag 4.0.4
 # and modified by Florin Strugariu
 
 # Altered features:
 # * returns 404 rather than dropping the whole HTTP/2 connection on the floor
 # * remove the replay packages that don't have any content in their response package
+from __future__ import absolute_import, print_function
 
 import os
 import json
 import hashlib
 import urllib
 
 import typing
 from urllib import parse
rename from testing/raptor/raptor/playback/scripts/catapult/LICENSE
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/LICENSE
rename from testing/raptor/raptor/playback/scripts/catapult/deterministic.js
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/deterministic.js
rename from testing/raptor/raptor/playback/scripts/inject-deterministic.py
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py
--- a/testing/raptor/raptor/playback/scripts/inject-deterministic.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py
@@ -1,8 +1,10 @@
+from __future__ import absolute_import, print_function
+
 import base64
 import hashlib
 import re
 import time
 
 from mitmproxy import ctx
 
 
--- a/testing/mozbase/mozproxy/tests/test_proxy.py
+++ b/testing/mozbase/mozproxy/tests/test_proxy.py
@@ -26,35 +26,37 @@ class Process:
     pid = 1234
     stderr = stdout = None
 
 
 @mock.patch("mozprocess.processhandler.ProcessHandlerMixin.Process", new=Process)
 @mock.patch("mozproxy.backends.mitm.tooltool_download", new=mock.DEFAULT)
 @mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy", lambda x: True)
 def test_mitm(*args):
-    bin_name = "mitmproxy-rel-bin-{platform}.manifest"
-    pageset_name = "mitmproxy-recordings-raptor-paypal.manifest"
+    bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    pageset_name = "mitm4-linux-firefox-amazon.manifest"
 
     config = {
         "playback_tool": "mitmproxy",
         "playback_binary_manifest": bin_name,
         "playback_pageset_manifest": pageset_name,
+        "playback_upstream_cert": 'false',
+        "playback_version": '4.0.4',
         "platform": mozinfo.os,
         "playback_recordings": os.path.join(here, "paypal.mp"),
         "run_local": True,
         "binary": "firefox",
         "app": "firefox",
         "host": "example.com",
     }
 
     with tempdir() as obj_path:
         config["obj_path"] = obj_path
         playback = get_playback(config)
-
+        playback.config['playback_files'] = config['playback_recordings']
     assert playback is not None
     try:
         playback.start()
     finally:
         playback.stop()
 
 
 @mock.patch("mozprocess.processhandler.ProcessHandlerMixin.Process", new=Process)
@@ -65,39 +67,42 @@ def test_playback_setup_failed(*args):
         pass
 
     def setup(*args, **kw):
         def _s(self):
             raise SetupFailed("Failed")
 
         return _s
 
-    bin_name = "mitmproxy-rel-bin-{platform}.manifest"
-    pageset_name = "mitmproxy-recordings-raptor-paypal.manifest"
+    bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    pageset_name = "mitm4-linux-firefox-amazon.manifest"
 
     config = {
         "playback_tool": "mitmproxy",
         "playback_binary_manifest": bin_name,
         "playback_pageset_manifest": pageset_name,
+        "playback_upstream_cert": 'false',
+        "playback_version": '4.0.4',
         "platform": mozinfo.os,
         "playback_recordings": os.path.join(here, "paypal.mp"),
         "run_local": True,
         "binary": "firefox",
         "app": "firefox",
         "host": "example.com",
     }
 
     prefix = "mozproxy.backends.mitm.MitmproxyDesktop."
 
     with tempdir() as obj_path:
         config["obj_path"] = obj_path
         with mock.patch(prefix + "setup", new_callable=setup):
             with mock.patch(prefix + "stop_mitmproxy_playback") as p:
                 try:
                     pb = get_playback(config)
+                    pb.config['playback_files'] = config['playback_recordings']
                     pb.start()
                 except SetupFailed:
                     assert p.call_count == 1
                 except Exception:
                     raise
 
 
 if __name__ == "__main__":
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -397,63 +397,22 @@ class Raptor(Perftest):
         if self.config['app'] in ['firefox', 'geckoview', 'fennec', 'refbrow', 'fenix']:
             self.profile.addons.remove_addon(self.webext_id)
 
         # for chrome the addon is just a list (appended to cmd line)
         chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"]
         if self.config['app'] in chrome_apps:
             self.profile.addons.remove(self.raptor_webext)
 
-    def get_proxy_command_for_mitm(self, test, version):
-        # Generate Mitmproxy playback args
-        script = os.path.join(here, "playback", "alternate-server-replay-{}.py".format(version))
-
-        recording_paths = self.get_recording_paths(test)
-
-        # this part is platform-specific
-        if mozinfo.os == "win":
-            script = script.replace("\\", "\\\\\\")
-            recording_paths = [recording_path.replace("\\", "\\\\\\")
-                               for recording_path in recording_paths]
-
-        if version == "2.0.2":
-            args = [
-                "--replay-kill-extra",
-                "-v",
-                "--script",
-                '""{} {}""'.format(script, " ".join(recording_paths)),
-            ]
-
-            if not self.config["playback_upstream_cert"]:
-                LOG.info("No upstream certificate sniffing")
-                args.insert(0, "--no-upstream-cert")
-            self.playback.config["playback_tool_args"] = args
-        elif version == "4.0.4":
-            args = [
-                "-v",
-                "--set",
-                "websocket=false",
-                "--set",
-                "server_replay_files={}".format(" ".join(recording_paths)),
-                "--scripts",
-                script,
-            ]
-            if not self.config["playback_upstream_cert"]:
-                LOG.info("No upstream certificate sniffing")
-                args = ["--set", "upstream_cert=false"] + args
-            self.playback.config["playback_tool_args"] = args
-        else:
-            raise Exception("Mitmproxy version is unknown!")
-
     def start_playback(self, test):
         # creating the playback tool
         self.get_playback_config(test)
         self.playback = get_playback(self.config, self.device)
 
-        self.get_proxy_command_for_mitm(test, self.config['playback_version'])
+        self.playback.config['playback_files'] = self.get_recording_paths(test)
 
         # let's start it!
         self.playback.start()
 
         self.log_recording_dates(test)
 
     def get_recording_paths(self, test):
         recordings = test.get("playback_recordings")
--- a/testing/raptor/test/test_playback.py
+++ b/testing/raptor/test/test_playback.py
@@ -1,11 +1,12 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
+import time
 
 import mozinfo
 import mozunit
 
 from mozlog.structuredlog import set_default_logger, StructuredLogger
 
 set_default_logger(StructuredLogger('test_playback'))
 
@@ -22,27 +23,34 @@ if os.environ.get('TOOLTOOLCACHE') is No
 def test_get_playback(get_binary):
     config['platform'] = mozinfo.os
     if 'win' in config['platform']:
         # this test is not yet supported on windows
         assert True
         return
     config['obj_path'] = os.path.dirname(get_binary('firefox'))
     config['playback_tool'] = 'mitmproxy'
-    config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
-    config['playback_pageset_manifest'] = 'mitmproxy-recordings-raptor-tp6-1.manifest'
+    config['playback_version'] = '4.0.4'
+    config['playback_upstream_cert'] = 'false'
+    config['playback_binary_manifest'] = 'mitmproxy-rel-bin-4.0.4-{platform}.manifest'
+    config['playback_pageset_manifest'] = os.path.join(
+        os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "raptor", "playback",
+        'mitm4-linux-firefox-amazon.manifest')
     config['playback_recordings'] = 'amazon.mp'
     config['binary'] = get_binary('firefox')
     config['run_local'] = run_local
     config['app'] = 'firefox'
-    config['host'] = 'example.com'
+    config['host'] = 'https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1'
 
     playback = get_playback(config)
+    playback.config['playback_files'] = [os.path.join(playback.mozproxy_dir,
+                                                      config['playback_recordings'])]
     assert isinstance(playback, MitmproxyDesktop)
     playback.start()
+    time.sleep(1)
     playback.stop()
 
 
 def test_get_unsupported_playback():
     config['playback_tool'] = 'unsupported'
     playback = get_playback(config)
     assert playback is None