--- a/testing/mozharness/configs/talos/linux_config.py
+++ b/testing/mozharness/configs/talos/linux_config.py
@@ -28,16 +28,17 @@ config = {
"title": os.uname()[1].lower().split('.')[0],
"default_actions": [
"clobber",
"read-buildbot-config",
"download-and-extract",
"populate-webroot",
"create-virtualenv",
"install",
+ "setup-mitmproxy",
"run-tests",
],
"default_blob_upload_servers": [
"https://blobupload.elasticbeanstalk.com",
],
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
"download_minidump_stackwalk": True,
"minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
--- a/testing/mozharness/configs/talos/mac_config.py
+++ b/testing/mozharness/configs/talos/mac_config.py
@@ -31,16 +31,17 @@ config = {
"title": os.uname()[1].lower().split('.')[0],
"default_actions": [
"clobber",
"read-buildbot-config",
"download-and-extract",
"populate-webroot",
"create-virtualenv",
"install",
+ "setup-mitmproxy",
"run-tests",
],
"run_cmd_checks_enabled": True,
"preflight_run_cmd_suites": [
SCREEN_RESOLUTION_CHECK,
],
"postflight_run_cmd_suites": [
SCREEN_RESOLUTION_CHECK,
--- a/testing/mozharness/configs/talos/windows_config.py
+++ b/testing/mozharness/configs/talos/windows_config.py
@@ -30,16 +30,17 @@ config = {
"title": socket.gethostname().split('.')[0],
"default_actions": [
"clobber",
"read-buildbot-config",
"download-and-extract",
"populate-webroot",
"create-virtualenv",
"install",
+ "setup-mitmproxy",
"run-tests",
],
"default_blob_upload_servers": [
"https://blobupload.elasticbeanstalk.com",
],
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
"metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
"download_minidump_stackwalk": True,
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -4,16 +4,17 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""
run talos tests in a virtualenv
"""
import os
+import sys
import pprint
import copy
import re
import shutil
import json
import mozharness
from mozharness.base.config import parse_config_file
@@ -29,16 +30,18 @@ from mozharness.mozilla.buildbot import
from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING
from mozharness.mozilla.tooltool import TooltoolMixin
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
'external_tools',
)
+scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
+
TalosErrorList = PythonErrorList + [
{'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
{'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
{'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
{'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
{'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
{'substr': r'''talosError''', 'level': CRITICAL},
{'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
@@ -143,39 +146,43 @@ class Talos(TestingMixin, MercurialScrip
def __init__(self, **kwargs):
kwargs.setdefault('config_options', self.config_options)
kwargs.setdefault('all_actions', ['clobber',
'read-buildbot-config',
'download-and-extract',
'populate-webroot',
'create-virtualenv',
'install',
+ 'setup-mitmproxy',
'run-tests',
])
kwargs.setdefault('default_actions', ['clobber',
'download-and-extract',
'populate-webroot',
'create-virtualenv',
'install',
+ 'setup-mitmproxy',
'run-tests',
])
kwargs.setdefault('config', {})
super(Talos, self).__init__(**kwargs)
self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
self.run_local = self.config.get('run_local')
self.installer_url = self.config.get("installer_url")
self.talos_json_url = self.config.get("talos_json_url")
self.talos_json = self.config.get("talos_json")
self.talos_json_config = self.config.get("talos_json_config")
self.tests = None
self.gecko_profile = self.config.get('gecko_profile')
self.gecko_profile_interval = self.config.get('gecko_profile_interval')
self.pagesets_name = None
+ self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings
+ self.mitmdump = None # path to mitdump tool itself, in py3 venv
# We accept some configuration options from the try commit message in the format mozharness: <options>
# Example try commit message:
# mozharness: --geckoProfile try: <stuff>
def query_gecko_profile_options(self):
gecko_results = []
if self.buildbot_config:
# this is inside automation
@@ -281,16 +288,20 @@ class Talos(TestingMixin, MercurialScrip
if 'suite' in self.config:
kw_options['suite'] = self.config['suite']
if self.config.get('title'):
kw_options['title'] = self.config['title']
if self.config.get('branch'):
kw_options['branchName'] = self.config['branch']
if self.symbols_path:
kw_options['symbolsPath'] = self.symbols_path
+ # if using mitmproxy, we've already created a py3 venv just
+ # for it; need to add the path to that env/mitdump tool
+ if self.mitmdump:
+ kw_options['mitmdumpPath'] = self.mitmdump
kw_options.update(kw)
# talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
tests = kw_options.get('activeTests')
if tests and not isinstance(tests, basestring):
tests = ':'.join(tests) # Talos expects this format
kw_options['activeTests'] = tests
for key, value in kw_options.items():
options.extend(['--%s' % key, value])
@@ -343,16 +354,81 @@ class Talos(TestingMixin, MercurialScrip
)
archive = os.path.join(src_talos_pageset, self.pagesets_name)
unzip = self.query_exe('unzip')
unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
self.run_command(unzip_cmd, halt_on_failure=True)
else:
self.info("Not downloading pageset because the no-download option was specified")
+ def setup_mitmproxy(self):
+ """Some talos tests require the use of mitmproxy to playback the pages,
+ set it up here.
+ """
+ if not self.query_mitmproxy_recording_set():
+ self.info("Skipping: mitmproxy is not required")
+ return
+
+ # setup python 3.x virtualenv
+ self.setup_py3_virtualenv()
+
+ # install mitmproxy
+ self.install_mitmproxy()
+
+ # download the recording set; will be overridden by the --no-download
+ if '--no-download' not in self.config['talos_extra_options']:
+ self.download_mitmproxy_recording_set()
+ else:
+ self.info("Not downloading mitmproxy recording set because no-download was specified")
+
+ def setup_py3_virtualenv(self):
+ """Mitmproxy needs Python 3.x; set up a separate py 3.x env here"""
+ self.info("Setting up python 3.x virtualenv, required for mitmproxy")
+ # first download the py3 package
+ self.py3_path = self.fetch_python3()
+ # now create the py3 venv
+ self.py3_venv_configuration(python_path=self.py3_path, venv_path='py3venv')
+ self.py3_create_venv()
+ requirements = [os.path.join(self.talos_path, 'talos', 'mitmproxy', 'mitmproxy_requirements.txt')]
+ self.py3_install_requirement_files(requirements)
+ # add py3 executables path to system path
+ sys.path.insert(1, self.py3_path_to_executables())
+
+ def install_mitmproxy(self):
+ """Install the mitmproxy tool into the Python 3.x env"""
+ self.info("Installing mitmproxy")
+ self.py3_install_modules(modules=['mitmproxy'])
+ self.mitmdump = os.path.join(self.py3_path_to_executables(), 'mitmdump')
+ self.run_command([self.mitmdump, '--version'], env=self.query_env())
+
+ def query_mitmproxy_recording_set(self):
+ """Mitmproxy requires external playback archives to be downloaded and extracted"""
+ if self.mitmproxy_recording_set:
+ return self.mitmproxy_recording_set
+ if self.query_talos_json_config() and self.suite is not None:
+ self.mitmproxy_recording_set = self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False)
+ if self.mitmproxy_recording_set is False:
+ self.fatal("Aborting: mitmproxy_recording_set is required")
+ return self.mitmproxy_recording_set
+
+ def download_mitmproxy_recording_set(self):
+ """Download the set of mitmproxy recording files that will be played back"""
+ self.info("Downloading the mitmproxy recording set using tooltool")
+ dest = os.path.join(self.talos_path, 'talos', 'mitmproxy')
+ manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', 'mitmproxy-playback-set.manifest')
+ self.tooltool_fetch(
+ manifest_file,
+ output_dir=dest,
+ cache=self.config.get('tooltool_cache')
+ )
+ archive = os.path.join(dest, self.mitmproxy_recording_set)
+ unzip = self.query_exe('unzip')
+ unzip_cmd = [unzip, '-q', '-o', archive, '-d', dest]
+ self.run_command(unzip_cmd, halt_on_failure=True)
+
# Action methods. {{{1
# clobber defined in BaseScript
# read_buildbot_config defined in BuildbotMixin
def download_and_extract(self, extract_dirs=None, suite_categories=None):
return super(Talos, self).download_and_extract(
suite_categories=['common', 'talos']
)
@@ -444,16 +520,19 @@ class Talos(TestingMixin, MercurialScrip
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env, log_level=INFO)
# adjust PYTHONPATH to be able to use talos as a python package
if 'PYTHONPATH' in env:
env['PYTHONPATH'] = self.talos_path + os.pathsep + env['PYTHONPATH']
else:
env['PYTHONPATH'] = self.talos_path
+ # mitmproxy needs path to mozharness when installing the cert
+ env['SCRIPTSPATH'] = scripts_path
+
# sets a timeout for how long talos should run without output
output_timeout = self.config.get('talos_output_timeout', 3600)
# run talos tests
run_tests = os.path.join(self.talos_path, 'talos', 'run_tests.py')
mozlog_opts = ['--log-tbpl-level=debug']
if not self.run_local and 'suite' in self.config:
fname_pattern = '%s_%%s.log' % self.config['suite']
@@ -495,16 +574,18 @@ class Talos(TestingMixin, MercurialScrip
self._artifact_perf_data(dest)
self.buildbot_status(parser.worst_tbpl_status,
level=parser.worst_log_level)
def fetch_python3(self):
manifest_file = os.path.join(
self.talos_path,
+ 'talos',
+ 'mitmproxy',
self.config['python3_manifest'][self.platform_name()])
output_dir = self.query_abs_dirs()['abs_work_dir']
# Slowdown: The unzipped Python3 installation gets deleted every time
self.tooltool_fetch(
manifest_file,
output_dir=output_dir,
cache=self.config['tooltool_cache']
)
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -59,16 +59,17 @@ class TalosRunner(MozbuildObject):
'exes': {
'python': self.python_interp,
'virtualenv': [self.python_interp, self.virtualenv_script]
},
'title': socket.gethostname(),
'default_actions': [
'populate-webroot',
'create-virtualenv',
+ 'setup-mitmproxy',
'run-tests',
],
'download_tooltool': True,
'talos_extra_options': ['--develop'] + self.talos_args,
}
def make_args(self):
self.args = {
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -2,22 +2,24 @@
"talos.zip": {
"url": "http://talos-bundles.pvt.build.mozilla.org/zips/talos.a6052c33d420.zip",
"path": ""
},
"extra_options": {
"android": [ "--apkPath=%(apk_path)s" ]
},
"suites": {
- "chromez": {
- "tests": ["tresize", "tcanvasmark"],
- "talos_options": ["--disable-e10s"]
- },
"chromez-e10s": {
- "tests": ["tresize", "tcanvasmark"]
+ "tests": ["Quantum_1"],
+ "mitmproxy_recording_set": "mitmproxy-recording-set.zip",
+ "talos_options": [
+ "--mitmproxy",
+ "mitmproxy-recording-1.mp",
+ "--firstNonBlankPaint"
+ ]
},
"dromaeojs": {
"tests": ["dromaeo_css", "kraken"],
"talos_options": ["--disable-e10s"]
},
"dromaeojs-e10s": {
"tests": ["dromaeo_css", "kraken"]
},
@@ -111,16 +113,26 @@
},
"xperf-e10s": {
"tests": ["tp5n"],
"pagesets_name": "tp5n.zip",
"talos_options": [
"--xperf_path",
"\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
]
+ },
+ "quantum-pageload-e10s": {
+ "tests": ["Quantum_1"],
+ "mitmproxy_recording_set": "mitmproxy-recording-set.zip",
+ "mitmdump_alt_playback_repo": "https://github.com/bsmedberg/mitmdump-alternate-server-replay",
+ "talos_options": [
+ "--mitmproxy",
+ "mitmproxy-recording-1.mp",
+ "--firstNonBlankPaint"
+ ]
}
},
"mobile-suites": {
"remote-tsvgx": {
"tests": ["tsvgm"],
"talos_options": [
"--noChrome",
"--tppagecycles", "7"
--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -111,16 +111,23 @@ def create_parser(mach_interface=False):
default=os.path.abspath('browser_failures.txt'),
help="Filename to store the errors found during the test."
" Currently used for xperf only.")
add_arg('--noShutdown', dest='shutdown', action='store_true',
help="Record time browser takes to shutdown after testing")
add_arg('--setpref', action='append', default=[], dest="extraPrefs",
metavar="PREF=VALUE",
help="defines an extra user preference")
+ add_arg('--mitmproxy',
+ help='Test uses mitmproxy to serve the pages, specify the '
+ 'path and name of the mitmdump file to playback')
+ add_arg('--mitmdumpPath',
+ help="Path to mitmproxy's mitmdump playback tool")
+ add_arg("--firstNonBlankPaint", action='store_true', dest="first_non_blank_paint",
+ help="Wait for firstNonBlankPaint event before recording the time")
add_arg('--webServer', dest='webserver',
help="DEPRECATED")
if not mach_interface:
add_arg('--develop', action='store_true', default=False,
help="useful for running tests on a developer machine."
" Doesn't upload to the graph servers.")
add_arg("--cycles", type=int,
help="number of browser cycles to run")
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/mitmproxy/alternate-server-replay.py
@@ -0,0 +1,185 @@
+# This file was copied from mitmproxy/addons/serverplayback.py release tag 2.0.2 and modified by
+# Benjamin Smedberg
+
+# Altered features:
+# * --kill returns 404 rather than dropping the whole HTTP/2 connection on the floor
+# * best-match response handling is used to improve success rates
+
+import hashlib
+import urllib
+import sys
+from collections import defaultdict
+from typing import Any # noqa
+from typing import List # noqa
+
+from mitmproxy import ctx
+from mitmproxy import exceptions
+from mitmproxy import io
+from mitmproxy import http
+
+
+class ServerPlayback:
+ def __init__(self, replayfiles):
+ self.options = None
+ self.replayfiles = replayfiles
+ self.flowmap = {}
+
+ def load(self, flows):
+ for i in flows:
+ if i.response:
+ l = self.flowmap.setdefault(self._hash(i.request), [])
+ l.append(i)
+
+ def clear(self):
+ self.flowmap = {}
+
+ def _parse(self, r):
+ """
+ Return (path, queries, formdata, content) for a request.
+ """
+ _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
+ queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
+ queries = defaultdict(list)
+ for k, v in queriesArray:
+ queries[k].append(v)
+
+ content = None
+ formdata = None
+ if r.raw_content != b'':
+ if r.multipart_form:
+ formdata = r.multipart_form
+ elif r.urlencoded_form:
+ formdata = r.urlencoded_form
+ else:
+ content = r.content
+ return (path, queries, formdata, content)
+
+ def _hash(self, r):
+ """
+ Calculates a loose hash of the flow request.
+ """
+ path, queries, _, _ = self._parse(r)
+
+ key = [str(r.port), str(r.scheme), str(r.method), str(path)] # type: List[Any]
+ if not self.options.server_replay_ignore_host:
+ key.append(r.host)
+
+ if len(queries):
+ key.append("?")
+
+ return hashlib.sha256(
+ repr(key).encode("utf8", "surrogateescape")
+ ).digest()
+
+ def _match(self, request_a, request_b):
+ """
+ Calculate a match score between two requests.
+ Match algorithm:
+ * identical query keys: 3 points
+ * matching query param present: 1 point
+ * matching query param value: 3 points
+ * identical form keys: 3 points
+ * matching form param present: 1 point
+ * matching form param value: 3 points
+ * matching body (no multipart or encoded form): 4 points
+ """
+ match = 0
+
+ path_a, queries_a, form_a, content_a = self._parse(request_a)
+ path_b, queries_b, form_b, content_b = self._parse(request_b)
+
+ keys_a = set(queries_a.keys())
+ keys_b = set(queries_b.keys())
+ if keys_a == keys_b:
+ match += 3
+
+ for key in keys_a:
+ values_a = set(queries_a[key])
+ values_b = set(queries_b[key])
+ if len(values_a) == len(values_b):
+ match += 1
+ if values_a == values_b:
+ match += 3
+
+ if form_a and form_b:
+ keys_a = set(form_a.keys())
+ keys_b = set(form_b.keys())
+ if keys_a == keys_b:
+ match += 3
+
+ for key in keys_a:
+ values_a = set(form_a.get_all(key))
+ values_b = set(form_b.get_all(key))
+ if len(values_a) == len(values_b):
+ match += 1
+ if values_a == values_b:
+ match += 3
+
+ elif content_a and (content_a == content_b):
+ match += 4
+
+ return match
+
+ def next_flow(self, request):
+ """
+ Returns the next flow object, or None if no matching flow was
+ found.
+ """
+ hsh = self._hash(request)
+ flows = self.flowmap.get(hsh, None)
+ if flows is None:
+ return None
+
+ # if it's an exact match, great!
+ if len(flows) == 1:
+ candidate = flows[0]
+ if (candidate.request.url == request.url and
+ candidate.request.raw_content == request.raw_content):
+ ctx.log.info("For request {} found exact replay match".format(request.url))
+ return candidate
+
+ # find the best match between the request and the available flow candidates
+ match = -1
+ flow = None
+ ctx.log.debug("Candiate flows for request: {}".format(request.url))
+ for candidate_flow in flows:
+ candidate_match = self._match(candidate_flow.request, request)
+ ctx.log.debug(" score={} url={}".format(candidate_match, candidate_flow.request.url))
+ if candidate_match > match:
+ match = candidate_match
+ flow = candidate_flow
+ ctx.log.info("For request {} best match {} with score=={}".format(request.url,
+ flow.request.url, match))
+ return candidate_flow
+
+ def configure(self, options, updated):
+ self.options = options
+ self.clear()
+ try:
+ flows = io.read_flows_from_paths(self.replayfiles)
+ except exceptions.FlowReadException as e:
+ raise exceptions.OptionsError(str(e))
+ self.load(flows)
+
+ def request(self, f):
+ if self.flowmap:
+ rflow = self.next_flow(f.request)
+ if rflow:
+ response = rflow.response.copy()
+ response.is_replay = True
+ if self.options.refresh_server_playback:
+ response.refresh()
+ f.response = response
+ elif self.options.replay_kill_extra:
+ ctx.log.warn(
+ "server_playback: killed non-replay request {}".format(
+ f.request.url
+ )
+ )
+ f.response = http.HTTPResponse.make(404, b'', {'content-type': 'text/plain'})
+
+
+def start():
+ files = sys.argv[1:]
+ print("Replaying from files: {}".format(files))
+ return ServerPlayback(files)
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/mitmproxy/mitmproxy-playback-set.manifest
@@ -0,0 +1,9 @@
+[
+ {
+ "filename": "mitmproxy-recording-set.zip",
+ "size": 679582,
+ "digest": "035e0729905ea1bb61efa4e720cdbc2a436dfebf364eb0d71a7a9ad04658635ede69aa5ef068e99bd515b0b68e2217a2ffe5961b3e8c7f8ead2c49ddcb9fe879",
+ "algorithm": "sha512",
+ "unpack": false
+ }
+]
\ No newline at end of file
rename from testing/mozharness/mozharness/mozilla/mitmproxy.py
rename to testing/talos/talos/mitmproxy/mitmproxy.py
--- a/testing/mozharness/mozharness/mozilla/mitmproxy.py
+++ b/testing/talos/talos/mitmproxy/mitmproxy.py
@@ -1,17 +1,33 @@
'''This helps loading mitmproxy's cert and change proxy settings for Firefox.'''
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
-from mozharness.mozilla.firefox.autoconfig import write_autoconfig_files
+import sys
+import subprocess
+import time
+
+import mozinfo
+
+from mozlog import get_proxy_logger
-DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
- '.mitmproxy', 'mitmproxy-ca-cert.cer')
+here = os.path.dirname(os.path.realpath(__file__))
+LOG = get_proxy_logger()
+
+# path for mitmproxy certificate, generated auto after mitmdump is started
+# on local machine it is 'HOME', however it is different on production machines
+try:
+ DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
+ '.mitmproxy', 'mitmproxy-ca-cert.cer')
+except:
+ DEFAULT_CERT_PATH = os.path.join(os.getenv('HOMEDRIVE'), os.getenv('HOMEPATH'),
+ '.mitmproxy', 'mitmproxy-ca-cert.cer')
+
MITMPROXY_SETTINGS = '''// Start with a comment
// Load up mitmproxy cert
var Cc = Components.classes;
var Ci = Components.interfaces;
var certdb = Cc["@mozilla.org/security/x509certdb;1"].getService(Ci.nsIX509CertDB);
var certdb2 = certdb;
try {
@@ -26,22 +42,115 @@ certdb2.addCertFromBase64(cert, "C,C,C",
pref("network.proxy.type", 1);
pref("network.proxy.http", "127.0.0.1");
pref("network.proxy.http_port", 8080);
pref("network.proxy.ssl", "127.0.0.1");
pref("network.proxy.ssl_port", 8080);
'''
-def configure_mitmproxy(fx_install_dir, certificate_path=DEFAULT_CERT_PATH):
- certificate = _read_certificate(certificate_path)
- write_autoconfig_files(fx_install_dir=fx_install_dir,
- cfg_contents=MITMPROXY_SETTINGS % {
- 'cert': certificate})
+def configure_mitmproxy(fx_install_dir,
+ scripts_path,
+ certificate_path=DEFAULT_CERT_PATH):
+ # sys.path.insert(1, "c:\\slave\\test\\scripts\\")
+ # sys.path.insert(1, os.path.join(os.path.dirname(sys.path[1]), 'mozharness'))
+ # scripts_path is path to mozharness on test machine; needed so can import
+ if scripts_path is not False:
+ LOG.info("RW: in conifgure_mitmproxy, sys.path before addition:")
+ LOG.info(str(sys.path))
+ sys.path.insert(1, scripts_path)
+ sys.path.insert(1, os.path.join(scripts_path, 'mozharness'))
+ LOG.info("RW: in configure_mitmproxy, sys.path after addition:")
+ LOG.info(str(sys.path))
+ from mozharness.mozilla.firefox.autoconfig import write_autoconfig_files
+ certificate = _read_certificate(certificate_path)
+ write_autoconfig_files(fx_install_dir=fx_install_dir,
+ cfg_contents=MITMPROXY_SETTINGS % {
+ 'cert': certificate})
def _read_certificate(certificate_path):
''' Return the certificate's hash from the certificate file.'''
# NOTE: mitmproxy's certificates do not exist until one of its binaries
# has been executed once on the host
with open(certificate_path, 'r') as fd:
contents = fd.read()
return ''.join(contents.splitlines()[1:-1])
+
+
+def is_mitmproxy_cert_installed():
+ """Verify mitmxproy CA cert was added to Firefox"""
+ # TODO: Bug 1366071
+ return True
+
+
+def install_mitmproxy_cert(mitmproxy_proc, browser_path, scripts_path):
+ """Install the CA certificate generated by mitmproxy, into Firefox"""
+ LOG.info("Installing mitmxproxy CA certficate into Firefox")
+ # browser_path is exe, we want install dir
+ browser_install = os.path.dirname(browser_path)
+ LOG.info('Calling configure_mitmproxy with browser folder: %s' % browser_install)
+ configure_mitmproxy(browser_install, scripts_path)
+ # cannot continue if failed to add CA cert to Firefox, need to check
+ if not is_mitmproxy_cert_installed():
+ LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox')
+ stop_mitmproxy_playback(mitmproxy_proc)
+ sys.exit()
+
+
+def start_mitmproxy_playback(mitmdump_path,
+ mitmproxy_recording_path,
+ mitmproxy_recordings_list,
+ browser_path):
+ """Startup mitmproxy and replay the specified flow file"""
+ mitmproxy_recordings = []
+ # recording names can be provided in comma-separated list; build py list including path
+ for recording in mitmproxy_recordings_list:
+ mitmproxy_recordings.append(os.path.join(mitmproxy_recording_path, recording))
+
+ # cmd line to start mitmproxy playback using custom playback script is as follows:
+ # <path>/mitmdump -s "<path>mitmdump-alternate-server-replay/alternate-server-replay.py
+ # <path>recording-1.mp <path>recording-2.mp..."
+ param = os.path.join(here, 'alternate-server-replay.py')
+
+ # this part is platform-specific
+ if mozinfo.os == 'win':
+ param2 = '""' + param.replace('\\', '\\\\\\') + ' ' + \
+ ' '.join(mitmproxy_recordings).replace('\\', '\\\\\\') + '""'
+ env = os.environ.copy()
+ sys.path.insert(1, mitmdump_path)
+ # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
+ env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
+ else:
+ # TODO: support other platforms, Bug 1366355
+ LOG.error('Aborting: talos mitmproxy is currently only supported on Windows')
+ sys.exit()
+
+ command = [mitmdump_path, '-s', param2]
+
+ LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
+ LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command))
+ # to turn off mitmproxy log output, use these params for Popen:
+ # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+ mitmproxy_proc = subprocess.Popen(command, env=env)
+ time.sleep(5)
+ data = mitmproxy_proc.poll()
+ if data is None:
+ LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid)
+ return mitmproxy_proc
+ # cannot continue as we won't be able to playback the pages
+ LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data)
+ sys.exit()
+
+
+def stop_mitmproxy_playback(mitmproxy_proc):
+ """Stop the mitproxy server playback"""
+ LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid)
+ mitmproxy_proc.kill()
+ time.sleep(5)
+ exit_code = mitmproxy_proc.poll()
+ if exit_code:
+ LOG.info("Successfully killed the mitmproxy playback process")
+ else:
+ # I *think* we can still continue, as process will be automatically
+ # killed anyway when mozharness is done (?) if not, we won't be able
+ # to startup mitmxproy next time if it is already running
+ LOG.error("Failed to kill the mitmproxy playback process")
rename from testing/talos/mitmproxy_requirements.txt
rename to testing/talos/talos/mitmproxy/mitmproxy_requirements.txt
rename from testing/talos/python3.manifest
rename to testing/talos/talos/mitmproxy/python3.manifest
rename from testing/talos/python3_x64.manifest
rename to testing/talos/talos/mitmproxy/python3_x64.manifest
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -10,16 +10,17 @@ import sys
import time
import traceback
import urllib
import utils
import mozhttpd
from mozlog import get_proxy_logger
+from talos.mitmproxy import mitmproxy
from talos.results import TalosResults
from talos.ttest import TTest
from talos.utils import TalosError, TalosRegression
from talos.config import get_configs, ConfigurationError
# directory of this file
here = os.path.dirname(os.path.realpath(__file__))
LOG = get_proxy_logger()
@@ -84,17 +85,16 @@ def setup_webserver(webserver):
def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
-
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
for test in tests:
# Check for profile_path, tpmanifest and interpolate based on Talos
# root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
# Build command line from config
for path in paths:
if test.get(path):
@@ -186,16 +186,48 @@ def run_tests(config, browser_config):
# if e10s add as extra results option
if config['e10s']:
talos_results.add_extra_option('e10s')
if config['gecko_profile']:
talos_results.add_extra_option('geckoProfile')
+ # some tests use mitmproxy to playback pages
+ mitmproxy_recordings_list = config.get('mitmproxy', False)
+ if mitmproxy_recordings_list is not False:
+ # needed so can tell talos ttest to allow external connections
+ browser_config['mitmproxy'] = True
+
+ # start mitmproxy playback; this also generates the CA certificate
+ mitmdump_path = config.get('mitmdumpPath', False)
+ if mitmdump_path is False:
+ # cannot continue, need path for mitmdump playback tool
+ LOG.error('Aborting: mitmdumpPath was not provided on cmd line but is required')
+ sys.exit()
+
+ mitmproxy_recording_path = os.path.join(here, 'mitmproxy')
+ mitmproxy_proc = mitmproxy.start_mitmproxy_playback(mitmdump_path,
+ mitmproxy_recording_path,
+ mitmproxy_recordings_list.split(),
+ browser_config['browser_path'])
+
+ # install the generated CA certificate into Firefox
+ # mitmproxy cert setup needs path to mozharness install; mozharness has set this
+ # value in the SCRIPTSPATH env var for us in mozharness/mozilla/testing/talos.py
+ scripts_path = os.environ.get('SCRIPTSPATH')
+ LOG.info('RW: scripts_path:')
+ LOG.info(str(scripts_path))
+ mitmproxy.install_mitmproxy_cert(mitmproxy_proc,
+ browser_config['browser_path'],
+ str(scripts_path))
+
+ if config.get('first_non_blank_paint', False):
+ browser_config['firstNonBlankPaint'] = True
+
testname = None
# run the tests
timer = utils.Timer()
LOG.suite_start(tests=[test['name'] for test in tests])
try:
for test in tests:
testname = test['name']
LOG.test_start(testname)
@@ -221,16 +253,20 @@ def run_tests(config, browser_config):
# indicate a failure to buildbot, turn the job red
return 2
finally:
LOG.suite_end()
httpd.stop()
LOG.info("Completed test suite (%s)" % timer.elapsed())
+ # if mitmproxy was used for page playback, stop it
+ if mitmproxy_recordings_list is not False:
+ mitmproxy.stop_mitmproxy_playback(mitmproxy_proc)
+
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop'] or config['gecko_profile']:
print("Thanks for running Talos locally. Results are in %s"
% (results_urls['output_urls']))
# we will stop running tests on a failed test, or we will return 0 for
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -774,8 +774,23 @@ class bloom_basic_ref(PageloaderTest):
tpcycles = 1
tppagecycles = 25
gecko_profile_interval = 1
gecko_profile_entries = 2000000
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
unit = 'ms'
lower_is_better = True
alert_threshold = 5.0
+
+
+@register_test()
+class Quantum_1(PageloaderTest):
+ """
+ Quantum Pageload Test 1
+ """
+ tpmanifest = '${talos}/tests/quantum_pageload/quantum_1.manifest'
+ tpcycles = 1
+ tppagecycles = 25
+ gecko_profile_interval = 1
+ gecko_profile_entries = 2000000
+ filters = filter.ignore_first.prepare(5) + filter.median.prepare()
+ unit = 'ms'
+ lower_is_better = True
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/quantum_pageload/quantum_1.manifest
@@ -0,0 +1,1 @@
+https://www.google.com
--- a/testing/talos/talos/ttest.py
+++ b/testing/talos/talos/ttest.py
@@ -99,16 +99,21 @@ class TTest(object):
if test_config.get('responsiveness') and \
platform.system() != "Darwin":
# ignore osx for now as per bug 1245793
setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
global_counters['responsiveness'] = []
+ # if using mitmproxy we must allow access to 'external' sites
+ if browser_config.get('mitmproxy', False):
+ LOG.info("Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0")
+ setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0'
+
# instantiate an object to hold test results
test_results = results.TestResults(
test_config,
global_counters,
browser_config.get('framework')
)
for i in range(test_config['cycles']):