Bug 1357382 - Add mitmproxy support to mozharness and talos, add first rev of quantum-pageload talos test; r=jmaher
authorRob Wood <rwood@mozilla.com>
Wed, 17 May 2017 12:14:59 -0400
changeset 409104 d3e552f716987251e8b3b776a9c3759dfb8f6c0b
parent 409103 59bbdf0d3ba2196e6cac27b38de450a8743df839
child 409105 fd1d772083aacbddb21a0bb23dfe85eb49fb3d8d
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1357382
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1357382 - Add mitmproxy support to mozharness and talos, add first rev of quantum-pageload talos test; r=jmaher MozReview-Commit-ID: 6947tGS9U9Y
testing/mozharness/configs/talos/linux_config.py
testing/mozharness/configs/talos/mac_config.py
testing/mozharness/configs/talos/windows_config.py
testing/mozharness/mozharness/mozilla/mitmproxy.py
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/mach_commands.py
testing/talos/mitmproxy_requirements.txt
testing/talos/python3.manifest
testing/talos/python3_x64.manifest
testing/talos/talos.json
testing/talos/talos/cmdline.py
testing/talos/talos/mitmproxy/__init__.py
testing/talos/talos/mitmproxy/alternate-server-replay.py
testing/talos/talos/mitmproxy/mitmproxy-playback-set.manifest
testing/talos/talos/mitmproxy/mitmproxy.py
testing/talos/talos/mitmproxy/mitmproxy_requirements.txt
testing/talos/talos/mitmproxy/python3.manifest
testing/talos/talos/mitmproxy/python3_x64.manifest
testing/talos/talos/run_tests.py
testing/talos/talos/test.py
testing/talos/talos/tests/quantum_pageload/quantum_1.manifest
testing/talos/talos/ttest.py
--- a/testing/mozharness/configs/talos/linux_config.py
+++ b/testing/mozharness/configs/talos/linux_config.py
@@ -28,16 +28,17 @@ config = {
     "title": os.uname()[1].lower().split('.')[0],
     "default_actions": [
         "clobber",
         "read-buildbot-config",
         "download-and-extract",
         "populate-webroot",
         "create-virtualenv",
         "install",
+        "setup-mitmproxy",
         "run-tests",
     ],
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
--- a/testing/mozharness/configs/talos/mac_config.py
+++ b/testing/mozharness/configs/talos/mac_config.py
@@ -31,16 +31,17 @@ config = {
     "title": os.uname()[1].lower().split('.')[0],
     "default_actions": [
         "clobber",
         "read-buildbot-config",
         "download-and-extract",
         "populate-webroot",
         "create-virtualenv",
         "install",
+        "setup-mitmproxy",
         "run-tests",
     ],
     "run_cmd_checks_enabled": True,
     "preflight_run_cmd_suites": [
         SCREEN_RESOLUTION_CHECK,
     ],
     "postflight_run_cmd_suites": [
         SCREEN_RESOLUTION_CHECK,
--- a/testing/mozharness/configs/talos/windows_config.py
+++ b/testing/mozharness/configs/talos/windows_config.py
@@ -30,16 +30,17 @@ config = {
     "title": socket.gethostname().split('.')[0],
     "default_actions": [
         "clobber",
         "read-buildbot-config",
         "download-and-extract",
         "populate-webroot",
         "create-virtualenv",
         "install",
+        "setup-mitmproxy",
         "run-tests",
     ],
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
     "download_minidump_stackwalk": True,
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -4,16 +4,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """
 run talos tests in a virtualenv
 """
 
 import os
+import sys
 import pprint
 import copy
 import re
 import shutil
 import json
 
 import mozharness
 from mozharness.base.config import parse_config_file
@@ -24,20 +25,18 @@ from mozharness.base.python import Pytho
 from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.testing.errors import TinderBoxPrintRe
 from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
 from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING
 from mozharness.mozilla.tooltool import TooltoolMixin
 
-external_tools_path = os.path.join(
-    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
-    'external_tools',
-)
+scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
+external_tools_path = os.path.join(scripts_path, 'external_tools')
 
 TalosErrorList = PythonErrorList + [
     {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
     {'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
     {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
     {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
     {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
     {'substr': r'''talosError''', 'level': CRITICAL},
@@ -143,39 +142,43 @@ class Talos(TestingMixin, MercurialScrip
     def __init__(self, **kwargs):
         kwargs.setdefault('config_options', self.config_options)
         kwargs.setdefault('all_actions', ['clobber',
                                           'read-buildbot-config',
                                           'download-and-extract',
                                           'populate-webroot',
                                           'create-virtualenv',
                                           'install',
+                                          'setup-mitmproxy',
                                           'run-tests',
                                           ])
         kwargs.setdefault('default_actions', ['clobber',
                                               'download-and-extract',
                                               'populate-webroot',
                                               'create-virtualenv',
                                               'install',
+                                              'setup-mitmproxy',
                                               'run-tests',
                                               ])
         kwargs.setdefault('config', {})
         super(Talos, self).__init__(**kwargs)
 
         self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
 
         self.run_local = self.config.get('run_local')
         self.installer_url = self.config.get("installer_url")
         self.talos_json_url = self.config.get("talos_json_url")
         self.talos_json = self.config.get("talos_json")
         self.talos_json_config = self.config.get("talos_json_config")
         self.tests = None
         self.gecko_profile = self.config.get('gecko_profile')
         self.gecko_profile_interval = self.config.get('gecko_profile_interval')
         self.pagesets_name = None
+        self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings
+        self.mitmdump = None # path to mitdump tool itself, in py3 venv
 
     # We accept some configuration options from the try commit message in the format mozharness: <options>
     # Example try commit message:
     #   mozharness: --geckoProfile try: <stuff>
     def query_gecko_profile_options(self):
         gecko_results = []
         if self.buildbot_config:
             # this is inside automation
@@ -281,16 +284,20 @@ class Talos(TestingMixin, MercurialScrip
         if 'suite' in self.config:
             kw_options['suite'] = self.config['suite']
         if self.config.get('title'):
             kw_options['title'] = self.config['title']
         if self.config.get('branch'):
             kw_options['branchName'] = self.config['branch']
         if self.symbols_path:
             kw_options['symbolsPath'] = self.symbols_path
+        # if using mitmproxy, we've already created a py3 venv just
+        # for it; need to add the path to that env/mitdump tool
+        if self.mitmdump:
+            kw_options['mitmdumpPath'] = self.mitmdump
         kw_options.update(kw)
         # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
         tests = kw_options.get('activeTests')
         if tests and not isinstance(tests, basestring):
             tests = ':'.join(tests)  # Talos expects this format
             kw_options['activeTests'] = tests
         for key, value in kw_options.items():
             options.extend(['--%s' % key, value])
@@ -343,16 +350,79 @@ class Talos(TestingMixin, MercurialScrip
                 )
                 archive = os.path.join(src_talos_pageset, self.pagesets_name)
                 unzip = self.query_exe('unzip')
                 unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
                 self.run_command(unzip_cmd, halt_on_failure=True)
             else:
                 self.info("Not downloading pageset because the no-download option was specified")
 
+    def setup_mitmproxy(self):
+        """Some talos tests require the use of mitmproxy to playback the pages,
+        set it up here.
+        """
+        if not self.query_mitmproxy_recording_set():
+            self.info("Skipping: mitmproxy is not required")
+            return
+
+        # setup python 3.x virtualenv
+        self.setup_py3_virtualenv()
+
+        # install mitmproxy
+        self.install_mitmproxy()
+
+        # download the recording set; will be overridden by the --no-download
+        if '--no-download' not in self.config['talos_extra_options']:
+            self.download_mitmproxy_recording_set()
+        else:
+            self.info("Not downloading mitmproxy recording set because no-download was specified")
+
+    def setup_py3_virtualenv(self):
+        """Mitmproxy needs Python 3.x; set up a separate py 3.x env here"""
+        self.info("Setting up python 3.x virtualenv, required for mitmproxy")
+        # first download the py3 package
+        self.py3_path = self.fetch_python3()
+        # now create the py3 venv
+        self.py3_venv_configuration(python_path=self.py3_path, venv_path='py3venv')
+        self.py3_create_venv()
+        requirements = [os.path.join(self.talos_path, 'talos', 'mitmproxy', 'mitmproxy_requirements.txt')]
+        self.py3_install_requirement_files(requirements)
+        # add py3 executables path to system path
+        sys.path.insert(1, self.py3_path_to_executables())
+
+    def install_mitmproxy(self):
+        """Install the mitmproxy tool into the Python 3.x env"""
+        self.info("Installing mitmproxy")
+        self.py3_install_modules(modules=['mitmproxy'])
+        self.mitmdump = os.path.join(self.py3_path_to_executables(), 'mitmdump')
+        self.run_command([self.mitmdump, '--version'], env=self.query_env())
+
+    def query_mitmproxy_recording_set(self):
+        """Mitmproxy requires external playback archives to be downloaded and extracted"""
+        if self.mitmproxy_recording_set:
+            return self.mitmproxy_recording_set
+        if self.query_talos_json_config() and self.suite is not None:
+            self.mitmproxy_recording_set = self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False)
+            return self.mitmproxy_recording_set
+
+    def download_mitmproxy_recording_set(self):
+        """Download the set of mitmproxy recording files that will be played back"""
+        self.info("Downloading the mitmproxy recording set using tooltool")
+        dest = os.path.join(self.talos_path, 'talos', 'mitmproxy')
+        manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', 'mitmproxy-playback-set.manifest')
+        self.tooltool_fetch(
+            manifest_file,
+            output_dir=dest,
+            cache=self.config.get('tooltool_cache')
+        )
+        archive = os.path.join(dest, self.mitmproxy_recording_set)
+        unzip = self.query_exe('unzip')
+        unzip_cmd = [unzip, '-q', '-o', archive, '-d', dest]
+        self.run_command(unzip_cmd, halt_on_failure=True)
+
     # Action methods. {{{1
     # clobber defined in BaseScript
     # read_buildbot_config defined in BuildbotMixin
 
     def download_and_extract(self, extract_dirs=None, suite_categories=None):
         return super(Talos, self).download_and_extract(
             suite_categories=['common', 'talos']
         )
@@ -444,16 +514,19 @@ class Talos(TestingMixin, MercurialScrip
             self.mkdir_p(env['MOZ_UPLOAD_DIR'])
         env = self.query_env(partial_env=env, log_level=INFO)
         # adjust PYTHONPATH to be able to use talos as a python package
         if 'PYTHONPATH' in env:
             env['PYTHONPATH'] = self.talos_path + os.pathsep + env['PYTHONPATH']
         else:
             env['PYTHONPATH'] = self.talos_path
 
+        # mitmproxy needs path to mozharness when installing the cert
+        env['SCRIPTSPATH'] = scripts_path
+
         # sets a timeout for how long talos should run without output
         output_timeout = self.config.get('talos_output_timeout', 3600)
         # run talos tests
         run_tests = os.path.join(self.talos_path, 'talos', 'run_tests.py')
 
         mozlog_opts = ['--log-tbpl-level=debug']
         if not self.run_local and 'suite' in self.config:
             fname_pattern = '%s_%%s.log' % self.config['suite']
@@ -495,16 +568,18 @@ class Talos(TestingMixin, MercurialScrip
                     self._artifact_perf_data(dest)
 
         self.buildbot_status(parser.worst_tbpl_status,
                              level=parser.worst_log_level)
 
     def fetch_python3(self):
         manifest_file = os.path.join(
             self.talos_path,
+            'talos',
+            'mitmproxy',
             self.config['python3_manifest'][self.platform_name()])
         output_dir = self.query_abs_dirs()['abs_work_dir']
         # Slowdown: The unzipped Python3 installation gets deleted every time
         self.tooltool_fetch(
             manifest_file,
             output_dir=output_dir,
             cache=self.config['tooltool_cache']
         )
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -59,16 +59,17 @@ class TalosRunner(MozbuildObject):
             'exes': {
                 'python': self.python_interp,
                 'virtualenv': [self.python_interp, self.virtualenv_script]
             },
             'title': socket.gethostname(),
             'default_actions': [
                 'populate-webroot',
                 'create-virtualenv',
+                'setup-mitmproxy',
                 'run-tests',
             ],
             'download_tooltool': True,
             'talos_extra_options': ['--develop'] + self.talos_args,
         }
 
     def make_args(self):
         self.args = {
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -2,20 +2,16 @@
     "talos.zip": {
         "url": "http://talos-bundles.pvt.build.mozilla.org/zips/talos.a6052c33d420.zip",
         "path": ""
     },
     "extra_options": {
         "android": [ "--apkPath=%(apk_path)s" ]
     },
     "suites": {
-        "chromez": {
-            "tests": ["tresize", "tcanvasmark"],
-            "talos_options": ["--disable-e10s"]
-        },
         "chromez-e10s": {
             "tests": ["tresize", "tcanvasmark"]
         },
         "dromaeojs": {
             "tests": ["dromaeo_css", "kraken"],
             "talos_options": ["--disable-e10s"]
         },
         "dromaeojs-e10s": {
@@ -111,16 +107,25 @@
         },
         "xperf-e10s": {
             "tests": ["tp5n"],
             "pagesets_name": "tp5n.zip",
             "talos_options": [
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
             ]
+        },
+        "quantum-pageload-e10s": {
+            "tests": ["Quantum_1"],
+            "mitmproxy_recording_set": "mitmproxy-recording-set.zip",
+            "talos_options": [
+                "--mitmproxy",
+                "mitmproxy-recording-1.mp",
+                "--firstNonBlankPaint"
+            ]
         }
     },
     "mobile-suites": {
         "remote-tsvgx": {
             "tests": ["tsvgm"],
             "talos_options": [
                 "--noChrome",
                 "--tppagecycles", "7"
--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -111,16 +111,23 @@ def create_parser(mach_interface=False):
             default=os.path.abspath('browser_failures.txt'),
             help="Filename to store the errors found during the test."
                  " Currently used for xperf only.")
     add_arg('--noShutdown', dest='shutdown', action='store_true',
             help="Record time browser takes to shutdown after testing")
     add_arg('--setpref', action='append', default=[], dest="extraPrefs",
             metavar="PREF=VALUE",
             help="defines an extra user preference")
+    add_arg('--mitmproxy',
+            help='Test uses mitmproxy to serve the pages, specify the '
+                 'path and name of the mitmdump file to playback')
+    add_arg('--mitmdumpPath',
+            help="Path to mitmproxy's mitmdump playback tool")
+    add_arg("--firstNonBlankPaint", action='store_true', dest="first_non_blank_paint",
+            help="Wait for firstNonBlankPaint event before recording the time")
     add_arg('--webServer', dest='webserver',
             help="DEPRECATED")
     if not mach_interface:
         add_arg('--develop', action='store_true', default=False,
                 help="useful for running tests on a developer machine."
                      " Doesn't upload to the graph servers.")
     add_arg("--cycles", type=int,
             help="number of browser cycles to run")
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/mitmproxy/alternate-server-replay.py
@@ -0,0 +1,185 @@
+# This file was copied from mitmproxy/addons/serverplayback.py release tag 2.0.2 and modified by
+# Benjamin Smedberg
+
+# Altered features:
+# * --kill returns 404 rather than dropping the whole HTTP/2 connection on the floor
+# * best-match response handling is used to improve success rates
+
+import hashlib
+import urllib
+import sys
+from collections import defaultdict
+from typing import Any  # noqa
+from typing import List  # noqa
+
+from mitmproxy import ctx
+from mitmproxy import exceptions
+from mitmproxy import io
+from mitmproxy import http
+
+
+class ServerPlayback:
+    def __init__(self, replayfiles):
+        self.options = None
+        self.replayfiles = replayfiles
+        self.flowmap = {}
+
+    def load(self, flows):
+        for i in flows:
+            if i.response:
+                l = self.flowmap.setdefault(self._hash(i.request), [])
+                l.append(i)
+
+    def clear(self):
+        self.flowmap = {}
+
+    def _parse(self, r):
+        """
+            Return (path, queries, formdata, content) for a request.
+        """
+        _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
+        queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
+        queries = defaultdict(list)
+        for k, v in queriesArray:
+            queries[k].append(v)
+
+        content = None
+        formdata = None
+        if r.raw_content != b'':
+            if r.multipart_form:
+                formdata = r.multipart_form
+            elif r.urlencoded_form:
+                formdata = r.urlencoded_form
+            else:
+                content = r.content
+        return (path, queries, formdata, content)
+
+    def _hash(self, r):
+        """
+            Calculates a loose hash of the flow request.
+        """
+        path, queries, _, _ = self._parse(r)
+
+        key = [str(r.port), str(r.scheme), str(r.method), str(path)]  # type: List[Any]
+        if not self.options.server_replay_ignore_host:
+            key.append(r.host)
+
+        if len(queries):
+            key.append("?")
+
+        return hashlib.sha256(
+            repr(key).encode("utf8", "surrogateescape")
+        ).digest()
+
+    def _match(self, request_a, request_b):
+        """
+            Calculate a match score between two requests.
+            Match algorithm:
+              * identical query keys: 3 points
+              * matching query param present: 1 point
+              * matching query param value: 3 points
+              * identical form keys: 3 points
+              * matching form param present: 1 point
+              * matching form param value: 3 points
+              * matching body (no multipart or encoded form): 4 points
+        """
+        match = 0
+
+        path_a, queries_a, form_a, content_a = self._parse(request_a)
+        path_b, queries_b, form_b, content_b = self._parse(request_b)
+
+        keys_a = set(queries_a.keys())
+        keys_b = set(queries_b.keys())
+        if keys_a == keys_b:
+            match += 3
+
+        for key in keys_a:
+            values_a = set(queries_a[key])
+            values_b = set(queries_b[key])
+            if len(values_a) == len(values_b):
+                match += 1
+            if values_a == values_b:
+                match += 3
+
+        if form_a and form_b:
+            keys_a = set(form_a.keys())
+            keys_b = set(form_b.keys())
+            if keys_a == keys_b:
+                match += 3
+
+            for key in keys_a:
+                values_a = set(form_a.get_all(key))
+                values_b = set(form_b.get_all(key))
+                if len(values_a) == len(values_b):
+                    match += 1
+                if values_a == values_b:
+                    match += 3
+
+        elif content_a and (content_a == content_b):
+            match += 4
+
+        return match
+
+    def next_flow(self, request):
+        """
+            Returns the next flow object, or None if no matching flow was
+            found.
+        """
+        hsh = self._hash(request)
+        flows = self.flowmap.get(hsh, None)
+        if flows is None:
+            return None
+
+        # if it's an exact match, great!
+        if len(flows) == 1:
+            candidate = flows[0]
+            if (candidate.request.url == request.url and
+               candidate.request.raw_content == request.raw_content):
+                ctx.log.info("For request {} found exact replay match".format(request.url))
+                return candidate
+
+        # find the best match between the request and the available flow candidates
+        match = -1
+        flow = None
+        ctx.log.debug("Candiate flows for request: {}".format(request.url))
+        for candidate_flow in flows:
+            candidate_match = self._match(candidate_flow.request, request)
+            ctx.log.debug("  score={} url={}".format(candidate_match, candidate_flow.request.url))
+            if candidate_match > match:
+                match = candidate_match
+                flow = candidate_flow
+        ctx.log.info("For request {} best match {} with score=={}".format(request.url,
+                     flow.request.url, match))
+        return candidate_flow
+
+    def configure(self, options, updated):
+        self.options = options
+        self.clear()
+        try:
+            flows = io.read_flows_from_paths(self.replayfiles)
+        except exceptions.FlowReadException as e:
+            raise exceptions.OptionsError(str(e))
+        self.load(flows)
+
+    def request(self, f):
+        if self.flowmap:
+            rflow = self.next_flow(f.request)
+            if rflow:
+                response = rflow.response.copy()
+                response.is_replay = True
+                if self.options.refresh_server_playback:
+                    response.refresh()
+                f.response = response
+            elif self.options.replay_kill_extra:
+                ctx.log.warn(
+                    "server_playback: killed non-replay request {}".format(
+                        f.request.url
+                    )
+                )
+                f.response = http.HTTPResponse.make(404, b'', {'content-type': 'text/plain'})
+
+
+def start():
+    files = sys.argv[1:]
+    print("Replaying from files: {}".format(files))
+    return ServerPlayback(files)
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/mitmproxy/mitmproxy-playback-set.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "mitmproxy-recording-set.zip",
+        "size": 679582,
+        "digest": "035e0729905ea1bb61efa4e720cdbc2a436dfebf364eb0d71a7a9ad04658635ede69aa5ef068e99bd515b0b68e2217a2ffe5961b3e8c7f8ead2c49ddcb9fe879",
+        "algorithm": "sha512",
+        "unpack": false
+    }
+]
\ No newline at end of file
rename from testing/mozharness/mozharness/mozilla/mitmproxy.py
rename to testing/talos/talos/mitmproxy/mitmproxy.py
--- a/testing/mozharness/mozharness/mozilla/mitmproxy.py
+++ b/testing/talos/talos/mitmproxy/mitmproxy.py
@@ -1,17 +1,33 @@
 '''This helps loading mitmproxy's cert and change proxy settings for Firefox.'''
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
-from mozharness.mozilla.firefox.autoconfig import write_autoconfig_files
+import sys
+import subprocess
+import time
+
+import mozinfo
+
+from mozlog import get_proxy_logger
 
-DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
-                                 '.mitmproxy', 'mitmproxy-ca-cert.cer')
+here = os.path.dirname(os.path.realpath(__file__))
+LOG = get_proxy_logger()
+
+# path for mitmproxy certificate, generated auto after mitmdump is started
+# on local machine it is 'HOME', however it is different on production machines
+try:
+    DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
+                                     '.mitmproxy', 'mitmproxy-ca-cert.cer')
+except:
+    DEFAULT_CERT_PATH = os.path.join(os.getenv('HOMEDRIVE'), os.getenv('HOMEPATH'),
+                                     '.mitmproxy', 'mitmproxy-ca-cert.cer')
+
 MITMPROXY_SETTINGS = '''// Start with a comment
 // Load up mitmproxy cert
 var Cc = Components.classes;
 var Ci = Components.interfaces;
 var certdb = Cc["@mozilla.org/security/x509certdb;1"].getService(Ci.nsIX509CertDB);
 var certdb2 = certdb;
 
 try {
@@ -26,22 +42,109 @@ certdb2.addCertFromBase64(cert, "C,C,C",
 pref("network.proxy.type", 1);
 pref("network.proxy.http", "127.0.0.1");
 pref("network.proxy.http_port", 8080);
 pref("network.proxy.ssl", "127.0.0.1");
 pref("network.proxy.ssl_port", 8080);
 '''
 
 
-def configure_mitmproxy(fx_install_dir, certificate_path=DEFAULT_CERT_PATH):
-        certificate = _read_certificate(certificate_path)
-        write_autoconfig_files(fx_install_dir=fx_install_dir,
-                               cfg_contents=MITMPROXY_SETTINGS % {
-                                  'cert': certificate})
+def configure_mitmproxy(fx_install_dir,
+                        scripts_path,
+                        certificate_path=DEFAULT_CERT_PATH):
+    # scripts_path is path to mozharness on test machine; needed so can import
+    if scripts_path is not False:
+        sys.path.insert(1, scripts_path)
+        sys.path.insert(1, os.path.join(scripts_path, 'mozharness'))
+    from mozharness.mozilla.firefox.autoconfig import write_autoconfig_files
+    certificate = _read_certificate(certificate_path)
+    write_autoconfig_files(fx_install_dir=fx_install_dir,
+                           cfg_contents=MITMPROXY_SETTINGS % {
+                              'cert': certificate})
 
 
 def _read_certificate(certificate_path):
     ''' Return the certificate's hash from the certificate file.'''
     # NOTE: mitmproxy's certificates do not exist until one of its binaries
     #       has been executed once on the host
     with open(certificate_path, 'r') as fd:
         contents = fd.read()
     return ''.join(contents.splitlines()[1:-1])
+
+
+def is_mitmproxy_cert_installed():
+    """Verify mitmxproy CA cert was added to Firefox"""
+    # TODO: Bug 1366071
+    return True
+
+
+def install_mitmproxy_cert(mitmproxy_proc, browser_path, scripts_path):
+    """Install the CA certificate generated by mitmproxy, into Firefox"""
+    LOG.info("Installing mitmxproxy CA certficate into Firefox")
+    # browser_path is exe, we want install dir
+    browser_install = os.path.dirname(browser_path)
+    LOG.info('Calling configure_mitmproxy with browser folder: %s' % browser_install)
+    configure_mitmproxy(browser_install, scripts_path)
+    # cannot continue if failed to add CA cert to Firefox, need to check
+    if not is_mitmproxy_cert_installed():
+        LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox')
+        stop_mitmproxy_playback(mitmproxy_proc)
+        sys.exit()
+
+
+def start_mitmproxy_playback(mitmdump_path,
+                             mitmproxy_recording_path,
+                             mitmproxy_recordings_list,
+                             browser_path):
+    """Startup mitmproxy and replay the specified flow file"""
+    mitmproxy_recordings = []
+    # recording names can be provided in comma-separated list; build py list including path
+    for recording in mitmproxy_recordings_list:
+        mitmproxy_recordings.append(os.path.join(mitmproxy_recording_path, recording))
+
+    # cmd line to start mitmproxy playback using custom playback script is as follows:
+    # <path>/mitmdump -s "<path>mitmdump-alternate-server-replay/alternate-server-replay.py
+    #  <path>recording-1.mp <path>recording-2.mp..."
+    param = os.path.join(here, 'alternate-server-replay.py')
+
+    # this part is platform-specific
+    if mozinfo.os == 'win':
+        param2 = '""' + param.replace('\\', '\\\\\\') + ' ' + \
+                 ' '.join(mitmproxy_recordings).replace('\\', '\\\\\\') + '""'
+        env = os.environ.copy()
+        sys.path.insert(1, mitmdump_path)
+        # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
+        env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
+    else:
+        # TODO: support other platforms, Bug 1366355
+        LOG.error('Aborting: talos mitmproxy is currently only supported on Windows')
+        sys.exit()
+
+    command = [mitmdump_path, '-s', param2]
+
+    LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
+    LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command))
+    # to turn off mitmproxy log output, use these params for Popen:
+    # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+    mitmproxy_proc = subprocess.Popen(command, env=env)
+    time.sleep(5)
+    data = mitmproxy_proc.poll()
+    if data is None:
+        LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid)
+        return mitmproxy_proc
+    # cannot continue as we won't be able to playback the pages
+    LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data)
+    sys.exit()
+
+
+def stop_mitmproxy_playback(mitmproxy_proc):
+    """Stop the mitproxy server playback"""
+    LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid)
+    mitmproxy_proc.kill()
+    time.sleep(5)
+    exit_code = mitmproxy_proc.poll()
+    if exit_code:
+        LOG.info("Successfully killed the mitmproxy playback process")
+    else:
+        # I *think* we can still continue, as process will be automatically
+        # killed anyway when mozharness is done (?) if not, we won't be able
+        # to startup mitmxproy next time if it is already running
+        LOG.error("Failed to kill the mitmproxy playback process")
rename from testing/talos/mitmproxy_requirements.txt
rename to testing/talos/talos/mitmproxy/mitmproxy_requirements.txt
rename from testing/talos/python3.manifest
rename to testing/talos/talos/mitmproxy/python3.manifest
rename from testing/talos/python3_x64.manifest
rename to testing/talos/talos/mitmproxy/python3_x64.manifest
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -10,16 +10,17 @@ import sys
 import time
 import traceback
 import urllib
 import utils
 import mozhttpd
 
 from mozlog import get_proxy_logger
 
+from talos.mitmproxy import mitmproxy
 from talos.results import TalosResults
 from talos.ttest import TTest
 from talos.utils import TalosError, TalosRegression
 from talos.config import get_configs, ConfigurationError
 
 # directory of this file
 here = os.path.dirname(os.path.realpath(__file__))
 LOG = get_proxy_logger()
@@ -84,17 +85,16 @@ def setup_webserver(webserver):
 
 
 def run_tests(config, browser_config):
     """Runs the talos tests on the given configuration and generates a report.
     """
     # get the test data
     tests = config['tests']
     tests = useBaseTestDefaults(config.get('basetest', {}), tests)
-
     paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
     for test in tests:
 
         # Check for profile_path, tpmanifest and interpolate based on Talos
         # root https://bugzilla.mozilla.org/show_bug.cgi?id=727711
         # Build command line from config
         for path in paths:
             if test.get(path):
@@ -186,16 +186,47 @@ def run_tests(config, browser_config):
 
     # if e10s add as extra results option
     if config['e10s']:
         talos_results.add_extra_option('e10s')
 
     if config['gecko_profile']:
         talos_results.add_extra_option('geckoProfile')
 
+    # some tests use mitmproxy to playback pages
+    mitmproxy_recordings_list = config.get('mitmproxy', False)
+    if mitmproxy_recordings_list is not False:
+        # needed so can tell talos ttest to allow external connections
+        browser_config['mitmproxy'] = True
+
+        # start mitmproxy playback; this also generates the CA certificate
+        mitmdump_path = config.get('mitmdumpPath', False)
+        if mitmdump_path is False:
+            # cannot continue, need path for mitmdump playback tool
+            LOG.error('Aborting: mitmdumpPath was not provided on cmd line but is required')
+            sys.exit()
+
+        mitmproxy_recording_path = os.path.join(here, 'mitmproxy')
+        mitmproxy_proc = mitmproxy.start_mitmproxy_playback(mitmdump_path,
+                                                            mitmproxy_recording_path,
+                                                            mitmproxy_recordings_list.split(),
+                                                            browser_config['browser_path'])
+
+        # install the generated CA certificate into Firefox
+        # mitmproxy cert setup needs path to mozharness install; mozharness has set this
+        # value in the SCRIPTSPATH env var for us in mozharness/mozilla/testing/talos.py
+        scripts_path = os.environ.get('SCRIPTSPATH')
+        LOG.info('scripts_path: %s' % str(scripts_path))
+        mitmproxy.install_mitmproxy_cert(mitmproxy_proc,
+                                         browser_config['browser_path'],
+                                         str(scripts_path))
+
+    if config.get('first_non_blank_paint', False):
+        browser_config['firstNonBlankPaint'] = True
+
     testname = None
     # run the tests
     timer = utils.Timer()
     LOG.suite_start(tests=[test['name'] for test in tests])
     try:
         for test in tests:
             testname = test['name']
             LOG.test_start(testname)
@@ -221,16 +252,20 @@ def run_tests(config, browser_config):
         # indicate a failure to buildbot, turn the job red
         return 2
     finally:
         LOG.suite_end()
         httpd.stop()
 
     LOG.info("Completed test suite (%s)" % timer.elapsed())
 
+    # if mitmproxy was used for page playback, stop it
+    if mitmproxy_recordings_list is not False:
+        mitmproxy.stop_mitmproxy_playback(mitmproxy_proc)
+
     # output results
     if results_urls:
         talos_results.output(results_urls)
         if browser_config['develop'] or config['gecko_profile']:
             print("Thanks for running Talos locally. Results are in %s"
                   % (results_urls['output_urls']))
 
     # we will stop running tests on a failed test, or we will return 0 for
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -787,8 +787,23 @@ class bloom_basic_ref(PageloaderTest):
     tpcycles = 1
     tppagecycles = 25
     gecko_profile_interval = 1
     gecko_profile_entries = 2000000
     filters = filter.ignore_first.prepare(5) + filter.median.prepare()
     unit = 'ms'
     lower_is_better = True
     alert_threshold = 5.0
+
+
+@register_test()
+class Quantum_1(PageloaderTest):
+    """
+    Quantum Pageload Test 1
+    """
+    tpmanifest = '${talos}/tests/quantum_pageload/quantum_1.manifest'
+    tpcycles = 1
+    tppagecycles = 25
+    gecko_profile_interval = 1
+    gecko_profile_entries = 2000000
+    filters = filter.ignore_first.prepare(5) + filter.median.prepare()
+    unit = 'ms'
+    lower_is_better = True
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/quantum_pageload/quantum_1.manifest
@@ -0,0 +1,1 @@
+https://www.google.com
--- a/testing/talos/talos/ttest.py
+++ b/testing/talos/talos/ttest.py
@@ -99,16 +99,21 @@ class TTest(object):
         if test_config.get('responsiveness') and \
            platform.system() != "Darwin":
             # ignore osx for now as per bug 1245793
             setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
             setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
             setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
             global_counters['responsiveness'] = []
 
+        # if using mitmproxy we must allow access to 'external' sites
+        if browser_config.get('mitmproxy', False):
+            LOG.info("Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0")
+            setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0'
+
         # instantiate an object to hold test results
         test_results = results.TestResults(
             test_config,
             global_counters,
             browser_config.get('framework')
         )
 
         for i in range(test_config['cycles']):