Bug 1357382 - Add mitmproxy support to mozharness and talos, and add starting revision of new quantum-pageload test draft
authorRob Wood <rwood@mozilla.com>
Wed, 17 May 2017 12:14:59 -0400
changeset 581420 6000261aa1c39498e93aa210cd2e3b3c9442e997
parent 579463 6e3ca5b38f7173b214b10de49e58cb01890bf39d
child 629581 119e9f3fd97134f6b0407e0389f71f94f5327a85
push id59873
push userrwood@mozilla.com
push dateFri, 19 May 2017 21:07:34 +0000
bugs1357382
milestone55.0a1
Bug 1357382 - Add mitmproxy support to mozharness and talos, and add starting revision of new quantum-pageload test MozReview-Commit-ID: 4iYLAZEVAOt
testing/mozharness/configs/talos/linux_config.py
testing/mozharness/configs/talos/mac_config.py
testing/mozharness/configs/talos/windows_config.py
testing/mozharness/mozharness/mozilla/mitmproxy.py
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/mach_commands.py
testing/talos/mitmproxy-playback-set.manifest
testing/talos/talos.json
testing/talos/talos/alternate-server-replay.py
testing/talos/talos/cmdline.py
testing/talos/talos/run_tests.py
testing/talos/talos/test.py
testing/talos/talos/tests/quantum_pageload/quantum_1.manifest
testing/talos/talos/ttest.py
--- a/testing/mozharness/configs/talos/linux_config.py
+++ b/testing/mozharness/configs/talos/linux_config.py
@@ -28,16 +28,17 @@ config = {
     "title": os.uname()[1].lower().split('.')[0],
     "default_actions": [
         "clobber",
         "read-buildbot-config",
         "download-and-extract",
         "populate-webroot",
         "create-virtualenv",
         "install",
+        "setup-mitmproxy",
         "run-tests",
     ],
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
--- a/testing/mozharness/configs/talos/mac_config.py
+++ b/testing/mozharness/configs/talos/mac_config.py
@@ -31,16 +31,17 @@ config = {
     "title": os.uname()[1].lower().split('.')[0],
     "default_actions": [
         "clobber",
         "read-buildbot-config",
         "download-and-extract",
         "populate-webroot",
         "create-virtualenv",
         "install",
+        "setup-mitmproxy",
         "run-tests",
     ],
     "run_cmd_checks_enabled": True,
     "preflight_run_cmd_suites": [
         SCREEN_RESOLUTION_CHECK,
     ],
     "postflight_run_cmd_suites": [
         SCREEN_RESOLUTION_CHECK,
--- a/testing/mozharness/configs/talos/windows_config.py
+++ b/testing/mozharness/configs/talos/windows_config.py
@@ -30,16 +30,17 @@ config = {
     "title": socket.gethostname().split('.')[0],
     "default_actions": [
         "clobber",
         "read-buildbot-config",
         "download-and-extract",
         "populate-webroot",
         "create-virtualenv",
         "install",
+        "setup-mitmproxy",
         "run-tests",
     ],
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
     "download_minidump_stackwalk": True,
--- a/testing/mozharness/mozharness/mozilla/mitmproxy.py
+++ b/testing/mozharness/mozharness/mozilla/mitmproxy.py
@@ -1,17 +1,24 @@
 '''This helps loading mitmproxy's cert and change proxy settings for Firefox.'''
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
 from mozharness.mozilla.firefox.autoconfig import write_autoconfig_files
 
-DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
-                                 '.mitmproxy', 'mitmproxy-ca-cert.cer')
+# path for mitmproxy certificate, generated auto after mitmdump is started
+# on local machine it is 'HOME', however it is different on production machines
+try:
+    DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
+                                     '.mitmproxy', 'mitmproxy-ca-cert.cer')
+except:
+    DEFAULT_CERT_PATH = os.path.join(os.getenv('HOMEDRIVE'), os.getenv('HOMEPATH'),
+                                     '.mitmproxy', 'mitmproxy-ca-cert.cer')
+
 MITMPROXY_SETTINGS = '''// Start with a comment
 // Load up mitmproxy cert
 var Cc = Components.classes;
 var Ci = Components.interfaces;
 var certdb = Cc["@mozilla.org/security/x509certdb;1"].getService(Ci.nsIX509CertDB);
 var certdb2 = certdb;
 
 try {
@@ -27,20 +34,20 @@ pref("network.proxy.type", 1);
 pref("network.proxy.http", "127.0.0.1");
 pref("network.proxy.http_port", 8080);
 pref("network.proxy.ssl", "127.0.0.1");
 pref("network.proxy.ssl_port", 8080);
 '''
 
 
 def configure_mitmproxy(fx_install_dir, certificate_path=DEFAULT_CERT_PATH):
-        certificate = _read_certificate(certificate_path)
-        write_autoconfig_files(fx_install_dir=fx_install_dir,
-                               cfg_contents=MITMPROXY_SETTINGS % {
-                                  'cert': certificate})
+    certificate = _read_certificate(certificate_path)
+    write_autoconfig_files(fx_install_dir=fx_install_dir,
+                           cfg_contents=MITMPROXY_SETTINGS % {
+                              'cert': certificate})
 
 
 def _read_certificate(certificate_path):
     ''' Return the certificate's hash from the certificate file.'''
     # NOTE: mitmproxy's certificates do not exist until one of its binaries
     #       has been executed once on the host
     with open(certificate_path, 'r') as fd:
         contents = fd.read()
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -143,39 +143,43 @@ class Talos(TestingMixin, MercurialScrip
     def __init__(self, **kwargs):
         kwargs.setdefault('config_options', self.config_options)
         kwargs.setdefault('all_actions', ['clobber',
                                           'read-buildbot-config',
                                           'download-and-extract',
                                           'populate-webroot',
                                           'create-virtualenv',
                                           'install',
+                                          'setup-mitmproxy',
                                           'run-tests',
                                           ])
         kwargs.setdefault('default_actions', ['clobber',
                                               'download-and-extract',
                                               'populate-webroot',
                                               'create-virtualenv',
                                               'install',
+                                              'setup-mitmproxy',
                                               'run-tests',
                                               ])
         kwargs.setdefault('config', {})
         super(Talos, self).__init__(**kwargs)
 
         self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
 
         self.run_local = self.config.get('run_local')
         self.installer_url = self.config.get("installer_url")
         self.talos_json_url = self.config.get("talos_json_url")
         self.talos_json = self.config.get("talos_json")
         self.talos_json_config = self.config.get("talos_json_config")
         self.tests = None
         self.gecko_profile = self.config.get('gecko_profile')
         self.gecko_profile_interval = self.config.get('gecko_profile_interval')
         self.pagesets_name = None
+        self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings
+        self.mitmdump = None # path to mitdump tool itself, in py3 venv
 
     # We accept some configuration options from the try commit message in the format mozharness: <options>
     # Example try commit message:
     #   mozharness: --geckoProfile try: <stuff>
     def query_gecko_profile_options(self):
         gecko_results = []
         if self.buildbot_config:
             # this is inside automation
@@ -281,16 +285,20 @@ class Talos(TestingMixin, MercurialScrip
         if 'suite' in self.config:
             kw_options['suite'] = self.config['suite']
         if self.config.get('title'):
             kw_options['title'] = self.config['title']
         if self.config.get('branch'):
             kw_options['branchName'] = self.config['branch']
         if self.symbols_path:
             kw_options['symbolsPath'] = self.symbols_path
+        # if using mitmproxy, we've already created a py3 venv just
+        # for it; need to add the path to that env/mitdump tool
+        if self.mitmdump:
+            kw_options['mitmdumpPath'] = self.mitmdump
         kw_options.update(kw)
         # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
         tests = kw_options.get('activeTests')
         if tests and not isinstance(tests, basestring):
             tests = ':'.join(tests)  # Talos expects this format
             kw_options['activeTests'] = tests
         for key, value in kw_options.items():
             options.extend(['--%s' % key, value])
@@ -343,16 +351,79 @@ class Talos(TestingMixin, MercurialScrip
                 )
                 archive = os.path.join(src_talos_pageset, self.pagesets_name)
                 unzip = self.query_exe('unzip')
                 unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
                 self.run_command(unzip_cmd, halt_on_failure=True)
             else:
                 self.info("Not downloading pageset because the no-download option was specified")
 
+    def setup_mitmproxy(self):
+        """Some talos tests require the use of mitmproxy to playback the pages,
+        set it up here.
+        """
+        if not self.query_mitmproxy_recording_set():
+            self.info("Skipping: mitmproxy is not required")
+            return
+
+        # setup python 3.x virtualenv
+        self.setup_py3_virtualenv()
+
+        # install mitmproxy
+        self.install_mitmproxy()
+
+        # download the recording set; will be overridden by the --no-download
+        if '--no-download' not in self.config['talos_extra_options']:
+            self.download_mitmproxy_recording_set()
+        else:
+            self.info("Not downloading mitmproxy recording set because no-download was specified")
+
+    def setup_py3_virtualenv(self):
+        """Mitmproxy needs Python 3.x; set up a separate py 3.x env here"""
+        self.info("Setting up python 3.x virtualenv, required for mitmproxy")
+        # first download the py3 package
+        self.py3_path = self.fetch_python3()
+        # now create the py3 venv
+        self.py3_venv_configuration(python_path=self.py3_path, venv_path='py3venv')
+        self.py3_create_venv()
+        requirements = [os.path.join(self.talos_path, 'mitmproxy_requirements.txt')]
+        self.py3_install_requirement_files(requirements)
+
+    def install_mitmproxy(self):
+        """Install the mitmproxy tool into the Python 3.x env"""
+        self.info("Installing mitmproxy")
+        self.py3_install_modules(modules=['mitmproxy'])
+        self.mitmdump = os.path.join(self.py3_path_to_executables(), 'mitmdump')
+        self.run_command([self.mitmdump, '--version'], env=self.query_env())
+
+    def query_mitmproxy_recording_set(self):
+        """Mitmproxy requires external playback archives to be downloaded and extracted"""
+        if self.mitmproxy_recording_set:
+            return self.mitmproxy_recording_set
+        if self.query_talos_json_config() and self.suite is not None:
+            self.mitmproxy_recording_set = self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False)
+            if self.mitmproxy_recording_set is False:
+                self.fatal("Aborting: mitmproxy_recording_set is required")
+            return self.mitmproxy_recording_set
+
+    def download_mitmproxy_recording_set(self):
+        """Download the set of mitmproxy recording files that will be played back"""
+        self.info("Downloading the mitmproxy recording set using tooltool")
+        dest = os.path.join(self.talos_path, 'talos')
+        manifest_file = os.path.join(self.talos_path, 'mitmproxy-playback-set.manifest')
+        self.tooltool_fetch(
+            manifest_file,
+            output_dir=dest,
+            cache=self.config.get('tooltool_cache')
+        )
+        archive = os.path.join(dest, self.mitmproxy_recording_set)
+        unzip = self.query_exe('unzip')
+        unzip_cmd = [unzip, '-q', '-o', archive, '-d', dest]
+        self.run_command(unzip_cmd, halt_on_failure=True)
+
     # Action methods. {{{1
     # clobber defined in BaseScript
     # read_buildbot_config defined in BuildbotMixin
 
     def download_and_extract(self, extract_dirs=None, suite_categories=None):
         return super(Talos, self).download_and_extract(
             suite_categories=['common', 'talos']
         )
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -59,16 +59,17 @@ class TalosRunner(MozbuildObject):
             'exes': {
                 'python': self.python_interp,
                 'virtualenv': [self.python_interp, self.virtualenv_script]
             },
             'title': socket.gethostname(),
             'default_actions': [
                 'populate-webroot',
                 'create-virtualenv',
+                'setup-mitmproxy',
                 'run-tests',
             ],
             'download_tooltool': True,
             'talos_extra_options': ['--develop'] + self.talos_args,
         }
 
     def make_args(self):
         self.args = {
new file mode 100644
--- /dev/null
+++ b/testing/talos/mitmproxy-playback-set.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "mitmproxy-recording-set.zip",
+        "size": 679582,
+        "digest": "035e0729905ea1bb61efa4e720cdbc2a436dfebf364eb0d71a7a9ad04658635ede69aa5ef068e99bd515b0b68e2217a2ffe5961b3e8c7f8ead2c49ddcb9fe879",
+        "algorithm": "sha512",
+        "unpack": false
+    }
+]
\ No newline at end of file
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -2,22 +2,25 @@
     "talos.zip": {
         "url": "http://talos-bundles.pvt.build.mozilla.org/zips/talos.a6052c33d420.zip",
         "path": ""
     },
     "extra_options": {
         "android": [ "--apkPath=%(apk_path)s" ]
     },
     "suites": {
-        "chromez": {
-            "tests": ["tresize", "tcanvasmark"],
-            "talos_options": ["--disable-e10s"]
-        },
         "chromez-e10s": {
-            "tests": ["tresize", "tcanvasmark"]
+            "tests": ["Quantum_1"],
+            "mitmproxy_recording_set": "mitmproxy-recording-set.zip",
+            "mitmdump_alt_playback_repo": "https://github.com/bsmedberg/mitmdump-alternate-server-replay",
+            "talos_options": [
+                "--mitmproxy",
+                "mitmproxy-recording-1.mp",
+                "--firstNonBlankPaint"
+            ]
         },
         "dromaeojs": {
             "tests": ["dromaeo_css", "kraken"],
             "talos_options": ["--disable-e10s"]
         },
         "dromaeojs-e10s": {
             "tests": ["dromaeo_css", "kraken"]
         },
@@ -111,16 +114,26 @@
         },
         "xperf-e10s": {
             "tests": ["tp5n"],
             "pagesets_name": "tp5n.zip",
             "talos_options": [
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
             ]
+        },
+        "quantum-pageload-e10s": {
+            "tests": ["Quantum_1"],
+            "mitmproxy_recording_set": "mitmproxy-recording-set.zip",
+            "mitmdump_alt_playback_repo": "https://github.com/bsmedberg/mitmdump-alternate-server-replay",
+            "talos_options": [
+                "--mitmproxy",
+                "mitmproxy-recording-1.mp",
+                "--firstNonBlankPaint"
+            ]
         }
     },
     "mobile-suites": {
         "remote-tsvgx": {
             "tests": ["tsvgm"],
             "talos_options": [
                 "--noChrome",
                 "--tppagecycles", "7"
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/alternate-server-replay.py
@@ -0,0 +1,188 @@
+# This file was copied from mitmproxy/addons/serverplayback.py release tag 2.0.2 and modified by
+# Benjamin Smedberg
+
+# Altered features:
+# * --kill returns 404 rather than dropping the whole HTTP/2 connection on the floor
+# * best-match response handling is used to improve success rates
+
+import hashlib
+import urllib
+import sys
+from collections import defaultdict
+from typing import Any  # noqa
+from typing import List  # noqa
+
+from mitmproxy import ctx
+from mitmproxy import exceptions
+from mitmproxy import io
+from mitmproxy import http
+
+
+class ServerPlayback:
+    def __init__(self, replayfiles):
+        self.options = None
+        self.replayfiles = replayfiles
+        self.flowmap = {}
+
+    def load(self, flows):
+        for i in flows:
+            if i.response:
+                l = self.flowmap.setdefault(self._hash(i.request), [])
+                l.append(i)
+
+    def clear(self):
+        self.flowmap = {}
+
+    # def count(self):
+    #    return sum([len(i) for i in self.flowmap.values()])
+
+    def _parse(self, r):
+        """
+            Return (path, queries, formdata, content) for a request.
+        """
+        _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
+        queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
+        queries = defaultdict(list)
+        for k, v in queriesArray:
+            queries[k].append(v)
+
+        content = None
+        formdata = None
+        if r.raw_content != b'':
+            if r.multipart_form:
+                formdata = r.multipart_form
+            elif r.urlencoded_form:
+                formdata = r.urlencoded_form
+            else:
+                content = r.content
+        return (path, queries, formdata, content)
+
+    def _hash(self, r):
+        """
+            Calculates a loose hash of the flow request.
+        """
+        path, queries, _, _ = self._parse(r)
+
+        key = [str(r.port), str(r.scheme), str(r.method), str(path)]  # type: List[Any]
+        if not self.options.server_replay_ignore_host:
+            key.append(r.host)
+
+        if len(queries):
+            key.append("?")
+
+        return hashlib.sha256(
+            repr(key).encode("utf8", "surrogateescape")
+        ).digest()
+
+    def _match(self, request_a, request_b):
+        """
+            Calculate a match score between two requests.
+            Match algorithm:
+              * identical query keys: 3 points
+              * matching query param present: 1 point
+              * matching query param value: 3 points
+              * identical form keys: 3 points
+              * matching form param present: 1 point
+              * matching form param value: 3 points
+              * matching body (no multipart or encoded form): 4 points
+        """
+        match = 0
+
+        path_a, queries_a, form_a, content_a = self._parse(request_a)
+        path_b, queries_b, form_b, content_b = self._parse(request_b)
+
+        keys_a = set(queries_a.keys())
+        keys_b = set(queries_b.keys())
+        if keys_a == keys_b:
+            match += 3
+
+        for key in keys_a:
+            values_a = set(queries_a[key])
+            values_b = set(queries_b[key])
+            if len(values_a) == len(values_b):
+                match += 1
+            if values_a == values_b:
+                match += 3
+
+        if form_a and form_b:
+            keys_a = set(form_a.keys())
+            keys_b = set(form_b.keys())
+            if keys_a == keys_b:
+                match += 3
+
+            for key in keys_a:
+                values_a = set(form_a.get_all(key))
+                values_b = set(form_b.get_all(key))
+                if len(values_a) == len(values_b):
+                    match += 1
+                if values_a == values_b:
+                    match += 3
+
+        elif content_a and (content_a == content_b):
+            match += 4
+
+        return match
+
+    def next_flow(self, request):
+        """
+            Returns the next flow object, or None if no matching flow was
+            found.
+        """
+        hsh = self._hash(request)
+        flows = self.flowmap.get(hsh, None)
+        if flows is None:
+            return None
+
+        # if it's an exact match, great!
+        if len(flows) == 1:
+            candidate = flows[0]
+            if (candidate.request.url == request.url and
+               candidate.request.raw_content == request.raw_content):
+                ctx.log.info("For request {} found exact replay match".format(request.url))
+                return candidate
+
+        # find the best match between the request and the available flow candidates
+        match = -1
+        flow = None
+        ctx.log.debug("Candiate flows for request: {}".format(request.url))
+        for candidate_flow in flows:
+            candidate_match = self._match(candidate_flow.request, request)
+            ctx.log.debug("  score={} url={}".format(candidate_match, candidate_flow.request.url))
+            if candidate_match > match:
+                match = candidate_match
+                flow = candidate_flow
+        ctx.log.info("For request {} best match {} with score=={}".format(request.url,
+                     flow.request.url, match))
+        return candidate_flow
+
+    def configure(self, options, updated):
+        self.options = options
+        self.clear()
+        try:
+            flows = io.read_flows_from_paths(self.replayfiles)
+        except exceptions.FlowReadException as e:
+            raise exceptions.OptionsError(str(e))
+        self.load(flows)
+
+    def request(self, f):
+        if self.flowmap:
+            rflow = self.next_flow(f.request)
+            if rflow:
+                response = rflow.response.copy()
+                response.is_replay = True
+                if self.options.refresh_server_playback:
+                    response.refresh()
+                f.response = response
+            elif self.options.replay_kill_extra:
+                ctx.log.warn(
+                    "server_playback: killed non-replay request {}".format(
+                        f.request.url
+                    )
+                )
+                f.response = http.HTTPResponse.make(404, b'', {'content-type': 'text/plain'})
+
+
+def start():
+    files = sys.argv[1:]
+    print("Replaying from files: {}".format(files))
+    return ServerPlayback(files)
--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -111,16 +111,23 @@ def create_parser(mach_interface=False):
             default=os.path.abspath('browser_failures.txt'),
             help="Filename to store the errors found during the test."
                  " Currently used for xperf only.")
     add_arg('--noShutdown', dest='shutdown', action='store_true',
             help="Record time browser takes to shutdown after testing")
     add_arg('--setpref', action='append', default=[], dest="extraPrefs",
             metavar="PREF=VALUE",
             help="defines an extra user preference")
+    add_arg('--mitmproxy',
+            help='Test uses mitmproxy to serve the pages, specify the '
+                 'path and name of the mitmdump file to playback')
+    add_arg('--mitmdumpPath',
+            help="Path to mitmproxy's mitmdump playback tool")
+    add_arg("--firstNonBlankPaint", action='store_true', dest="first_non_blank_paint",
+            help="Wait for firstNonBlankPaint event before recording the time")
     add_arg('--webServer', dest='webserver',
             help="DEPRECATED")
     if not mach_interface:
         add_arg('--develop', action='store_true', default=False,
                 help="useful for running tests on a developer machine."
                      " Doesn't upload to the graph servers.")
     add_arg("--cycles", type=int,
             help="number of browser cycles to run")
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -2,16 +2,17 @@
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import mozversion
 import os
 import sys
+import subprocess
 import time
 import traceback
 import urllib
 import utils
 import mozhttpd
 
 from mozlog import get_proxy_logger
 
@@ -78,16 +79,103 @@ def buildCommandLine(test):
 def setup_webserver(webserver):
     """use mozhttpd to setup a webserver"""
     LOG.info("starting webserver on %r" % webserver)
 
     host, port = webserver.split(':')
     return mozhttpd.MozHttpd(host=host, port=int(port), docroot=here)
 
 
+def start_mitmproxy_playback(mitmdump_path,
+                             mitmproxy_recording_path,
+                             mitmproxy_recordings_list):
+    """Startup mitmproxy and replay the specified flow file"""
+    mitmproxy_recordings = []
+    # recording names can be provided in comma-separated list; build py list including path
+    for recording in mitmproxy_recordings_list:
+        mitmproxy_recordings.append(os.path.join(mitmproxy_recording_path, recording))
+
+    # cmd line to start mitmproxy playback using custom playback script is as follows:
+    # <path>/mitmdump -s "<path>mitmdump-alternate-server-replay/alternate-server-replay.py
+    #  <path>recording-1.mp <path>recording-2.mp..."
+    param = os.path.join(here, 'alternate-server-replay.py')
+
+    # this part is platform-specific
+    from mozharness.base.script import platform_name
+    platform = platform_name()
+    if platform == 'win32' or platform == 'win64':
+        param2 = '""' + param.replace('\\', '\\\\\\') + ' ' + \
+                 ' '.join(mitmproxy_recordings).replace('\\', '\\\\\\') + '""'
+        env = os.environ.copy()
+        env["PATH"] = "C:\\slave\\test\\build\\application\\firefox;" + env["PATH"]
+    else:
+        # TODO: support other platforms, Bug 1366355
+        LOG.error('Aborting: talos mitmproxy is currently only supported on Windows')
+        sys.exit()
+
+    command = [mitmdump_path, '-s', param2, '-k']
+
+    LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
+    LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command))
+    # to turn off mitmproxy log output, use these params for Popen:
+    # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+    mitmproxy_proc = subprocess.Popen(command, env=env)
+    time.sleep(5)
+    data = mitmproxy_proc.poll()
+    if data is None:
+        LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid)
+        return mitmproxy_proc
+    # cannot continue as we won't be able to playback the pages
+    LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data)
+    sys.exit()
+
+
+def stop_mitmproxy_playback(mitmproxy_proc):
+    """Stop the mitproxy server playback"""
+    LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid)
+    mitmproxy_proc.kill()
+    time.sleep(5)
+    exit_code = mitmproxy_proc.poll()
+    if exit_code:
+        LOG.info("Successfully killed the mitmproxy playback process")
+    else:
+        # I *think* we can still continue, as process will be automatically
+        # killed anyway when mozharness is done (?) if not, we won't be able
+        # to startup mitmxproy next time if it is already running
+        LOG.error("Failed to kill the mitmproxy playback process")
+
+
+def is_mitmproxy_cert_installed():
+    """Verify mitmxproy CA cert was added to Firefox"""
+    # TODO: Bug 1366071
+    return True
+
+
+def install_mitmproxy_cert(mitmproxy_proc, browser_path, config):
+    """Install the CA certificate generated by mitmproxy, into Firefox"""
+    LOG.info("Installing mitmxproxy CA certficate into Firefox")
+    # need mozharness mitmproxy module
+    LOG.info("sys.path = %s" % sys.path[1])
+    LOG.info("inserting sys.path = %s" % os.path.join(os.path.dirname(sys.path[1]), 'mozharness'))
+    os.path.join(os.path.dirname(sys.path[1]), 'mozharness')
+    sys.path.insert(1, os.path.join(os.path.dirname(sys.path[1]), 'mozharness'))
+    sys.path.insert(1, "c:\\slave\\test\\scripts\\")
+
+    # browser_path is exe, we want install dir
+    browser_install = os.path.dirname(browser_path)
+    LOG.info('Calling configure_mitmproxy with browser folder: %s' % browser_install)
+    from mozharness.mozilla.mitmproxy import configure_mitmproxy
+    configure_mitmproxy(browser_install)
+    # cannot continue if failed to add CA cert to Firefox, need to check
+    if not is_mitmproxy_cert_installed():
+        LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox')
+        stop_mitmproxy_playback(mitmproxy_proc)
+        sys.exit()
+
+
 def run_tests(config, browser_config):
     """Runs the talos tests on the given configuration and generates a report.
     """
     # get the test data
     tests = config['tests']
     tests = useBaseTestDefaults(config.get('basetest', {}), tests)
 
     paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
@@ -186,16 +274,40 @@ def run_tests(config, browser_config):
 
     # if e10s add as extra results option
     if config['e10s']:
         talos_results.add_extra_option('e10s')
 
     if config['gecko_profile']:
         talos_results.add_extra_option('geckoProfile')
 
+    # some tests use mitmproxy to playback pages
+    mitmproxy_recordings_list = config.get('mitmproxy', False)
+    if mitmproxy_recordings_list is not False:
+        # needed so can tell talos ttest to allow external connections
+        browser_config['mitmproxy'] = True
+
+        # start mitmproxy playback; this also generates the CA certificate
+        mitmdump_path = config.get('mitmdumpPath', False)
+        if mitmdump_path is False:
+            # cannot continue, need path for mitmdump playback tool
+            LOG.error('Aborting: mitmdumpPath was not provided on cmd line but is required')
+            sys.exit()
+
+        mitmproxy_recording_path = here
+        mitmproxy_proc = start_mitmproxy_playback(mitmdump_path,
+                                                  mitmproxy_recording_path,
+                                                  mitmproxy_recordings_list.split())
+
+        # install the generated CA certificate into Firefox
+        install_mitmproxy_cert(mitmproxy_proc, browser_config['browser_path'], config)
+
+    if config.get('first_non_blank_paint', False):
+        browser_config['firstNonBlankPaint'] = True
+
     testname = None
     # run the tests
     timer = utils.Timer()
     LOG.suite_start(tests=[test['name'] for test in tests])
     try:
         for test in tests:
             testname = test['name']
             LOG.test_start(testname)
@@ -221,16 +333,20 @@ def run_tests(config, browser_config):
         # indicate a failure to buildbot, turn the job red
         return 2
     finally:
         LOG.suite_end()
         httpd.stop()
 
     LOG.info("Completed test suite (%s)" % timer.elapsed())
 
+    # if mitmproxy was used for page playback, stop it
+    if mitmproxy_recordings_list is not False:
+        stop_mitmproxy_playback(mitmproxy_proc)
+
     # output results
     if results_urls:
         talos_results.output(results_urls)
         if browser_config['develop'] or config['gecko_profile']:
             print("Thanks for running Talos locally. Results are in %s"
                   % (results_urls['output_urls']))
 
     # we will stop running tests on a failed test, or we will return 0 for
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -774,8 +774,23 @@ class bloom_basic_ref(PageloaderTest):
     tpcycles = 1
     tppagecycles = 25
     gecko_profile_interval = 1
     gecko_profile_entries = 2000000
     filters = filter.ignore_first.prepare(5) + filter.median.prepare()
     unit = 'ms'
     lower_is_better = True
     alert_threshold = 5.0
+
+
+@register_test()
+class Quantum_1(PageloaderTest):
+    """
+    Quantum Pageload Test 1
+    """
+    tpmanifest = '${talos}/tests/quantum_pageload/quantum_1.manifest'
+    tpcycles = 1
+    tppagecycles = 25
+    gecko_profile_interval = 1
+    gecko_profile_entries = 2000000
+    filters = filter.ignore_first.prepare(5) + filter.median.prepare()
+    unit = 'ms'
+    lower_is_better = True
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/quantum_pageload/quantum_1.manifest
@@ -0,0 +1,1 @@
+https://www.google.com
--- a/testing/talos/talos/ttest.py
+++ b/testing/talos/talos/ttest.py
@@ -99,16 +99,21 @@ class TTest(object):
         if test_config.get('responsiveness') and \
            platform.system() != "Darwin":
             # ignore osx for now as per bug 1245793
             setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
             setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
             setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
             global_counters['responsiveness'] = []
 
+        # if using mitmproxy we must allow access to 'external' sites
+        if browser_config.get('mitmproxy', False):
+            LOG.info("Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0")
+            setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0'
+
         # instantiate an object to hold test results
         test_results = results.TestResults(
             test_config,
             global_counters,
             browser_config.get('framework')
         )
 
         for i in range(test_config['cycles']):