Bug 1528992 - extract raptor mitmproxy integration - r=rwood,Bebe,gbrown
authorTarek Ziadé <tarek@mozilla.com>
Wed, 06 Mar 2019 17:13:21 +0000
changeset 520570 76b84ca9cfaa348dba43d74575e7a3b133d4bed3
parent 520569 dc078ca4d52733fa0905d28d3736dc3a3850d2a1
child 520571 21c52ca0266ebfda36480f1e6acd961f7d43b0dd
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood, Bebe, gbrown
bugs1528992
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1528992 - extract raptor mitmproxy integration - r=rwood,Bebe,gbrown This patch moves testing/raptor/raptor/playback into its own testing package in testing/mozbase/mozproxy so we can use the proxy in other places than Raptor. Differential Revision: https://phabricator.services.mozilla.com/D21200
testing/config/mozbase_requirements.txt
testing/config/mozbase_source_requirements.txt
testing/mozbase/docs/mozproxy.rst
testing/mozbase/moz.build
testing/mozbase/mozproxy/MANIFEST.in
testing/mozbase/mozproxy/mozproxy/__init__.py
testing/mozbase/mozproxy/mozproxy/backends/__init__.py
testing/mozbase/mozproxy/mozproxy/backends/base.py
testing/mozbase/mozproxy/mozproxy/backends/mitm.py
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-linux64.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-osx.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-win.manifest
testing/mozbase/mozproxy/mozproxy/backends/mitmproxy_requirements.txt
testing/mozbase/mozproxy/mozproxy/server.py
testing/mozbase/mozproxy/mozproxy/utils.py
testing/mozbase/mozproxy/setup.py
testing/mozbase/mozproxy/tests/firefox
testing/mozbase/mozproxy/tests/manifest.ini
testing/mozbase/mozproxy/tests/paypal.mp
testing/mozbase/mozproxy/tests/test.py
testing/mozbase/packages.txt
testing/mozharness/mozharness/mozilla/mozbase.py
testing/raptor/raptor/manifest.py
testing/raptor/raptor/playback/__init__.py
testing/raptor/raptor/playback/base.py
testing/raptor/raptor/playback/mitmproxy-rel-bin-linux64.manifest
testing/raptor/raptor/playback/mitmproxy-rel-bin-osx.manifest
testing/raptor/raptor/playback/mitmproxy-rel-bin-win.manifest
testing/raptor/raptor/playback/mitmproxy.py
testing/raptor/raptor/playback/mitmproxy_requirements.txt
testing/raptor/raptor/raptor.py
testing/raptor/raptor/utils.py
testing/raptor/requirements.txt
testing/raptor/test/test_playback.py
--- a/testing/config/mozbase_requirements.txt
+++ b/testing/config/mozbase_requirements.txt
@@ -8,12 +8,13 @@
 ../mozbase/mozhttpd
 ../mozbase/mozinfo
 ../mozbase/mozinstall
 ../mozbase/mozleak
 ../mozbase/mozlog
 ../mozbase/moznetwork
 ../mozbase/mozprocess
 ../mozbase/mozprofile
+../mozbase/mozproxy
 ../mozbase/mozrunner
 ../mozbase/mozscreenshot
 ../mozbase/moztest
 ../mozbase/mozversion
--- a/testing/config/mozbase_source_requirements.txt
+++ b/testing/config/mozbase_source_requirements.txt
@@ -8,12 +8,13 @@
 ../mozbase/mozhttpd
 ../mozbase/mozinfo
 ../mozbase/mozinstall
 ../mozbase/mozleak
 ../mozbase/mozlog
 ../mozbase/moznetwork
 ../mozbase/mozprocess
 ../mozbase/mozprofile
+../mozbase/mozproxy
 ../mozbase/mozrunner
 ../mozbase/mozscreenshot
 ../mozbase/moztest
 ../mozbase/mozversion
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/docs/mozproxy.rst
@@ -0,0 +1,47 @@
+:mod:`mozproxy` --- Provides an HTTP proxy
+==========================================
+
+Mozproxy let you launch an HTTP proxy when we need to run tests against
+third-part websites in a reliable and reproducible way.
+
+Mozproxy provides an interface to a proxy software, and the currently
+supported backend is **mitmproxy** for Desktop and Android.
+
+Mozproxy is used by Raptor to run performance test without having to interact
+with the real web site.
+
+Mozproxy provide a function that returns a playback class. The usage pattern is
+::
+
+   from mozproxy import get_playback
+
+   config = {'playback_tool': 'mitmproxy'}
+   pb = get_playback(config)
+   pb.start()
+   try:
+     # do your test
+   finally:
+     pb.stop()
+
+**config** is a dict with the following options:
+
+- **playback_tool**: name of the backend. can be "mitmproxy", "mitmproxy-android"
+- **playback_recordings**: list of recording files
+- **playback_binary_manifest**: tooltool manifests for the proxy backend binary
+- **playback_pageset_manifest**: tooltool manifest for the pagesets archive
+- **binary**: path of the browser binary
+- **obj_path**: build dir
+- **platform**: platform name (provided by mozinfo.os)
+- **run_local**: if True, the test is running locally.
+- **custom_script**: name of the mitm custom script (-s option)
+- **app**: tested app. Can be "firefox",  "geckoview", "refbrow", "fenix" or  "firefox"
+- **host**: hostname for the policies.json file
+- **local_profile_dir**: profile dir
+
+
+Supported environment variables:
+
+- **MOZ_UPLOAD_DIR**: upload directory path
+- **GECKO_HEAD_REPOSITORY**: used to find the certutils binary path from the CI
+- **GECKO_HEAD_REV**: used to find the certutils binary path frmo the CI
+- **HOSTUTILS_MANIFEST_PATH**: used to find the certutils binary path from the CI
--- a/testing/mozbase/moz.build
+++ b/testing/mozbase/moz.build
@@ -13,16 +13,17 @@ PYTHON_UNITTEST_MANIFESTS += [
     'mozhttpd/tests/manifest.ini',
     'mozinfo/tests/manifest.ini',
     'mozinstall/tests/manifest.ini',
     'mozleak/tests/manifest.ini',
     'mozlog/tests/manifest.ini',
     'moznetwork/tests/manifest.ini',
     'mozprocess/tests/manifest.ini',
     'mozprofile/tests/manifest.ini',
+    'mozproxy/tests/manifest.ini',
     'mozrunner/tests/manifest.ini',
     'moztest/tests/manifest.ini',
     'mozversion/tests/manifest.ini',
 ]
 
 python_modules = [
     'manifestparser',
     'mozcrash',
@@ -32,16 +33,17 @@ python_modules = [
     'mozhttpd',
     'mozinfo',
     'mozinstall',
     'mozleak',
     'mozlog',
     'moznetwork',
     'mozprocess',
     'mozprofile',
+    'mozproxy',
     'mozrunner',
     'mozscreenshot',
     'mozsystemmonitor',
     'moztest',
     'mozversion',
 ]
 
 TEST_HARNESS_FILES.mozbase += [m + '/**' for m in python_modules]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/MANIFEST.in
@@ -0,0 +1,1 @@
+recursive-include mozproxy *
rename from testing/raptor/raptor/playback/__init__.py
rename to testing/mozbase/mozproxy/mozproxy/__init__.py
--- a/testing/raptor/raptor/playback/__init__.py
+++ b/testing/mozbase/mozproxy/mozproxy/__init__.py
@@ -1,27 +1,49 @@
-from __future__ import absolute_import
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from mozlog import get_proxy_logger
-from .mitmproxy import MitmproxyDesktop, MitmproxyAndroid
+from __future__ import absolute_import
+import sys
+import os
+
+
+def path_join(*args):
+    path = os.path.join(*args)
+    return os.path.abspath(path)
 
-LOG = get_proxy_logger(component='mitmproxy')
+
+mozproxy_src_dir = os.path.dirname(os.path.realpath(__file__))
+mozproxy_dir = path_join(mozproxy_src_dir, "..")
+mozbase_dir = path_join(mozproxy_dir, "..")
 
-playback_cls = {
-    'mitmproxy': MitmproxyDesktop,
-    'mitmproxy-android': MitmproxyAndroid,
-}
+# needed so unit tests can find their imports
+if os.environ.get("SCRIPTSPATH", None) is not None:
+    # in production it is env SCRIPTS_PATH
+    mozharness_dir = os.environ["SCRIPTSPATH"]
+else:
+    # locally it's in source tree
+    mozharness_dir = path_join(mozbase_dir, "..", "mozharness")
 
 
 def get_playback(config, android_device=None):
-    tool_name = config.get('playback_tool', None)
+    """ Returns an instance of the right Playback class
+    """
+    sys.path.insert(0, mozharness_dir)
+    sys.path.insert(0, mozproxy_dir)
+    sys.path.insert(0, mozproxy_src_dir)
+
+    from .server import get_backend
+    from .utils import LOG
+
+    tool_name = config.get("playback_tool", None)
     if tool_name is None:
         LOG.critical("playback_tool name not found in config")
-        return
-    if playback_cls.get(tool_name, None) is None:
+        return None
+    try:
+        if android_device is None:
+            return get_backend(tool_name, config)
+        else:
+            return get_backend(tool_name, config, android_device)
+    except KeyError:
         LOG.critical("specified playback tool is unsupported: %s" % tool_name)
         return None
-
-    cls = playback_cls.get(tool_name)
-    if android_device is None:
-        return cls(config)
-    else:
-        return cls(config, android_device)
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/mozproxy/backends/__init__.py
@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
rename from testing/raptor/raptor/playback/base.py
rename to testing/mozbase/mozproxy/mozproxy/backends/base.py
--- a/testing/raptor/raptor/playback/base.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/base.py
@@ -1,18 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# abstract class for all playback tools
 from __future__ import absolute_import
-
 from abc import ABCMeta, abstractmethod
 
 
+# abstract class for all playback tools
 class Playback(object):
     __metaclass__ = ABCMeta
 
     def __init__(self, config, android_device=None):
         self.config = config
         self.android_device = android_device
 
     @abstractmethod
rename from testing/raptor/raptor/playback/mitmproxy.py
rename to testing/mozbase/mozproxy/mozproxy/backends/mitm.py
--- a/testing/raptor/raptor/playback/mitmproxy.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm.py
@@ -5,339 +5,356 @@
 from __future__ import absolute_import
 
 import os
 import subprocess
 import sys
 import time
 
 import mozinfo
-
-from mozlog import get_proxy_logger
 from mozprocess import ProcessHandler
 
-from .base import Playback
-
-here = os.path.dirname(os.path.realpath(__file__))
-LOG = get_proxy_logger(component='raptor-mitmproxy')
+from mozproxy.backends.base import Playback
+from mozproxy.utils import (
+    transform_platform,
+    tooltool_download,
+    download_file_from_url,
+    LOG,
+)
 
-# needed so unit tests can find their imports
-if os.environ.get('SCRIPTSPATH', None) is not None:
-    # in production it is env SCRIPTS_PATH
-    mozharness_dir = os.environ['SCRIPTSPATH']
-else:
-    # locally it's in source tree
-    mozharness_dir = os.path.join(here, '../../../mozharness')
-sys.path.insert(0, mozharness_dir)
 
-raptor_dir = os.path.join(here, '..')
-sys.path.insert(0, raptor_dir)
-
-from utils import transform_platform, tooltool_download, download_file_from_url
-
+here = os.path.dirname(__file__)
 # path for mitmproxy certificate, generated auto after mitmdump is started
 # on local machine it is 'HOME', however it is different on production machines
 try:
-    DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
-                                     '.mitmproxy', 'mitmproxy-ca-cert.cer')
+    DEFAULT_CERT_PATH = os.path.join(
+        os.getenv("HOME"), ".mitmproxy", "mitmproxy-ca-cert.cer"
+    )
 except Exception:
-    DEFAULT_CERT_PATH = os.path.join(os.getenv('HOMEDRIVE'), os.getenv('HOMEPATH'),
-                                     '.mitmproxy', 'mitmproxy-ca-cert.cer')
+    DEFAULT_CERT_PATH = os.path.join(
+        os.getenv("HOMEDRIVE"),
+        os.getenv("HOMEPATH"),
+        ".mitmproxy",
+        "mitmproxy-ca-cert.cer",
+    )
 
 # On Windows, deal with mozilla-build having forward slashes in $HOME:
-if os.name == 'nt' and '/' in DEFAULT_CERT_PATH:
-    DEFAULT_CERT_PATH = DEFAULT_CERT_PATH.replace('/', '\\')
+if os.name == "nt" and "/" in DEFAULT_CERT_PATH:
+    DEFAULT_CERT_PATH = DEFAULT_CERT_PATH.replace("/", "\\")
 
 # sleep in seconds after issuing a `mitmdump` command
 MITMDUMP_SLEEP = 10
 
 # to install mitmproxy certificate into Firefox and turn on/off proxy
-POLICIES_CONTENT_ON = '''{
+POLICIES_CONTENT_ON = """{
   "policies": {
     "Certificates": {
       "Install": ["%(cert)s"]
     },
     "Proxy": {
       "Mode": "manual",
       "HTTPProxy": "%(host)s:8080",
       "SSLProxy": "%(host)s:8080",
       "Passthrough": "%(host)s",
       "Locked": true
     }
   }
-}'''
+}"""
 
-POLICIES_CONTENT_OFF = '''{
+POLICIES_CONTENT_OFF = """{
   "policies": {
     "Proxy": {
       "Mode": "none",
       "Locked": false
     }
   }
-}'''
+}"""
 
 
 class Mitmproxy(Playback):
-
     def __init__(self, config):
         self.config = config
         self.mitmproxy_proc = None
         self.mitmdump_path = None
-        self.recordings = config.get('playback_recordings', None)
-        self.browser_path = config.get('binary', None)
+        self.recordings = config.get("playback_recordings")
+        self.browser_path = config.get("binary")
 
-        # raptor_dir is where we will download all mitmproxy required files
+        # mozproxy_dir is where we will download all mitmproxy required files
         # when running locally it comes from obj_path via mozharness/mach
-        if self.config.get("obj_path", None) is not None:
-            self.raptor_dir = self.config.get("obj_path")
+        if self.config.get("obj_path") is not None:
+            self.mozproxy_dir = self.config.get("obj_path")
         else:
             # in production it is ../tasks/task_N/build/, in production that dir
             # is not available as an envvar, however MOZ_UPLOAD_DIR is set as
             # ../tasks/task_N/build/blobber_upload_dir so take that and go up 1 level
-            self.raptor_dir = os.path.dirname(os.path.dirname(os.environ['MOZ_UPLOAD_DIR']))
+            self.mozproxy_dir = os.path.dirname(
+                os.path.dirname(os.environ["MOZ_UPLOAD_DIR"])
+            )
 
-        # add raptor to raptor_dir
-        self.raptor_dir = os.path.join(self.raptor_dir, "testing", "raptor")
-        self.recordings_path = self.raptor_dir
-        LOG.info("raptor_dir used for mitmproxy downloads and exe files: %s" % self.raptor_dir)
+        self.mozproxy_dir = os.path.join(self.mozproxy_dir, "testing", "mozproxy")
+        self.recordings_path = self.mozproxy_dir
+        LOG.info(
+            "mozproxy_dir used for mitmproxy downloads and exe files: %s"
+            % self.mozproxy_dir
+        )
 
         # go ahead and download and setup mitmproxy
         self.download()
 
         # mitmproxy must be started before setup, so that the CA cert is available
         self.start()
+
+        # XXX if this fails, we fail to create an insatnce and mitdump has
+        # started and become a ghost
         self.setup()
 
     def download(self):
         """Download and unpack mitmproxy binary and pageset using tooltool"""
-        if not os.path.exists(self.raptor_dir):
-            os.makedirs(self.raptor_dir)
+        if not os.path.exists(self.mozproxy_dir):
+            os.makedirs(self.mozproxy_dir)
 
         LOG.info("downloading mitmproxy binary")
-        _manifest = os.path.join(here, self.config['playback_binary_manifest'])
-        transformed_manifest = transform_platform(_manifest, self.config['platform'])
-        tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
+        _manifest = os.path.join(here, self.config["playback_binary_manifest"])
+        transformed_manifest = transform_platform(_manifest, self.config["platform"])
+        tooltool_download(
+            transformed_manifest, self.config["run_local"], self.mozproxy_dir
+        )
 
         # we use one pageset for all platforms
         LOG.info("downloading mitmproxy pageset")
-        _manifest = os.path.join(here, self.config['playback_pageset_manifest'])
-        transformed_manifest = transform_platform(_manifest, self.config['platform'])
-        tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
-        return
+        _manifest = self.config["playback_pageset_manifest"]
+        transformed_manifest = transform_platform(_manifest, self.config["platform"])
+        tooltool_download(
+            transformed_manifest, self.config["run_local"], self.mozproxy_dir
+        )
 
     def start(self):
         """Start playing back the mitmproxy recording."""
 
-        self.mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump')
+        self.mitmdump_path = os.path.join(self.mozproxy_dir, "mitmdump")
 
         recordings_list = self.recordings.split()
-        self.mitmproxy_proc = self.start_mitmproxy_playback(self.mitmdump_path,
-                                                            self.recordings_path,
-                                                            recordings_list,
-                                                            self.browser_path)
-        return
+        self.mitmproxy_proc = self.start_mitmproxy_playback(
+            self.mitmdump_path, self.recordings_path, recordings_list, self.browser_path
+        )
 
     def stop(self):
         self.stop_mitmproxy_playback()
-        return
 
-    def start_mitmproxy_playback(self,
-                                 mitmdump_path,
-                                 mitmproxy_recording_path,
-                                 mitmproxy_recordings_list,
-                                 browser_path):
+    def start_mitmproxy_playback(
+        self,
+        mitmdump_path,
+        mitmproxy_recording_path,
+        mitmproxy_recordings_list,
+        browser_path,
+    ):
         """Startup mitmproxy and replay the specified flow file"""
 
         LOG.info("mitmdump path: %s" % mitmdump_path)
         LOG.info("recording path: %s" % mitmproxy_recording_path)
         LOG.info("recordings list: %s" % mitmproxy_recordings_list)
         LOG.info("browser path: %s" % browser_path)
-
         mitmproxy_recordings = []
         # recording names can be provided in comma-separated list; build py list including path
         for recording in mitmproxy_recordings_list:
             if not os.path.isfile(os.path.join(mitmproxy_recording_path, recording)):
-                LOG.critical('Recording file {} cannot be found!'.
-                             format(os.path.join(mitmproxy_recording_path, recording)))
-                raise Exception('Recording file {} cannot be found!'.
-                                format(os.path.join(mitmproxy_recording_path, recording)))
-
-            mitmproxy_recordings.append(os.path.join(mitmproxy_recording_path, recording))
+                LOG.critical(
+                    "Recording file {} cannot be found!".format(
+                        os.path.join(mitmproxy_recording_path, recording)
+                    )
+                )
+                raise Exception(
+                    "Recording file {} cannot be found!".format(
+                        os.path.join(mitmproxy_recording_path, recording)
+                    )
+                )
 
-        # cmd line to start mitmproxy playback using custom playback script is as follows:
-        # <path>/mitmdump -s "<path>mitmdump-alternate-server-replay/alternate-server-replay.py
-        #  <path>recording-1.mp <path>recording-2.mp..."
-        param = os.path.join(here, 'alternate-server-replay.py')
-        env = os.environ.copy()
-
-        # this part is platform-specific
-        if mozinfo.os == 'win':
-            param2 = '""' + param.replace('\\', '\\\\\\') + ' ' + \
-                     ' '.join(mitmproxy_recordings).replace('\\', '\\\\\\') + '""'
-            sys.path.insert(1, mitmdump_path)
-        else:
-            # mac and linux
-            param2 = param + ' ' + ' '.join(mitmproxy_recordings)
+            mitmproxy_recordings.append(
+                os.path.join(mitmproxy_recording_path, recording)
+            )
 
         # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
+        env = os.environ.copy()
         env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
+        command = [mitmdump_path, "-k", "-q"]
 
-        command = [mitmdump_path, '-k', '-q', '-s', param2]
+        if "custom_script" in self.config:
+            # cmd line to start mitmproxy playback using custom playback script is as follows:
+            # <path>/mitmdump -s "<path>/alternate-server-replay.py
+            #  <path>recording-1.mp <path>recording-2.mp..."
+            custom_script = self.config["custom_script"] + " " + " ".join(mitmproxy_recordings)
+
+            # this part is platform-specific
+            if mozinfo.os == "win":
+                custom_script = '""' + custom_script.replace("\\", "\\\\\\") + '""'
+                sys.path.insert(1, mitmdump_path)
+
+            command.extend(["-s", custom_script])
 
         LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
-        LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command))
+        LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command))
         # to turn off mitmproxy log output, use these params for Popen:
         # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
         mitmproxy_proc = ProcessHandler(command, env=env)
         mitmproxy_proc.run()
 
+        # XXX replace the code below with a loop with a connection attempt
+        # Bug 1532557
         time.sleep(MITMDUMP_SLEEP)
         data = mitmproxy_proc.poll()
         if data is None:  # None value indicates process hasn't terminated
-            LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid)
+            LOG.info(
+                "Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid
+            )
             return mitmproxy_proc
         # cannot continue as we won't be able to playback the pages
-        LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data)
+        LOG.error(
+            "Aborting: mitmproxy playback process failed to start, poll returned: %s"
+            % data
+        )
+        # XXX here we might end up with a ghost mitmproxy
         sys.exit()
 
     def stop_mitmproxy_playback(self):
         """Stop the mitproxy server playback"""
         mitmproxy_proc = self.mitmproxy_proc
-        LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid)
+        LOG.info("Stopping mitmproxy playback, killing process %d" % mitmproxy_proc.pid)
         mitmproxy_proc.kill()
 
         time.sleep(MITMDUMP_SLEEP)
         status = mitmproxy_proc.poll()
         if status is None:  # None value indicates process hasn't terminated
             # I *think* we can still continue, as process will be automatically
             # killed anyway when mozharness is done (?) if not, we won't be able
             # to startup mitmxproy next time if it is already running
             LOG.error("Failed to kill the mitmproxy playback process")
             LOG.info(str(status))
         else:
             LOG.info("Successfully killed the mitmproxy playback process")
 
 
 class MitmproxyDesktop(Mitmproxy):
-
     def __init__(self, config):
         Mitmproxy.__init__(self, config)
 
     def setup(self):
-        """For Firefox we need to install the generated mitmproxy CA cert. For Chromium this is
-        not necessary as it will be started with the --ignore-certificate-errors cmd line arg"""
-        if self.config['app'] == "firefox":
-            # install the generated CA certificate into Firefox desktop
-            self.install_mitmproxy_cert(self.mitmproxy_proc,
-                                        self.browser_path)
-        else:
+        """
+        Installs certificates.
+
+        For Firefox we need to install the generated mitmproxy CA cert. For
+        Chromium this is not necessary as it will be started with the
+        --ignore-certificate-errors cmd line arg.
+        """
+        if not self.config["app"] == "firefox":
             return
+        # install the generated CA certificate into Firefox desktop
+        self.install_mitmproxy_cert(self.mitmproxy_proc, self.browser_path)
 
     def install_mitmproxy_cert(self, mitmproxy_proc, browser_path):
         """Install the CA certificate generated by mitmproxy, into Firefox
         1. Create a dir called 'distribution' in the same directory as the Firefox executable
         2. Create the policies.json file inside that folder; which points to the certificate
            location, and turns on the the browser proxy settings
         """
         LOG.info("Installing mitmproxy CA certficate into Firefox")
 
         # browser_path is the exe, we want the folder
         self.policies_dir = os.path.dirname(browser_path)
         # on macosx we need to remove the last folders 'MacOS'
         # and the policies json needs to go in ../Content/Resources/
-        if 'mac' in self.config['platform']:
+        if "mac" in self.config["platform"]:
             self.policies_dir = os.path.join(self.policies_dir[:-6], "Resources")
         # for all platforms the policies json goes in a 'distribution' dir
         self.policies_dir = os.path.join(self.policies_dir, "distribution")
 
         self.cert_path = DEFAULT_CERT_PATH
         # for windows only
-        if mozinfo.os == 'win':
-            self.cert_path = self.cert_path.replace('\\', '\\\\')
+        if mozinfo.os == "win":
+            self.cert_path = self.cert_path.replace("\\", "\\\\")
 
         if not os.path.exists(self.policies_dir):
             LOG.info("creating folder: %s" % self.policies_dir)
             os.makedirs(self.policies_dir)
         else:
             LOG.info("folder already exists: %s" % self.policies_dir)
 
-        self.write_policies_json(self.policies_dir,
-                                 policies_content=POLICIES_CONTENT_ON %
-                                 {'cert': self.cert_path,
-                                  'host': self.config['host']})
+        self.write_policies_json(
+            self.policies_dir,
+            policies_content=POLICIES_CONTENT_ON
+            % {"cert": self.cert_path, "host": self.config["host"]},
+        )
 
         # cannot continue if failed to add CA cert to Firefox, need to check
         if not self.is_mitmproxy_cert_installed():
-            LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox desktop')
+            LOG.error(
+                "Aborting: failed to install mitmproxy CA cert into Firefox desktop"
+            )
             self.stop_mitmproxy_playback()
             sys.exit()
 
     def write_policies_json(self, location, policies_content):
         policies_file = os.path.join(location, "policies.json")
         LOG.info("writing: %s" % policies_file)
 
-        with open(policies_file, 'w') as fd:
+        with open(policies_file, "w") as fd:
             fd.write(policies_content)
 
     def read_policies_json(self, location):
         policies_file = os.path.join(location, "policies.json")
         LOG.info("reading: %s" % policies_file)
 
-        with open(policies_file, 'r') as fd:
+        with open(policies_file, "r") as fd:
             return fd.read()
 
     def is_mitmproxy_cert_installed(self):
         """Verify mitmxproy CA cert was added to Firefox"""
         try:
             # read autoconfig file, confirm mitmproxy cert is in there
             contents = self.read_policies_json(self.policies_dir)
             LOG.info("Firefox policies file contents:")
             LOG.info(contents)
-            if (POLICIES_CONTENT_ON % {
-                'cert': self.cert_path,
-                'host': self.config['host']}) in contents:
+            if (
+                POLICIES_CONTENT_ON
+                % {"cert": self.cert_path, "host": self.config["host"]}
+            ) in contents:
                 LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
             else:
 
                 return False
         except Exception as e:
             LOG.info("failed to read Firefox policies file, exeption: %s" % e)
             return False
         return True
 
     def stop(self):
         self.stop_mitmproxy_playback()
         self.turn_off_browser_proxy()
-        return
 
     def turn_off_browser_proxy(self):
         """Turn off the browser proxy that was used for mitmproxy playback. In Firefox
         we need to change the autoconfig files to revert the proxy; for Chromium the proxy
         was setup on the cmd line, so nothing is required here."""
-        if self.config['app'] == "firefox":
+        if self.config["app"] == "firefox":
             LOG.info("Turning off the browser proxy")
 
-            self.write_policies_json(self.policies_dir,
-                                     policies_content=POLICIES_CONTENT_OFF)
+            self.write_policies_json(
+                self.policies_dir, policies_content=POLICIES_CONTENT_OFF
+            )
 
 
 class MitmproxyAndroid(Mitmproxy):
-
     def __init__(self, config, android_device):
         Mitmproxy.__init__(self, config)
         self.android_device = android_device
 
     def setup(self):
         """For geckoview we need to install the generated mitmproxy CA cert"""
-        if self.config['app'] in ["geckoview", "refbrow", "fenix"]:
+        if self.config["app"] in ["geckoview", "refbrow", "fenix"]:
             # install the generated CA certificate into android geckoview
-            self.install_mitmproxy_cert(self.mitmproxy_proc,
-                                        self.browser_path)
-        else:
-            return
+            self.install_mitmproxy_cert(self.mitmproxy_proc, self.browser_path)
 
     def install_mitmproxy_cert(self, mitmproxy_proc, browser_path):
         """Install the CA certificate generated by mitmproxy, into geckoview android
         If running locally:
         1. Will use the `certutil` tool from the local Firefox desktop build
 
         If running in production:
         1. Get the tooltools manifest file for downloading hostutils (contains certutil)
@@ -379,27 +396,27 @@ class MitmproxyAndroid(Mitmproxy):
                                             "raw-file",
                                             os.environ['GECKO_HEAD_REV'],
                                             os.environ['HOSTUTILS_MANIFEST_PATH'])
             else:
                 LOG.critical("Abort: unable to get HOSTUTILS_MANIFEST_PATH!")
                 raise
 
             # first need to download the hostutils tooltool manifest file itself
-            _dest = os.path.join(self.raptor_dir, 'hostutils.manifest')
+            _dest = os.path.join(self.mozproxy_dir, 'hostutils.manifest')
             have_manifest = download_file_from_url(manifest_url, _dest)
             if not have_manifest:
                 LOG.critical('failed to download the hostutils tooltool manifest')
                 raise
 
             # now use the manifest to download hostutils so we can get certutil
-            tooltool_download(_dest, self.config['run_local'], self.raptor_dir)
+            tooltool_download(_dest, self.config['run_local'], self.mozproxy_dir)
 
             # the production bitbar container host is always linux
-            self.certutil = os.path.join(self.raptor_dir, 'host-utils-67.0a1.en-US.linux-x86_64')
+            self.certutil = os.path.join(self.mozproxy_dir, 'host-utils-67.0a1.en-US.linux-x86_64')
 
             # must add hostutils/certutil to the path
             os.environ['LD_LIBRARY_PATH'] = self.certutil
 
         bin_suffix = mozinfo.info.get('bin_suffix', '')
         self.certutil = os.path.join(self.certutil, "certutil" + bin_suffix)
 
         if os.path.isfile(self.certutil):
@@ -408,79 +425,100 @@ class MitmproxyAndroid(Mitmproxy):
             LOG.critical("unable to find certutil at %s" % self.certutil)
             raise
 
         # DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e.
         # /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer
         self.local_cert_path = DEFAULT_CERT_PATH
 
         # check if the nss ca cert db already exists in the device profile
-        LOG.info("checking if the nss cert db already exists in the android browser profile")
-        param1 = "sql:%s/" % self.config['local_profile_dir']
-        command = [self.certutil, '-d', param1, '-L']
+        LOG.info(
+            "checking if the nss cert db already exists in the android browser profile"
+        )
+        param1 = "sql:%s/" % self.config["local_profile_dir"]
+        command = [self.certutil, "-d", param1, "-L"]
 
         try:
             subprocess.check_output(command, env=os.environ.copy())
             LOG.info("the nss cert db already exists")
             cert_db_exists = True
         except subprocess.CalledProcessError:
             # this means the nss cert db doesn't exist yet
             LOG.info("nss cert db doesn't exist yet")
             cert_db_exists = False
 
         # try a forced pause between certutil cmds; possibly reduce later
         time.sleep(self.CERTUTIL_SLEEP)
 
         if not cert_db_exists:
             # create cert db if it doesn't already exist; it may exist already
             # if a previous pageload test ran in the same test suite
-            param1 = "sql:%s/" % self.config['local_profile_dir']
-            command = [self.certutil, '-N', '-v', '-d', param1, '--empty-password']
+            param1 = "sql:%s/" % self.config["local_profile_dir"]
+            command = [self.certutil, "-N", "-v", "-d", param1, "--empty-password"]
 
-            LOG.info("creating nss cert database using command: %s" % ' '.join(command))
+            LOG.info("creating nss cert database using command: %s" % " ".join(command))
             cmd_proc = subprocess.Popen(command, env=os.environ.copy())
             time.sleep(self.CERTUTIL_SLEEP)
             cmd_terminated = cmd_proc.poll()
             if cmd_terminated is None:  # None value indicates process hasn't terminated
                 LOG.critical("nss cert db creation command failed to complete")
                 raise
 
         # import mitmproxy cert into the db
-        command = [self.certutil, '-A', '-d', param1, '-n',
-                   'mitmproxy-cert', '-t', 'TC,,', '-a', '-i', self.local_cert_path]
+        command = [
+            self.certutil,
+            "-A",
+            "-d",
+            param1,
+            "-n",
+            "mitmproxy-cert",
+            "-t",
+            "TC,,",
+            "-a",
+            "-i",
+            self.local_cert_path,
+        ]
 
-        LOG.info("importing mitmproxy cert into db using command: %s" % ' '.join(command))
+        LOG.info(
+            "importing mitmproxy cert into db using command: %s" % " ".join(command)
+        )
         cmd_proc = subprocess.Popen(command, env=os.environ.copy())
         time.sleep(self.CERTUTIL_SLEEP)
         cmd_terminated = cmd_proc.poll()
         if cmd_terminated is None:  # None value indicates process hasn't terminated
-            LOG.critical("command to import mitmproxy cert into cert db failed to complete")
+            LOG.critical(
+                "command to import mitmproxy cert into cert db failed to complete"
+            )
 
         # cannot continue if failed to add CA cert to Firefox, need to check
         if not self.is_mitmproxy_cert_installed():
             LOG.error("Aborting: failed to install mitmproxy CA cert into Firefox")
             self.stop_mitmproxy_playback()
             sys.exit()
 
     def is_mitmproxy_cert_installed(self):
         """Verify mitmxproy CA cert was added to Firefox on android"""
         LOG.info("verifying that the mitmproxy ca cert is installed on android")
 
         # list the certifcates that are in the nss cert db (inside the browser profile dir)
-        LOG.info("getting the list of certs in the nss cert db in the android browser profile")
-        param1 = "sql:%s/" % self.config['local_profile_dir']
-        command = [self.certutil, '-d', param1, '-L']
+        LOG.info(
+            "getting the list of certs in the nss cert db in the android browser profile"
+        )
+        param1 = "sql:%s/" % self.config["local_profile_dir"]
+        command = [self.certutil, "-d", param1, "-L"]
 
         try:
             cmd_output = subprocess.check_output(command, env=os.environ.copy())
 
         except subprocess.CalledProcessError:
             # cmd itself failed
             LOG.critical("certutil command failed")
             raise
 
         # check output from the certutil command, see if 'mitmproxy-cert' is listed
         time.sleep(self.CERTUTIL_SLEEP)
         LOG.info(cmd_output)
-        if 'mitmproxy-cert' in cmd_output:
-            LOG.info("verfied the mitmproxy-cert is installed in the nss cert db on android")
+        if "mitmproxy-cert" in cmd_output:
+            LOG.info(
+                "verfied the mitmproxy-cert is installed in the nss cert db on android"
+            )
             return True
         return False
rename from testing/raptor/raptor/playback/mitmproxy-rel-bin-linux64.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-linux64.manifest
rename from testing/raptor/raptor/playback/mitmproxy-rel-bin-osx.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-osx.manifest
rename from testing/raptor/raptor/playback/mitmproxy-rel-bin-win.manifest
rename to testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-win.manifest
rename from testing/raptor/raptor/playback/mitmproxy_requirements.txt
rename to testing/mozbase/mozproxy/mozproxy/backends/mitmproxy_requirements.txt
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/mozproxy/server.py
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+from mozproxy.backends.mitm import MitmproxyDesktop, MitmproxyAndroid
+
+_BACKENDS = {"mitmproxy": MitmproxyDesktop, "mitmproxy-android": MitmproxyAndroid}
+
+
+def get_backend(name, *args, **kw):
+    """Returns the class that implements the backend.
+
+    Raises KeyError in case the backend does not exists.
+    """
+    return _BACKENDS[name](*args, **kw)
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/mozproxy/utils.py
@@ -0,0 +1,108 @@
+"""Utility functions for Raptor"""
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import os
+import signal
+import sys
+import urllib
+
+from mozlog import get_proxy_logger
+from mozprocess import ProcessHandler
+from mozproxy import mozharness_dir
+
+
+LOG = get_proxy_logger(component="mozproxy")
+
+external_tools_path = os.environ.get("EXTERNALTOOLSPATH", None)
+if external_tools_path is not None:
+    # running in production via mozharness
+    TOOLTOOL_PATH = os.path.join(external_tools_path, "tooltool.py")
+else:
+    # running locally via mach
+    TOOLTOOL_PATH = os.path.join(mozharness_dir, "external_tools", "tooltool.py")
+
+
+def transform_platform(str_to_transform, config_platform, config_processor=None):
+    """Transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest'
+
+    transforms to 'mitmproxy-rel-bin-osx.manifest'.
+    Also transform '{x64}' if needed for 64 bit / win 10
+    """
+    if "{platform}" not in str_to_transform and "{x64}" not in str_to_transform:
+        return str_to_transform
+
+    if "win" in config_platform:
+        platform_id = "win"
+    elif config_platform == "mac":
+        platform_id = "osx"
+    else:
+        platform_id = "linux64"
+
+    if "{platform}" in str_to_transform:
+        str_to_transform = str_to_transform.replace("{platform}", platform_id)
+
+    if "{x64}" in str_to_transform and config_processor is not None:
+        if "x86_64" in config_processor:
+            str_to_transform = str_to_transform.replace("{x64}", "_x64")
+        else:
+            str_to_transform = str_to_transform.replace("{x64}", "")
+
+    return str_to_transform
+
+
+def tooltool_download(manifest, run_local, raptor_dir):
+    """Download a file from tooltool using the provided tooltool manifest"""
+    def outputHandler(line):
+        LOG.info(line)
+
+    if run_local:
+        command = [sys.executable, TOOLTOOL_PATH, "fetch", "-o", "-m", manifest]
+    else:
+        # we want to use the tooltool cache in production
+        if os.environ.get("TOOLTOOLCACHE") is not None:
+            _cache = os.environ["TOOLTOOLCACHE"]
+        else:
+            # XXX top level dir? really?
+            # that gets run locally on any platform
+            # when you call ./mach python-test
+            _cache = "/builds/tooltool_cache"
+
+        command = [
+            sys.executable,
+            TOOLTOOL_PATH,
+            "fetch",
+            "-o",
+            "-m",
+            manifest,
+            "-c",
+            _cache,
+        ]
+
+    proc = ProcessHandler(
+        command, processOutputLine=outputHandler, storeOutput=False, cwd=raptor_dir
+    )
+
+    proc.run()
+
+    try:
+        proc.wait()
+    except Exception:
+        if proc.poll() is None:
+            proc.kill(signal.SIGTERM)
+
+
+def download_file_from_url(url, local_dest):
+    """Receive a file in a URL and download it, i.e. for the hostutils tooltool manifest
+    the url received would be formatted like this:
+    https://hg.mozilla.org/try/raw-file/acb5abf52c04da7d4548fa13bd6c6848a90c32b8/testing/
+      config/tooltool-manifests/linux64/hostutils.manifest"""
+    if os.path.exists(local_dest):
+        LOG.info("file already exists at: %s" % local_dest)
+        return True
+    LOG.info("downloading: %s to %s" % (url, local_dest))
+    _file, _headers = urllib.urlretrieve(url, local_dest)
+    return os.path.exists(local_dest)
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/setup.py
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from setuptools import setup
+
+PACKAGE_NAME = "mozproxy"
+PACKAGE_VERSION = "1.0"
+
+# dependencies
+deps = []
+
+setup(
+    name=PACKAGE_NAME,
+    version=PACKAGE_VERSION,
+    description="Proxy for playback" "left behind by crashed processes",
+    long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html",
+    classifiers=[
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3.5",
+    ],
+    # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+    keywords="mozilla",
+    author="Mozilla Automation and Tools team",
+    author_email="tools@lists.mozilla.org",
+    url="https://wiki.mozilla.org/Auto-tools/Projects/Mozbase",
+    license="MPL",
+    packages=["mozproxy"],
+    include_package_data=True,
+    zip_safe=False,
+    install_requires=deps,
+)
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/tests/firefox
@@ -0,0 +1,1 @@
+# I am firefox
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/tests/manifest.ini
@@ -0,0 +1,3 @@
+[DEFAULT]
+subsuite = mozbase
+[test.py]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/tests/paypal.mp
@@ -0,0 +1,1 @@
+# fake recorded playback
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozproxy/tests/test.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+from __future__ import absolute_import, print_function
+import os
+import subprocess
+
+import mozprocess
+import mozunit
+import mozinfo
+from mozproxy import get_playback
+from mozproxy import utils
+
+
+here = os.path.dirname(__file__)
+# XXX we need to create a temp dir where things happen, for cleanup
+# otherwise, the source tree gets polluted
+
+# XXX will do better later using a real mock tool
+
+
+def _no_download(*args, **kw):
+    pass
+
+
+utils.tooltool_download = _no_download
+
+
+def check_output(*args, **kw):
+    return "yea ok"
+
+
+subprocess.check_output = check_output
+
+
+class ProcessHandler:
+    def __init__(self, *args, **kw):
+        self.pid = 12345
+
+    def run(self):
+        pass
+
+    def poll(self):
+        return None
+
+
+mozprocess.ProcessHandler = ProcessHandler
+
+
+def test_mitm():
+    from mozproxy.backends import mitm
+
+    mitm.MITMDUMP_SLEEP = 0.1
+
+    bin_name = "mitmproxy-rel-bin-{platform}.manifest"
+    pageset_name = "mitmproxy-recordings-raptor-paypal.manifest"
+
+    config = {
+        "playback_tool": "mitmproxy",
+        "playback_binary_manifest": bin_name,
+        "playback_pageset_manifest": pageset_name,
+        "platform": mozinfo.os,
+        "playback_recordings": os.path.join(here, "paypal.mp"),
+        "run_local": True,
+        "obj_path": here,   # XXX tmp?
+        "binary": "firefox",
+        "app": "firefox",
+        "host": "example.com",
+    }
+
+    playback = get_playback(config)
+    assert playback is not None
+
+
+if __name__ == "__main__":
+    mozunit.main(runwith="pytest")
--- a/testing/mozbase/packages.txt
+++ b/testing/mozbase/packages.txt
@@ -6,13 +6,14 @@ mozfile.pth:testing/mozbase/mozfile
 mozhttpd.pth:testing/mozbase/mozhttpd
 mozinfo.pth:testing/mozbase/mozinfo
 mozinstall.pth:testing/mozbase/mozinstall
 mozleak.pth:testing/mozbase/mozleak
 mozlog.pth:testing/mozbase/mozlog
 moznetwork.pth:testing/mozbase/moznetwork
 mozprocess.pth:testing/mozbase/mozprocess
 mozprofile.pth:testing/mozbase/mozprofile
+mozproxy.pth:testing/mozbase/mozproxy
 mozrunner.pth:testing/mozbase/mozrunner
 mozsystemmonitor.pth:testing/mozbase/mozsystemmonitor
 mozscreenshot.pth:testing/mozbase/mozscreenshot
 moztest.pth:testing/mozbase/moztest
 mozversion.pth:testing/mozbase/mozversion
--- a/testing/mozharness/mozharness/mozilla/mozbase.py
+++ b/testing/mozharness/mozharness/mozilla/mozbase.py
@@ -28,12 +28,12 @@ class MozbaseMixin(object):
         mozbase_dir = os.path.join('tests', 'mozbase')
         self.register_virtualenv_module(
             'manifestparser',
             url=os.path.join(mozbase_dir, 'manifestdestiny')
         )
 
         for m in ('mozfile', 'mozlog', 'mozinfo', 'moznetwork', 'mozhttpd',
                   'mozcrash', 'mozinstall', 'mozdevice', 'mozprofile',
-                  'mozprocess', 'mozrunner'):
+                  'mozprocess', 'mozproxy', 'mozrunner'):
             self.register_virtualenv_module(
                 m, url=os.path.join(mozbase_dir, m)
             )
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -13,17 +13,17 @@ from utils import transform_platform
 here = os.path.abspath(os.path.dirname(__file__))
 raptor_ini = os.path.join(here, 'raptor.ini')
 tests_dir = os.path.join(here, 'tests')
 LOG = get_proxy_logger(component="raptor-manifest")
 
 required_settings = ['apps', 'type', 'page_cycles', 'test_url', 'measure',
                      'unit', 'lower_is_better', 'alert_threshold']
 
-playback_settings = ['playback_binary_manifest', 'playback_pageset_manifest',
+playback_settings = ['playback_pageset_manifest',
                      'playback_recordings']
 
 
 def filter_app(tests, values):
     for test in tests:
         if values["app"] in test['apps']:
             yield test
 
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -18,40 +18,45 @@ import mozinfo
 
 from mozdevice import ADBDevice
 from mozlog import commandline, get_default_logger
 from mozprofile import create_profile
 from mozrunner import runners
 
 # need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
+paths = [here]
 if os.environ.get('SCRIPTSPATH', None) is not None:
     # in production it is env SCRIPTS_PATH
-    mozharness_dir = os.environ['SCRIPTSPATH']
+    paths.append(os.environ['SCRIPTSPATH'])
 else:
     # locally it's in source tree
-    mozharness_dir = os.path.join(here, '../../../mozharness')
-sys.path.insert(0, mozharness_dir)
+    paths.append(os.path.join(here, '..', '..', 'mozharness'))
+
+webext_dir = os.path.join(here, '..', 'webext')
+paths.append(webext_dir)
 
-webext_dir = os.path.join(os.path.dirname(here), 'webext')
-sys.path.insert(0, here)
+for path in paths:
+    if not os.path.exists(path):
+        raise IOError("%s does not exist. " % path)
+    sys.path.insert(0, path)
 
 try:
     from mozbuild.base import MozbuildObject
     build = MozbuildObject.from_environment(cwd=here)
 except ImportError:
     build = None
 
 from benchmark import Benchmark
 from cmdline import parse_args
 from control_server import RaptorControlServer
 from gen_test_config import gen_test_config
 from outputhandler import OutputHandler
 from manifest import get_raptor_test_list
-from playback import get_playback
+from mozproxy import get_playback
 from results import RaptorResultsHandler
 from gecko_profile import GeckoProfile
 from power import init_geckoview_power_test, finish_geckoview_power_test
 from utils import view_gecko_profile
 
 
 class Raptor(object):
     """Container class for Raptor"""
@@ -191,16 +196,22 @@ class Raptor(object):
         self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
         self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
         _key = 'playback_binary_zip_%s' % self.config['platform']
         self.config['playback_binary_zip'] = test.get(_key, None)
         self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
         _key = 'playback_pageset_zip_%s' % self.config['platform']
         self.config['playback_pageset_zip'] = test.get(_key, None)
         self.config['playback_recordings'] = test.get('playback_recordings', None)
+        playback_dir = os.path.join(here, 'playback')
+        for key in ('playback_pageset_manifest', 'playback_pageset_zip'):
+            if self.config.get(key) is None:
+                continue
+            self.config[key] = os.path.join(playback_dir, self.config[key])
+        self.config['custom_script'] = os.path.join(playback_dir, 'alternate-server-replay.py')
 
     def serve_benchmark_source(self, test):
         # benchmark-type tests require the benchmark test to be served out
         self.benchmark = Benchmark(self.config, test)
         self.benchmark_port = int(self.benchmark.port)
 
         # for android we must make the benchmarks server available to the device
         if self.config['app'] in self.firefox_android_apps and \
--- a/testing/raptor/raptor/utils.py
+++ b/testing/raptor/raptor/utils.py
@@ -1,24 +1,21 @@
 """Utility functions for Raptor"""
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 import os
-import signal
 import subprocess
 import sys
 import time
-import urllib
 
 from mozlog import get_proxy_logger, get_default_logger
-from mozprocess import ProcessHandler
 
 LOG = get_proxy_logger(component="raptor-utils")
 here = os.path.dirname(os.path.realpath(__file__))
 
 if os.environ.get('SCRIPTSPATH', None) is not None:
     # in production it is env SCRIPTS_PATH
     mozharness_dir = os.environ['SCRIPTSPATH']
 else:
@@ -57,67 +54,16 @@ def transform_platform(str_to_transform,
         if 'x86_64' in config_processor:
             str_to_transform = str_to_transform.replace('{x64}', '_x64')
         else:
             str_to_transform = str_to_transform.replace('{x64}', '')
 
     return str_to_transform
 
 
-def tooltool_download(manifest, run_local, raptor_dir):
-    """Download a file from tooltool using the provided tooltool manifest"""
-    def outputHandler(line):
-        LOG.info(line)
-    if run_local:
-        command = [sys.executable,
-                   TOOLTOOL_PATH,
-                   'fetch',
-                   '-o',
-                   '-m', manifest]
-    else:
-        # we want to use the tooltool cache in production
-        if os.environ.get('TOOLTOOLCACHE', None) is not None:
-            _cache = os.environ['TOOLTOOLCACHE']
-        else:
-            _cache = "/builds/tooltool_cache"
-
-        command = [sys.executable,
-                   TOOLTOOL_PATH,
-                   'fetch',
-                   '-o',
-                   '-m', manifest,
-                   '-c',
-                   _cache]
-
-    proc = ProcessHandler(
-        command, processOutputLine=outputHandler, storeOutput=False,
-        cwd=raptor_dir)
-
-    proc.run()
-
-    try:
-        proc.wait()
-    except Exception:
-        if proc.poll() is None:
-            proc.kill(signal.SIGTERM)
-
-
-def download_file_from_url(url, local_dest):
-    """Receive a file in a URL and download it, i.e. for the hostutils tooltool manifest
-    the url received would be formatted like this:
-    https://hg.mozilla.org/try/raw-file/acb5abf52c04da7d4548fa13bd6c6848a90c32b8/testing/
-      config/tooltool-manifests/linux64/hostutils.manifest"""
-    if os.path.exists(local_dest):
-        LOG.info("file already exists at: %s" % local_dest)
-        return True
-    LOG.info("downloading: %s to %s" % (url, local_dest))
-    _file, _headers = urllib.urlretrieve(url, local_dest)
-    return os.path.exists(local_dest)
-
-
 def view_gecko_profile(ffox_bin):
     # automatically load the latest talos gecko-profile archive in profiler.firefox.com
     LOG = get_default_logger(component='raptor-view-gecko-profile')
 
     if sys.platform.startswith('win') and not ffox_bin.endswith(".exe"):
         ffox_bin = ffox_bin + ".exe"
 
     if not os.path.exists(ffox_bin):
--- a/testing/raptor/requirements.txt
+++ b/testing/raptor/requirements.txt
@@ -1,6 +1,7 @@
 mozcrash ~= 1.0
 mozrunner ~= 7.0
 mozprofile ~= 2.1
 manifestparser >= 1.1
 wptserve ~= 1.4.0
 mozdevice >= 1.1.6
+mozproxy >= 1.0
--- a/testing/raptor/test/test_playback.py
+++ b/testing/raptor/test/test_playback.py
@@ -4,38 +4,41 @@ import os
 
 import mozinfo
 import mozunit
 
 from mozlog.structuredlog import set_default_logger, StructuredLogger
 
 set_default_logger(StructuredLogger('test_playback'))
 
-from raptor.playback import get_playback, MitmproxyDesktop
+from mozproxy import get_playback
+from mozproxy.backends.mitm import MitmproxyDesktop
 
 config = {}
 
 run_local = True
-if os.environ.get('TOOLTOOLCACHE', None) is None:
+if os.environ.get('TOOLTOOLCACHE') is None:
     run_local = False
 
 
 def test_get_playback(get_binary):
     config['platform'] = mozinfo.os
     if 'win' in config['platform']:
         # this test is not yet supported on windows
         assert True
         return
     config['obj_path'] = os.path.dirname(get_binary('firefox'))
     config['playback_tool'] = 'mitmproxy'
     config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
     config['playback_pageset_manifest'] = 'mitmproxy-recordings-raptor-tp6-1.manifest'
     config['playback_recordings'] = 'amazon.mp'
     config['binary'] = get_binary('firefox')
     config['run_local'] = run_local
+    config['app'] = 'firefox'
+    config['host'] = 'example.com'
 
     playback = get_playback(config)
     assert isinstance(playback, MitmproxyDesktop)
     playback.stop()
 
 
 def test_get_unsupported_playback():
     config['playback_tool'] = 'unsupported'