Bug 1506912 - Raptor tp6m-1 pageload test on android geckoview; r=jmaher
authorRob Wood <rwood@mozilla.com>
Thu, 17 Jan 2019 03:01:40 +0000
changeset 514216 ef3912c3403b4b99af5e2188eb44f9ca067ebf1c
parent 514182 e3cb5a5ef667973b60314b67e13184f5d9cedb18
child 514217 671054606438709ebb65855558624cf280bcef55
push id1953
push userffxbld-merge
push dateMon, 11 Mar 2019 12:10:20 +0000
treeherdermozilla-release@9c35dcbaa899 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1506912
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1506912 - Raptor tp6m-1 pageload test on android geckoview; r=jmaher Differential Revision: https://phabricator.services.mozilla.com/D15143
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/mozharness/configs/raptor/android_hw_config.py
testing/raptor/raptor/playback/__init__.py
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-amazon.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-facebook.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-google.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-youtube.manifest
testing/raptor/raptor/playback/mitmproxy.py
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/raptor-tp6m-1.ini
testing/raptor/raptor/utils.py
testing/raptor/requirements.txt
testing/raptor/test/test_playback.py
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -342,16 +342,29 @@ raptor-tp6-10-chrome:
         by-test-platform:
             linux64.*: 3
             default: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6-10
             - --app=chrome
 
+raptor-tp6m-1-geckoview:
+    description: "Raptor tp6m-1 on Geckoview"
+    try-name: raptor-tp6m-1-geckoview
+    treeherder-symbol: Rap(tp6m-1)
+    target: geckoview_example.apk
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 3
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-1
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     run-on-projects:
         by-test-platform:
             windows10-64-ux: ['try', 'mozilla-central']
             default: built-projects
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -401,22 +401,23 @@ android-hw-arm7-opt-unittests:
     - mochitest-media
 
 android-hw-arm7-debug-unittests:
     - mochitest-media
 
 android-hw-aarch64-opt-unittests:
     - jittest
 
-# Coming soonish!
 android-hw-arm7-raptor:
     - raptor-speedometer-geckoview
+    - raptor-tp6m-1-geckoview
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
+    - raptor-tp6m-1-geckoview
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-aarch64-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-arm7-raptor-nightly:
--- a/testing/mozharness/configs/raptor/android_hw_config.py
+++ b/testing/mozharness/configs/raptor/android_hw_config.py
@@ -12,9 +12,14 @@ config = {
         "install",
         "run-tests",
     ],
     "tooltool_cache": "/builds/tooltool_cache",
     "download_tooltool": True,
     "minidump_stackwalk_path": "linux64-minidump_stackwalk",
     "tooltool_servers": ['https://tooltool.mozilla-releng.net/'],
     "minidump_tooltool_manifest_path": "config/tooltool-manifests/linux64/releng.manifest",
+    "hostutils_manifest_path": "testing/config/tooltool-manifests/linux64/hostutils.manifest",
 }
+
+# raptor will pick these up in mitmproxy.py, doesn't use the mozharness config
+os.environ['TOOLTOOLCACHE'] = config['tooltool_cache']
+os.environ['HOSTUTILS_MANIFEST_PATH'] = config['hostutils_manifest_path']
--- a/testing/raptor/raptor/playback/__init__.py
+++ b/testing/raptor/raptor/playback/__init__.py
@@ -1,23 +1,27 @@
 from __future__ import absolute_import
 
 from mozlog import get_proxy_logger
-from .mitmproxy import Mitmproxy
+from .mitmproxy import MitmproxyDesktop, MitmproxyAndroid
 
 LOG = get_proxy_logger(component='mitmproxy')
 
 playback_cls = {
-    'mitmproxy': Mitmproxy,
+    'mitmproxy': MitmproxyDesktop,
+    'mitmproxy-android': MitmproxyAndroid,
 }
 
 
 def get_playback(config, android_device=None):
     tool_name = config.get('playback_tool', None)
     if tool_name is None:
         LOG.critical("playback_tool name not found in config")
         return
     if playback_cls.get(tool_name, None) is None:
         LOG.critical("specified playback tool is unsupported: %s" % tool_name)
         return None
 
     cls = playback_cls.get(tool_name)
-    return cls(config, android_device)
+    if android_device is None:
+        return cls(config)
+    else:
+        return cls(config, android_device)
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-amazon.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 2503727,
+    "visibility": "public",
+    "digest": "37a2a9a751452d67b76b386faa034727a93d1737c4fe74216c17f6b8cb58bb8770e5adecddfb1d8d3bab1e5a2f4fb14dde98ee4a90709871db172a9012c58511",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-tp6m-amazon.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-facebook.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 66557,
+    "visibility": "public",
+    "digest": "d76080610277be4ce1a0a5983f88a7b028f8f347eeb14f9672542ae427d2bdd7f55bea62327651acb1b440df5a36ca6046382099b82676dbc3682f5299ececf6",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-tp6m-facebook.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-google.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 105653,
+    "visibility": "public",
+    "digest": "8f513ea138b722f96c1413d1399afc116c3b4d45c9784b521a920cdd74ec41a81ba32f8e4edae558fc4362122dda32f4074957d1e526aad1cdc777f941d8e81a",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-tp6m-google.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-youtube.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 1040164,
+    "visibility": "public",
+    "digest": "6dbeaaa3174720de6080a812732b69514237c4cf802dac2752cc6cffe7ad2f6b340dd82194b9416f23b0a04be5f44f628e99f89ec162e30d31b13483f5e93912",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-tp6m-youtube.zip",
+    "unpack": true
+  }
+]
--- a/testing/raptor/raptor/playback/mitmproxy.py
+++ b/testing/raptor/raptor/playback/mitmproxy.py
@@ -1,25 +1,22 @@
-'''Functions to download, install, setup, and use the mitmproxy playback tool'''
+"""Functions to download, install, setup, and use the mitmproxy playback tool"""
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import absolute_import
 
 import os
-import signal
 import subprocess
 import sys
-
 import time
 
 import mozinfo
 
 from mozlog import get_proxy_logger
-from mozprocess import ProcessHandler
 
 from .base import Playback
 
 here = os.path.dirname(os.path.realpath(__file__))
 LOG = get_proxy_logger(component='raptor-mitmproxy')
 
 # needed so unit tests can find their imports
 if os.environ.get('SCRIPTSPATH', None) is not None:
@@ -33,40 +30,34 @@ sys.path.insert(0, mozharness_dir)
 # required for using a python3 virtualenv on win for mitmproxy
 from mozharness.base.python import Python3Virtualenv
 from mozharness.mozilla.testing.testbase import TestingMixin
 from mozharness.base.vcs.vcsbase import MercurialScript
 
 raptor_dir = os.path.join(here, '..')
 sys.path.insert(0, raptor_dir)
 
-from utils import transform_platform
-
-external_tools_path = os.environ.get('EXTERNALTOOLSPATH', None)
-
-if external_tools_path is not None:
-    # running in production via mozharness
-    TOOLTOOL_PATH = os.path.join(external_tools_path, 'tooltool.py')
-else:
-    # running locally via mach
-    TOOLTOOL_PATH = os.path.join(mozharness_dir, 'external_tools', 'tooltool.py')
+from utils import transform_platform, tooltool_download, download_file_from_url
 
 # path for mitmproxy certificate, generated auto after mitmdump is started
 # on local machine it is 'HOME', however it is different on production machines
 try:
     DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
                                      '.mitmproxy', 'mitmproxy-ca-cert.cer')
 except Exception:
     DEFAULT_CERT_PATH = os.path.join(os.getenv('HOMEDRIVE'), os.getenv('HOMEPATH'),
                                      '.mitmproxy', 'mitmproxy-ca-cert.cer')
 
 # On Windows, deal with mozilla-build having forward slashes in $HOME:
 if os.name == 'nt' and '/' in DEFAULT_CERT_PATH:
     DEFAULT_CERT_PATH = DEFAULT_CERT_PATH.replace('/', '\\')
 
+# sleep in seconds after issuing a `mitmdump` command
+MITMDUMP_SLEEP = 10
+
 # to install mitmproxy certificate into Firefox and turn on/off proxy
 POLICIES_CONTENT_ON = '''{
   "policies": {
     "Certificates": {
       "Install": ["%(cert)s"]
     },
     "Proxy": {
       "Mode": "manual",
@@ -85,23 +76,22 @@ POLICIES_CONTENT_OFF = '''{
       "Locked": false
     }
   }
 }'''
 
 
 class Mitmproxy(Playback, Python3Virtualenv, TestingMixin, MercurialScript):
 
-    def __init__(self, config, android_device=None):
+    def __init__(self, config):
         self.config = config
         self.mitmproxy_proc = None
         self.mitmdump_path = None
         self.recordings = config.get('playback_recordings', None)
         self.browser_path = config.get('binary', None)
-        self.android_device = android_device
 
         # raptor_dir is where we will download all mitmproxy required files
         # when running locally it comes from obj_path via mozharness/mach
         if self.config.get("obj_path", None) is not None:
             self.raptor_dir = self.config.get("obj_path")
         else:
             # in production it is ../tasks/task_N/build/, in production that dir
             # is not available as an envvar, however MOZ_UPLOAD_DIR is set as
@@ -119,66 +109,48 @@ class Mitmproxy(Playback, Python3Virtual
         # on windows we must use a python3 virtualen for mitmproxy
         if 'win' in self.config['platform']:
             self.setup_py3_virtualenv()
 
         # mitmproxy must be started before setup, so that the CA cert is available
         self.start()
         self.setup()
 
-    def _tooltool_fetch(self, manifest):
-        def outputHandler(line):
-            LOG.info(line)
-        command = [sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest]
-
-        proc = ProcessHandler(
-            command, processOutputLine=outputHandler, storeOutput=False,
-            cwd=self.raptor_dir)
-
-        proc.run()
-
-        try:
-            proc.wait()
-        except Exception:
-            if proc.poll() is None:
-                proc.kill(signal.SIGTERM)
-
     def download(self):
-        # download mitmproxy binary and pageset using tooltool
-        # note: tooltool automatically unpacks the files as well
+        """Download and unpack mitmproxy binary and pageset using tooltool"""
         if not os.path.exists(self.raptor_dir):
             os.makedirs(self.raptor_dir)
 
         if 'win' in self.config['platform']:
             # on windows we need a python3 environment and use our own package from tooltool
             self.py3_path = self.fetch_python3()
             LOG.info("python3 path is: %s" % self.py3_path)
         else:
             # on osx and linux we use pre-built binaries
             LOG.info("downloading mitmproxy binary")
             _manifest = os.path.join(here, self.config['playback_binary_manifest'])
             transformed_manifest = transform_platform(_manifest, self.config['platform'])
-            self._tooltool_fetch(transformed_manifest)
+            tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
 
         # we use one pageset for all platforms
         LOG.info("downloading mitmproxy pageset")
         _manifest = os.path.join(here, self.config['playback_pageset_manifest'])
         transformed_manifest = transform_platform(_manifest, self.config['platform'])
-        self._tooltool_fetch(transformed_manifest)
+        tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
         return
 
     def fetch_python3(self):
         """Mitmproxy on windows needs Python 3.x"""
         python3_path = os.path.join(self.raptor_dir, 'python3.6', 'python')
         if not os.path.exists(os.path.dirname(python3_path)):
             _manifest = os.path.join(here, self.config['python3_win_manifest'])
             transformed_manifest = transform_platform(_manifest, self.config['platform'],
                                                       self.config['processor'])
             LOG.info("downloading py3 package for mitmproxy windows: %s" % transformed_manifest)
-            self._tooltool_fetch(transformed_manifest)
+            tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
         cmd = [python3_path, '--version']
         # just want python3 ver printed in production log
         subprocess.Popen(cmd, env=os.environ.copy())
         return python3_path
 
     def setup_py3_virtualenv(self):
         """Mitmproxy on windows needs Python 3.x; set up a separate py 3.x env here"""
         LOG.info("Setting up python 3.x virtualenv, required for mitmproxy on windows")
@@ -193,182 +165,33 @@ class Mitmproxy(Playback, Python3Virtual
         requirements = [os.path.join(here, "mitmproxy_requirements.txt")]
         self.py3_install_requirement_files(requirements)
         # add py3 executables path to system path
         sys.path.insert(1, self.py3_path_to_executables())
         # install mitmproxy itself
         self.py3_install_modules(modules=['mitmproxy'])
         self.mitmdump_path = os.path.join(self.py3_path_to_executables(), 'mitmdump')
 
-    def setup(self):
-        # for firefox we need to install the generated mitmproxy CA cert
-        # for google chromium this is not necessary as chromium will be
-        # started with --ignore-certificate-errors cmd line arg
-        if self.config['app'] == "firefox":
-            # install the generated CA certificate into Firefox desktop
-            self.install_mitmproxy_cert_desktop(self.mitmproxy_proc,
-                                                self.browser_path)
-        elif self.config['app'] == "geckoview":
-            # install the generated CA certificate into android geckoview
-            self.install_mitmproxy_cert_android(self.mitmproxy_proc,
-                                                self.browser_path)
-        else:
-            return
-
     def start(self):
-        # if on windows, the mitmdump_path was already set when creating py3 env
+        """Start playing back the mitmproxy recording. If on windows, the mitmdump_path was
+        already set when creating py3 env"""
         if self.mitmdump_path is None:
             self.mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump')
 
         recordings_list = self.recordings.split()
         self.mitmproxy_proc = self.start_mitmproxy_playback(self.mitmdump_path,
                                                             self.recordings_path,
                                                             recordings_list,
                                                             self.browser_path)
         return
 
     def stop(self):
         self.stop_mitmproxy_playback()
-        self.turn_off_browser_proxy()
         return
 
-    def install_mitmproxy_cert_desktop(self, mitmproxy_proc, browser_path):
-        """Install the CA certificate generated by mitmproxy, into Firefox
-        1. Create a dir called 'distribution' in the same directory as the Firefox executable
-        2. Create the policies.json file inside that folder; which points to the certificate
-           location, and turns on the the browser proxy settings
-        """
-        LOG.info("Installing mitmproxy CA certficate into Firefox")
-
-        # browser_path is the exe, we want the folder
-        self.policies_dir = os.path.dirname(browser_path)
-        # on macosx we need to remove the last folders 'MacOS'
-        # and the policies json needs to go in ../Content/Resources/
-        if 'mac' in self.config['platform']:
-            self.policies_dir = os.path.join(self.policies_dir[:-6], "Resources")
-        # for all platforms the policies json goes in a 'distribution' dir
-        self.policies_dir = os.path.join(self.policies_dir, "distribution")
-
-        self.cert_path = DEFAULT_CERT_PATH
-        # for windows only
-        if mozinfo.os == 'win':
-            self.cert_path = self.cert_path.replace('\\', '\\\\')
-
-        if not os.path.exists(self.policies_dir):
-            LOG.info("creating folder: %s" % self.policies_dir)
-            os.makedirs(self.policies_dir)
-        else:
-            LOG.info("folder already exists: %s" % self.policies_dir)
-
-        self.write_policies_json(self.policies_dir,
-                                 policies_content=POLICIES_CONTENT_ON %
-                                 {'cert': self.cert_path,
-                                  'host': self.config['host']})
-
-        # cannot continue if failed to add CA cert to Firefox, need to check
-        if not self.is_mitmproxy_cert_installed_desktop():
-            LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox desktop')
-            self.stop_mitmproxy_playback()
-            sys.exit()
-
-    def install_mitmproxy_cert_android(self, mitmproxy_proc, browser_path):
-        """Install the CA certificate generated by mitmproxy, into geckoview android
-        1. Get the `certutil` tool.
-        2. Create an NSS certificate database in the geckoview browser profile dir.
-           `certutil -N -d sql:<path to profile> --empty-password`
-        3. Import the mitmproxy certificate into the database.
-           `certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>`
-        """
-        # get the certutil tool
-        if self.config.get("obj_path", None) is not None:
-            # when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
-            self.certutil = os.path.join(self.config['obj_path'], 'dist', 'bin')
-        else:
-            # in production it is already downloaded on the host automation machines via hostutils
-            # self.certutil = TODO
-            LOG.info("TODO: where is the path in production to certutil/hostutils?")
-
-        bin_suffix = mozinfo.info.get('bin_suffix', '')
-        self.certutil = os.path.join(self.certutil, "certutil" + bin_suffix)
-
-        if os.path.isfile(self.certutil):
-            LOG.info("certutil is found at: %s" % self.certutil)
-        else:
-            LOG.critical("unable to find certutil at %s" % self.certutil)
-
-        # DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e.
-        # /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer
-        self.local_cert_path = DEFAULT_CERT_PATH
-
-        # create cert db
-        param1 = "sql:%s/" % self.config['local_profile_dir']
-        command = [self.certutil, '-N', '-d', param1, '--empty-password']
-
-        LOG.info("creating nss cert database using command: %s" % ' '.join(command))
-        cmd_proc = subprocess.Popen(command, env=os.environ.copy())
-        time.sleep(3)
-        cmd_terminated = cmd_proc.poll()
-        if cmd_terminated is None:  # None value indicates process hasn't terminated
-            LOG.critical("nss cert db creation command failed to complete")
-
-        # import mitmproxy cert into the db
-        command = [self.certutil, '-A', '-d', param1, '-n',
-                   'mitmproxy-cert', '-t', 'TC,,', '-a', '-i', self.local_cert_path]
-
-        LOG.info("importing mitmproxy cert into db using command: %s" % ' '.join(command))
-        cmd_proc = subprocess.Popen(command, env=os.environ.copy())
-        time.sleep(3)
-        cmd_terminated = cmd_proc.poll()
-        if cmd_terminated is None:  # None value indicates process hasn't terminated
-            LOG.critical("command to import mitmproxy cert into cert db failed to complete")
-
-        # cannot continue if failed to add CA cert to Firefox, need to check
-        # if not self.is_mitmproxy_cert_installed_android():
-        #    LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox')
-        #    self.stop_mitmproxy_playback()
-        #    sys.exit()
-
-    def write_policies_json(self, location, policies_content):
-        policies_file = os.path.join(location, "policies.json")
-        LOG.info("writing: %s" % policies_file)
-
-        with open(policies_file, 'w') as fd:
-            fd.write(policies_content)
-
-    def read_policies_json(self, location):
-        policies_file = os.path.join(location, "policies.json")
-        LOG.info("reading: %s" % policies_file)
-
-        with open(policies_file, 'r') as fd:
-            return fd.read()
-
-    def is_mitmproxy_cert_installed_desktop(self):
-        """Verify mitmxproy CA cert was added to Firefox"""
-        try:
-            # read autoconfig file, confirm mitmproxy cert is in there
-            contents = self.read_policies_json(self.policies_dir)
-            LOG.info("Firefox policies file contents:")
-            LOG.info(contents)
-            if (POLICIES_CONTENT_ON % {
-                    'cert': self.cert_path,
-                    'host': self.config['host']}) in contents:
-                LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
-            else:
-
-                return False
-        except Exception as e:
-            LOG.info("failed to read Firefox policies file, exeption: %s" % e)
-            return False
-        return True
-
-    def is_mitmproxy_cert_installed_android(self):
-        """Verify mitmxproy CA cert was added to Firefox"""
-        LOG.info("* TODO: verify cert is installed on android")
-        return False
-
     def start_mitmproxy_playback(self,
                                  mitmdump_path,
                                  mitmproxy_recording_path,
                                  mitmproxy_recordings_list,
                                  browser_path):
         """Startup mitmproxy and replay the specified flow file"""
 
         LOG.info("mitmdump path: %s" % mitmdump_path)
@@ -401,46 +224,304 @@ class Mitmproxy(Playback, Python3Virtual
 
         command = [mitmdump_path, '-k', '-q', '-s', param2]
 
         LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
         LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command))
         # to turn off mitmproxy log output, use these params for Popen:
         # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
         mitmproxy_proc = subprocess.Popen(command, env=env)
-        time.sleep(10)
+        time.sleep(MITMDUMP_SLEEP)
         data = mitmproxy_proc.poll()
         if data is None:  # None value indicates process hasn't terminated
             LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid)
             return mitmproxy_proc
         # cannot continue as we won't be able to playback the pages
         LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data)
         sys.exit()
 
     def stop_mitmproxy_playback(self):
         """Stop the mitproxy server playback"""
         mitmproxy_proc = self.mitmproxy_proc
         LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid)
         if mozinfo.os == 'win':
             mitmproxy_proc.kill()
         else:
             mitmproxy_proc.terminate()
-        time.sleep(10)
+        time.sleep(MITMDUMP_SLEEP)
         status = mitmproxy_proc.poll()
         if status is None:  # None value indicates process hasn't terminated
             # I *think* we can still continue, as process will be automatically
             # killed anyway when mozharness is done (?) if not, we won't be able
             # to startup mitmxproy next time if it is already running
             LOG.error("Failed to kill the mitmproxy playback process")
             LOG.info(str(status))
         else:
             LOG.info("Successfully killed the mitmproxy playback process")
 
+
+class MitmproxyDesktop(Mitmproxy):
+
+    def __init__(self, config):
+        Mitmproxy.__init__(self, config)
+
+    def setup(self):
+        """For Firefox we need to install the generated mitmproxy CA cert. For Chromium this is
+        not necessary as it will be started with the --ignore-certificate-errors cmd line arg"""
+        if self.config['app'] == "firefox":
+            # install the generated CA certificate into Firefox desktop
+            self.install_mitmproxy_cert(self.mitmproxy_proc,
+                                        self.browser_path)
+        else:
+            return
+
+    def install_mitmproxy_cert(self, mitmproxy_proc, browser_path):
+        """Install the CA certificate generated by mitmproxy, into Firefox
+        1. Create a dir called 'distribution' in the same directory as the Firefox executable
+        2. Create the policies.json file inside that folder; which points to the certificate
+           location, and turns on the the browser proxy settings
+        """
+        LOG.info("Installing mitmproxy CA certficate into Firefox")
+
+        # browser_path is the exe, we want the folder
+        self.policies_dir = os.path.dirname(browser_path)
+        # on macosx we need to remove the last folders 'MacOS'
+        # and the policies json needs to go in ../Content/Resources/
+        if 'mac' in self.config['platform']:
+            self.policies_dir = os.path.join(self.policies_dir[:-6], "Resources")
+        # for all platforms the policies json goes in a 'distribution' dir
+        self.policies_dir = os.path.join(self.policies_dir, "distribution")
+
+        self.cert_path = DEFAULT_CERT_PATH
+        # for windows only
+        if mozinfo.os == 'win':
+            self.cert_path = self.cert_path.replace('\\', '\\\\')
+
+        if not os.path.exists(self.policies_dir):
+            LOG.info("creating folder: %s" % self.policies_dir)
+            os.makedirs(self.policies_dir)
+        else:
+            LOG.info("folder already exists: %s" % self.policies_dir)
+
+        self.write_policies_json(self.policies_dir,
+                                 policies_content=POLICIES_CONTENT_ON %
+                                 {'cert': self.cert_path,
+                                  'host': self.config['host']})
+
+        # cannot continue if failed to add CA cert to Firefox, need to check
+        if not self.is_mitmproxy_cert_installed():
+            LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox desktop')
+            self.stop_mitmproxy_playback()
+            sys.exit()
+
+    def write_policies_json(self, location, policies_content):
+        policies_file = os.path.join(location, "policies.json")
+        LOG.info("writing: %s" % policies_file)
+
+        with open(policies_file, 'w') as fd:
+            fd.write(policies_content)
+
+    def read_policies_json(self, location):
+        policies_file = os.path.join(location, "policies.json")
+        LOG.info("reading: %s" % policies_file)
+
+        with open(policies_file, 'r') as fd:
+            return fd.read()
+
+    def is_mitmproxy_cert_installed(self):
+        """Verify mitmxproy CA cert was added to Firefox"""
+        try:
+            # read autoconfig file, confirm mitmproxy cert is in there
+            contents = self.read_policies_json(self.policies_dir)
+            LOG.info("Firefox policies file contents:")
+            LOG.info(contents)
+            if (POLICIES_CONTENT_ON % {
+                    'cert': self.cert_path,
+                    'host': self.config['host']}) in contents:
+                LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
+            else:
+
+                return False
+        except Exception as e:
+            LOG.info("failed to read Firefox policies file, exeption: %s" % e)
+            return False
+        return True
+
+    def stop(self):
+        self.stop_mitmproxy_playback()
+        self.turn_off_browser_proxy()
+        return
+
     def turn_off_browser_proxy(self):
-        """Turn off the browser proxy that was used for mitmproxy playback"""
-        # in firefox we need to change the autoconfig files to revert
-        # the proxy; for google chromium the proxy was setup on the cmd line
-        # so nothing is required here
+        """Turn off the browser proxy that was used for mitmproxy playback. In Firefox
+        we need to change the autoconfig files to revert the proxy; for Chromium the proxy
+        was setup on the cmd line, so nothing is required here."""
         if self.config['app'] == "firefox":
             LOG.info("Turning off the browser proxy")
 
             self.write_policies_json(self.policies_dir,
                                      policies_content=POLICIES_CONTENT_OFF)
+
+
+class MitmproxyAndroid(Mitmproxy):
+
+    def __init__(self, config, android_device):
+        Mitmproxy.__init__(self, config)
+        self.android_device = android_device
+
+    def setup(self):
+        """For geckoview we need to install the generated mitmproxy CA cert"""
+        if self.config['app'] == "geckoview":
+            # install the generated CA certificate into android geckoview
+            self.install_mitmproxy_cert(self.mitmproxy_proc,
+                                        self.browser_path)
+        else:
+            return
+
+    def install_mitmproxy_cert(self, mitmproxy_proc, browser_path):
+        """Install the CA certificate generated by mitmproxy, into geckoview android
+        If running locally:
+        1. Will use the `certutil` tool from the local Firefox desktop build
+
+        If running in production:
+        1. Get the tooltools manifest file for downloading hostutils (contains certutil)
+        2. Get the `certutil` tool by downloading hostutils using the tooltool manifest
+
+        Then, both locally and in production:
+        1. Create an NSS certificate database in the geckoview browser profile dir, only
+           if it doesn't already exist. Use this certutil command:
+           `certutil -N -d sql:<path to profile> --empty-password`
+        2. Import the mitmproxy certificate into the database, i.e.:
+           `certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>`
+        """
+        self.CERTUTIL_SLEEP = 10
+        if self.config['run_local']:
+            # when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
+            self.certutil = os.path.join(self.config['obj_path'], 'dist', 'bin')
+        else:
+            # must download certutil inside hostutils via tooltool; use this manifest:
+            # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
+            # after it will be found here inside the worker/bitbar container:
+            # /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64
+            LOG.info("downloading certutil binary (hostutils)")
+
+            # get path to the hostutils tooltool manifest; was set earlier in
+            # mozharness/configs/raptor/android_hw_config.py, to the path i.e.
+            # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
+            # the bitbar container is always linux64
+            if os.environ.get('GECKO_HEAD_REPOSITORY', None) is None:
+                LOG.critical('Abort: unable to get GECKO_HEAD_REPOSITORY')
+                raise
+
+            if os.environ.get('GECKO_HEAD_REV', None) is None:
+                LOG.critical('Abort: unable to get GECKO_HEAD_REV')
+                raise
+
+            if os.environ.get('HOSTUTILS_MANIFEST_PATH', None) is not None:
+                manifest_url = os.path.join(os.environ['GECKO_HEAD_REPOSITORY'],
+                                            "raw-file",
+                                            os.environ['GECKO_HEAD_REV'],
+                                            os.environ['HOSTUTILS_MANIFEST_PATH'])
+            else:
+                LOG.critical("Abort: unable to get HOSTUTILS_MANIFEST_PATH!")
+                raise
+
+            # first need to download the hostutils tooltool manifest file itself
+            _dest = os.path.join(self.raptor_dir, 'hostutils.manifest')
+            have_manifest = download_file_from_url(manifest_url, _dest)
+            if not have_manifest:
+                LOG.critical('failed to download the hostutils tooltool manifest')
+                raise
+
+            # now use the manifest to download hostutils so we can get certutil
+            tooltool_download(_dest, self.config['run_local'], self.raptor_dir)
+
+            # the production bitbar container host is always linux
+            self.certutil = os.path.join(self.raptor_dir, 'host-utils-66.0a1.en-US.linux-x86_64')
+
+            # must add hostutils/certutil to the path
+            os.environ['LD_LIBRARY_PATH'] = self.certutil
+
+        bin_suffix = mozinfo.info.get('bin_suffix', '')
+        self.certutil = os.path.join(self.certutil, "certutil" + bin_suffix)
+
+        if os.path.isfile(self.certutil):
+            LOG.info("certutil is found at: %s" % self.certutil)
+        else:
+            LOG.critical("unable to find certutil at %s" % self.certutil)
+            raise
+
+        # DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e.
+        # /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer
+        self.local_cert_path = DEFAULT_CERT_PATH
+
+        # check if the nss ca cert db already exists in the device profile
+        LOG.info("checking if the nss cert db already exists in the android browser profile")
+        param1 = "sql:%s/" % self.config['local_profile_dir']
+        command = [self.certutil, '-d', param1, '-L']
+
+        try:
+            subprocess.check_output(command, env=os.environ.copy())
+            LOG.info("the nss cert db already exists")
+            cert_db_exists = True
+        except subprocess.CalledProcessError:
+            # this means the nss cert db doesn't exist yet
+            LOG.info("nss cert db doesn't exist yet")
+            cert_db_exists = False
+
+        # try a forced pause between certutil cmds; possibly reduce later
+        time.sleep(self.CERTUTIL_SLEEP)
+
+        if not cert_db_exists:
+            # create cert db if it doesn't already exist; it may exist already
+            # if a previous pageload test ran in the same test suite
+            param1 = "sql:%s/" % self.config['local_profile_dir']
+            command = [self.certutil, '-N', '-v', '-d', param1, '--empty-password']
+
+            LOG.info("creating nss cert database using command: %s" % ' '.join(command))
+            cmd_proc = subprocess.Popen(command, env=os.environ.copy())
+            time.sleep(self.CERTUTIL_SLEEP)
+            cmd_terminated = cmd_proc.poll()
+            if cmd_terminated is None:  # None value indicates process hasn't terminated
+                LOG.critical("nss cert db creation command failed to complete")
+                raise
+
+        # import mitmproxy cert into the db
+        command = [self.certutil, '-A', '-d', param1, '-n',
+                   'mitmproxy-cert', '-t', 'TC,,', '-a', '-i', self.local_cert_path]
+
+        LOG.info("importing mitmproxy cert into db using command: %s" % ' '.join(command))
+        cmd_proc = subprocess.Popen(command, env=os.environ.copy())
+        time.sleep(self.CERTUTIL_SLEEP)
+        cmd_terminated = cmd_proc.poll()
+        if cmd_terminated is None:  # None value indicates process hasn't terminated
+            LOG.critical("command to import mitmproxy cert into cert db failed to complete")
+
+        # cannot continue if failed to add CA cert to Firefox, need to check
+        if not self.is_mitmproxy_cert_installed():
+            LOG.error("Aborting: failed to install mitmproxy CA cert into Firefox")
+            self.stop_mitmproxy_playback()
+            sys.exit()
+
+    def is_mitmproxy_cert_installed(self):
+        """Verify mitmxproy CA cert was added to Firefox on android"""
+        LOG.info("verifying that the mitmproxy ca cert is installed on android")
+
+        # list the certifcates that are in the nss cert db (inside the browser profile dir)
+        LOG.info("getting the list of certs in the nss cert db in the android browser profile")
+        param1 = "sql:%s/" % self.config['local_profile_dir']
+        command = [self.certutil, '-d', param1, '-L']
+
+        try:
+            cmd_output = subprocess.check_output(command, env=os.environ.copy())
+
+        except subprocess.CalledProcessError:
+            # cmd itself failed
+            LOG.critical("certutil command failed")
+            raise
+
+        # check output from the certutil command, see if 'mitmproxy-cert' is listed
+        time.sleep(self.CERTUTIL_SLEEP)
+        LOG.info(cmd_output)
+        if 'mitmproxy-cert' in cmd_output:
+            LOG.info("verfied the mitmproxy-cert is installed in the nss cert db on android")
+            return True
+        return False
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -1,20 +1,23 @@
-# raptor pageload tests
+# raptor pageload tests desktop
 [include:tests/raptor-tp6-1.ini]
 [include:tests/raptor-tp6-2.ini]
 [include:tests/raptor-tp6-3.ini]
 [include:tests/raptor-tp6-4.ini]
 [include:tests/raptor-tp6-5.ini]
 [include:tests/raptor-tp6-6.ini]
 [include:tests/raptor-tp6-7.ini]
 [include:tests/raptor-tp6-8.ini]
 [include:tests/raptor-tp6-9.ini]
 [include:tests/raptor-tp6-10.ini]
 
+# raptor pageload tests mobile
+[include:tests/raptor-tp6m-1.ini]
+
 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-speedometer.ini]
 [include:tests/raptor-stylebench.ini]
 [include:tests/raptor-sunspider.ini]
 [include:tests/raptor-unity-webgl.ini]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -175,34 +175,35 @@ class Raptor(object):
         self.log.info("starting raptor test: %s" % test['name'])
         self.log.info("test settings: %s" % str(test))
         self.log.info("raptor config: %s" % str(self.config))
 
         # benchmark-type tests require the benchmark test to be served out
         if test.get('type') == "benchmark":
             self.benchmark = Benchmark(self.config, test)
             benchmark_port = int(self.benchmark.port)
+
+            # for android we must make the benchmarks server available to the device
+            if self.config['app'] in ['geckoview', 'fennec'] and \
+                    self.config['host'] in ('localhost', '127.0.0.1'):
+                self.log.info("making the raptor benchmarks server port available to device")
+                _tcp_port = "tcp:%s" % benchmark_port
+                self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
+
         else:
             benchmark_port = 0
 
         gen_test_config(self.config['app'],
                         test['name'],
                         self.control_server.port,
                         self.post_startup_delay,
                         host=self.config['host'],
                         b_port=benchmark_port,
                         debug_mode=1 if self.debug_mode else 0)
 
-        # for android we must make the benchmarks server available to the device
-        if self.config['app'] in ['geckoview', 'fennec'] and \
-                self.config['host'] in ('localhost', '127.0.0.1'):
-            self.log.info("making the raptor benchmarks server port available to device")
-            _tcp_port = "tcp:%s" % benchmark_port
-            self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
-
         # must intall raptor addon each time because we dynamically update some content
         # note: for chrome the addon is just a list of paths that ultimately are added
         # to the chromium command line '--load-extension' argument
         raptor_webext = os.path.join(webext_dir, 'raptor')
         self.log.info("installing webext %s" % raptor_webext)
         self.profile.addons.install(raptor_webext)
 
         # add test specific preferences
@@ -226,18 +227,18 @@ class Raptor(object):
                 self.log.info("deleting existing device raptor dir: %s" % self.device_raptor_dir)
                 self.device.rm(self.device_raptor_dir, recursive=True)
             self.log.info("creating raptor folder on sdcard: %s" % self.device_raptor_dir)
             self.device.mkdir(self.device_raptor_dir)
             self.device.chmod(self.device_raptor_dir, recursive=True)
 
         # some tests require tools to playback the test pages
         if test.get('playback', None) is not None:
+            # startup the playback tool
             self.get_playback_config(test)
-            # startup the playback tool
             self.playback = get_playback(self.config, self.device)
 
             # for android we must make the playback server available to the device
             if self.config['app'] == "geckoview" and self.config['host'] \
                     in ('localhost', '127.0.0.1'):
                 self.log.info("making the raptor playback server port available to device")
                 _tcp_port = "tcp:8080"
                 self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-1.ini
@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-1
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+page_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+
+[raptor-tp6m-amazon-geckoview]
+apps = geckoview
+test_url = https://www.amazon.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon.manifest
+playback_recordings = android-amazon.mp
+measure = fnbpaint, dcf, ttfi, loadtime
+
+[raptor-tp6m-facebook-geckoview]
+apps = geckoview
+test_url = https://m.facebook.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook.manifest
+playback_recordings = android-facebook.mp
+measure = fnbpaint, dcf, ttfi, loadtime
+
+[raptor-tp6m-google-geckoview]
+apps = geckoview
+test_url = https://www.google.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google.manifest
+playback_recordings = android-google.mp
+measure = fnbpaint, dcf, ttfi, loadtime
+
+[raptor-tp6m-youtube-geckoview]
+apps = geckoview
+test_url = https://www.youtube.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube.manifest
+playback_recordings = android-youtube.mp
+measure = fnbpaint, dcf, loadtime
--- a/testing/raptor/raptor/utils.py
+++ b/testing/raptor/raptor/utils.py
@@ -1,24 +1,48 @@
-'''Utility functions for Raptor'''
+"""Utility functions for Raptor"""
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
+import os
+import signal
+import sys
+import urllib
+
 from mozlog import get_proxy_logger
+from mozprocess import ProcessHandler
 
 LOG = get_proxy_logger(component="raptor-utils")
+here = os.path.dirname(os.path.realpath(__file__))
+
+if os.environ.get('SCRIPTSPATH', None) is not None:
+    # in production it is env SCRIPTS_PATH
+    mozharness_dir = os.environ['SCRIPTSPATH']
+else:
+    # locally it's in source tree
+    mozharness_dir = os.path.join(here, '../../mozharness')
+sys.path.insert(0, mozharness_dir)
+
+external_tools_path = os.environ.get('EXTERNALTOOLSPATH', None)
+
+if external_tools_path is not None:
+    # running in production via mozharness
+    TOOLTOOL_PATH = os.path.join(external_tools_path, 'tooltool.py')
+else:
+    # running locally via mach
+    TOOLTOOL_PATH = os.path.join(mozharness_dir, 'external_tools', 'tooltool.py')
 
 
 def transform_platform(str_to_transform, config_platform, config_processor=None):
-    # transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest'
-    # transforms to 'mitmproxy-rel-bin-osx.manifest'
-    # also will transform '{x64}' if needed for 64 bit / win 10
+    """Transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest'
+    transforms to 'mitmproxy-rel-bin-osx.manifest'.
+    Also transform '{x64}' if needed for 64 bit / win 10"""
     if '{platform}' not in str_to_transform and '{x64}' not in str_to_transform:
         return str_to_transform
 
     if 'win' in config_platform:
         platform_id = 'win'
     elif config_platform == 'mac':
         platform_id = 'osx'
     else:
@@ -29,8 +53,59 @@ def transform_platform(str_to_transform,
 
     if '{x64}' in str_to_transform and config_processor is not None:
         if 'x86_64' in config_processor:
             str_to_transform = str_to_transform.replace('{x64}', '_x64')
         else:
             str_to_transform = str_to_transform.replace('{x64}', '')
 
     return str_to_transform
+
+
+def tooltool_download(manifest, run_local, raptor_dir):
+    """Download a file from tooltool using the provided tooltool manifest"""
+    def outputHandler(line):
+        LOG.info(line)
+    if run_local:
+        command = [sys.executable,
+                   TOOLTOOL_PATH,
+                   'fetch',
+                   '-o',
+                   '-m', manifest]
+    else:
+        # we want to use the tooltool cache in production
+        if os.environ.get('TOOLTOOLCACHE', None) is not None:
+            _cache = os.environ['TOOLTOOLCACHE']
+        else:
+            _cache = "/builds/tooltool_cache"
+
+        command = [sys.executable,
+                   TOOLTOOL_PATH,
+                   'fetch',
+                   '-o',
+                   '-m', manifest,
+                   '-c',
+                   _cache]
+
+    proc = ProcessHandler(
+        command, processOutputLine=outputHandler, storeOutput=False,
+        cwd=raptor_dir)
+
+    proc.run()
+
+    try:
+        proc.wait()
+    except Exception:
+        if proc.poll() is None:
+            proc.kill(signal.SIGTERM)
+
+
+def download_file_from_url(url, local_dest):
+    """Receive a file in a URL and download it, i.e. for the hostutils tooltool manifest
+    the url received would be formatted like this:
+    https://hg.mozilla.org/try/raw-file/acb5abf52c04da7d4548fa13bd6c6848a90c32b8/testing/
+      config/tooltool-manifests/linux64/hostutils.manifest"""
+    if os.path.exists(local_dest):
+        LOG.info("file already exists at: %s" % local_dest)
+        return True
+    LOG.info("downloading: %s to %s" % (url, local_dest))
+    _file, _headers = urllib.urlretrieve(url, local_dest)
+    return os.path.exists(local_dest)
--- a/testing/raptor/requirements.txt
+++ b/testing/raptor/requirements.txt
@@ -1,5 +1,6 @@
+mozcrash ~= 1.0
 mozrunner ~= 7.0
 mozprofile ~= 2.1
 manifestparser >= 1.1
 wptserve ~= 1.4.0
 mozdevice >= 1.1.6
--- a/testing/raptor/test/test_playback.py
+++ b/testing/raptor/test/test_playback.py
@@ -4,35 +4,41 @@ import os
 
 import mozinfo
 import mozunit
 
 from mozlog.structuredlog import set_default_logger, StructuredLogger
 
 set_default_logger(StructuredLogger('test_playback'))
 
-from raptor.playback import get_playback, Mitmproxy
+from raptor.playback import get_playback, MitmproxyDesktop
 
 config = {}
 
+run_local = True
+if os.environ.get('TOOLTOOLCACHE', None) is None:
+    run_local = False
+
 
 def test_get_playback(get_binary):
     config['platform'] = mozinfo.os
     if 'win' in config['platform']:
         # this test is not yet supported on windows
         assert True
         return
     config['obj_path'] = os.path.dirname(get_binary('firefox'))
     config['playback_tool'] = 'mitmproxy'
     config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
     config['playback_pageset_manifest'] = 'mitmproxy-playback-set.manifest'
     config['playback_recordings'] = 'mitmproxy-recording-amazon.mp'
     config['binary'] = get_binary('firefox')
+    config['run_local'] = run_local
+
     playback = get_playback(config)
-    assert isinstance(playback, Mitmproxy)
+    assert isinstance(playback, MitmproxyDesktop)
     playback.stop()
 
 
 def test_get_unsupported_playback():
     config['playback_tool'] = 'unsupported'
     playback = get_playback(config)
     assert playback is None