Backed out changeset a5ad662d5bbb (bug 1520447) for android build bustages. CLOSED TREE
authorCosmin Sabou <csabou@mozilla.com>
Wed, 13 Feb 2019 17:04:19 +0200
changeset 458898 23a24796db2b
parent 458897 97dd82032154
child 458899 6540e3aba359
push id35551
push usershindli@mozilla.com
push dateWed, 13 Feb 2019 21:34:09 +0000
treeherdermozilla-central@08f794a4928e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1520447
milestone67.0a1
backs outa5ad662d5bbb
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset a5ad662d5bbb (bug 1520447) for android build bustages. CLOSED TREE
testing/raptor/mach_commands.py
testing/raptor/raptor/manifest.py
testing/raptor/raptor/playback/mitmproxy-rel-bin-win.manifest
testing/raptor/raptor/playback/mitmproxy.py
testing/raptor/raptor/playback/python3.manifest
testing/raptor/raptor/playback/python3_x64.manifest
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/raptor-tp6-1.ini
testing/raptor/raptor/tests/raptor-tp6-10.ini
testing/raptor/raptor/tests/raptor-tp6-2.ini
testing/raptor/raptor/tests/raptor-tp6-3.ini
testing/raptor/raptor/tests/raptor-tp6-4.ini
testing/raptor/raptor/tests/raptor-tp6-5.ini
testing/raptor/raptor/tests/raptor-tp6-6.ini
testing/raptor/raptor/tests/raptor-tp6-7.ini
testing/raptor/raptor/tests/raptor-tp6-8.ini
testing/raptor/raptor/tests/raptor-tp6-9.ini
testing/raptor/raptor/tests/raptor-tp6m-1.ini
testing/raptor/test/test_manifest.py
--- a/testing/raptor/mach_commands.py
+++ b/testing/raptor/mach_commands.py
@@ -127,16 +127,20 @@ class RaptorRunner(MozbuildObject):
             'base_work_dir': self.mozharness_dir,
             'exes': {
                 'python': self.python_interp,
                 'virtualenv': [self.python_interp, self.virtualenv_script],
             },
             'title': socket.gethostname(),
             'default_actions': default_actions,
             'raptor_cmd_line_args': self.raptor_args,
+            'python3_manifest': {
+                'win32': 'python3.manifest',
+                'win64': 'python3_x64.manifest',
+            },
             'host': self.host,
             'power_test': self.power_test,
             'is_release_build': self.is_release_build,
         }
 
     def make_args(self):
         self.args = {
             'config': {},
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -14,17 +14,17 @@ here = os.path.abspath(os.path.dirname(_
 raptor_ini = os.path.join(here, 'raptor.ini')
 tests_dir = os.path.join(here, 'tests')
 LOG = get_proxy_logger(component="raptor-manifest")
 
 required_settings = ['apps', 'type', 'page_cycles', 'test_url', 'measure',
                      'unit', 'lower_is_better', 'alert_threshold']
 
 playback_settings = ['playback_binary_manifest', 'playback_pageset_manifest',
-                     'playback_recordings']
+                     'playback_recordings', 'python3_win_manifest']
 
 
 def filter_app(tests, values):
     for test in tests:
         if values["app"] in test['apps']:
             yield test
 
 
deleted file mode 100644
--- a/testing/raptor/raptor/playback/mitmproxy-rel-bin-win.manifest
+++ /dev/null
@@ -1,10 +0,0 @@
-[
-    {
-    "size": 13576786,
-    "visibility": "public",
-    "digest": "5d471e470369381f130de28a3c54db27c3724e1e9269c510a593d61c4cf41713c9424da62e46ae98fd1decce00ecca876209ed059487f5a02882ba16a50daed1",
-    "algorithm": "sha512",
-    "filename": "mitmdump-win-2.0.2.zip",
-    "unpack": true
-    }
-]
\ No newline at end of file
--- a/testing/raptor/raptor/playback/mitmproxy.py
+++ b/testing/raptor/raptor/playback/mitmproxy.py
@@ -7,32 +7,36 @@ from __future__ import absolute_import
 import os
 import subprocess
 import sys
 import time
 
 import mozinfo
 
 from mozlog import get_proxy_logger
-from mozprocess import ProcessHandler
 
 from .base import Playback
 
 here = os.path.dirname(os.path.realpath(__file__))
 LOG = get_proxy_logger(component='raptor-mitmproxy')
 
 # needed so unit tests can find their imports
 if os.environ.get('SCRIPTSPATH', None) is not None:
     # in production it is env SCRIPTS_PATH
     mozharness_dir = os.environ['SCRIPTSPATH']
 else:
     # locally it's in source tree
     mozharness_dir = os.path.join(here, '../../../mozharness')
 sys.path.insert(0, mozharness_dir)
 
+# required for using a python3 virtualenv on win for mitmproxy
+from mozharness.base.python import Python3Virtualenv
+from mozharness.mozilla.testing.testbase import TestingMixin
+from mozharness.base.vcs.vcsbase import MercurialScript
+
 raptor_dir = os.path.join(here, '..')
 sys.path.insert(0, raptor_dir)
 
 from utils import transform_platform, tooltool_download, download_file_from_url
 
 # path for mitmproxy certificate, generated auto after mitmdump is started
 # on local machine it is 'HOME', however it is different on production machines
 try:
@@ -70,17 +74,17 @@ POLICIES_CONTENT_OFF = '''{
     "Proxy": {
       "Mode": "none",
       "Locked": false
     }
   }
 }'''
 
 
-class Mitmproxy(Playback):
+class Mitmproxy(Playback, Python3Virtualenv, TestingMixin, MercurialScript):
 
     def __init__(self, config):
         self.config = config
         self.mitmproxy_proc = None
         self.mitmdump_path = None
         self.recordings = config.get('playback_recordings', None)
         self.browser_path = config.get('binary', None)
 
@@ -97,41 +101,85 @@ class Mitmproxy(Playback):
         # add raptor to raptor_dir
         self.raptor_dir = os.path.join(self.raptor_dir, "testing", "raptor")
         self.recordings_path = self.raptor_dir
         LOG.info("raptor_dir used for mitmproxy downloads and exe files: %s" % self.raptor_dir)
 
         # go ahead and download and setup mitmproxy
         self.download()
 
+        # on windows we must use a python3 virtualen for mitmproxy
+        if 'win' in self.config['platform']:
+            self.setup_py3_virtualenv()
+
         # mitmproxy must be started before setup, so that the CA cert is available
         self.start()
         self.setup()
 
     def download(self):
         """Download and unpack mitmproxy binary and pageset using tooltool"""
         if not os.path.exists(self.raptor_dir):
             os.makedirs(self.raptor_dir)
 
-        LOG.info("downloading mitmproxy binary")
-        _manifest = os.path.join(here, self.config['playback_binary_manifest'])
-        transformed_manifest = transform_platform(_manifest, self.config['platform'])
-        tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
+        if 'win' in self.config['platform']:
+            # on windows we need a python3 environment and use our own package from tooltool
+            self.py3_path = self.fetch_python3()
+            LOG.info("python3 path is: %s" % self.py3_path)
+        else:
+            # on osx and linux we use pre-built binaries
+            LOG.info("downloading mitmproxy binary")
+            _manifest = os.path.join(here, self.config['playback_binary_manifest'])
+            transformed_manifest = transform_platform(_manifest, self.config['platform'])
+            tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
 
         # we use one pageset for all platforms
         LOG.info("downloading mitmproxy pageset")
         _manifest = os.path.join(here, self.config['playback_pageset_manifest'])
         transformed_manifest = transform_platform(_manifest, self.config['platform'])
         tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
         return
 
+    def fetch_python3(self):
+        """Mitmproxy on windows needs Python 3.x"""
+        python3_path = os.path.join(self.raptor_dir, 'python3.6', 'python')
+        if not os.path.exists(os.path.dirname(python3_path)):
+            _manifest = os.path.join(here, self.config['python3_win_manifest'])
+            transformed_manifest = transform_platform(_manifest, self.config['platform'],
+                                                      self.config['processor'])
+            LOG.info("downloading py3 package for mitmproxy windows: %s" % transformed_manifest)
+            tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir)
+        cmd = [python3_path, '--version']
+        # just want python3 ver printed in production log
+        subprocess.Popen(cmd, env=os.environ.copy())
+        return python3_path
+
+    def setup_py3_virtualenv(self):
+        """Mitmproxy on windows needs Python 3.x; set up a separate py 3.x env here"""
+        LOG.info("Setting up python 3.x virtualenv, required for mitmproxy on windows")
+        # these next two are required for py3_venv_configuration
+        self.abs_dirs = {'base_work_dir': mozharness_dir}
+        self.log_obj = None
+        # now create the py3 venv
+        venv_path = os.path.join(self.raptor_dir, 'py3venv')
+        self.py3_venv_configuration(python_path=self.py3_path, venv_path=venv_path)
+        self.py3_create_venv()
+        self.py3_install_modules(["cffi==1.10.0"])
+        requirements = [os.path.join(here, "mitmproxy_requirements.txt")]
+        self.py3_install_requirement_files(requirements)
+        # add py3 executables path to system path
+        sys.path.insert(1, self.py3_path_to_executables())
+        # install mitmproxy itself
+        self.py3_install_modules(modules=['mitmproxy'])
+        self.mitmdump_path = os.path.join(self.py3_path_to_executables(), 'mitmdump')
+
     def start(self):
-        """Start playing back the mitmproxy recording."""
-
-        self.mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump')
+        """Start playing back the mitmproxy recording. If on windows, the mitmdump_path was
+        already set when creating py3 env"""
+        if self.mitmdump_path is None:
+            self.mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump')
 
         recordings_list = self.recordings.split()
         self.mitmproxy_proc = self.start_mitmproxy_playback(self.mitmdump_path,
                                                             self.recordings_path,
                                                             recordings_list,
                                                             self.browser_path)
         return
 
@@ -175,34 +223,34 @@ class Mitmproxy(Playback):
         env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
 
         command = [mitmdump_path, '-k', '-q', '-s', param2]
 
         LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
         LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command))
         # to turn off mitmproxy log output, use these params for Popen:
         # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
-        mitmproxy_proc = ProcessHandler(command, env=env)
-        mitmproxy_proc.run()
-
+        mitmproxy_proc = subprocess.Popen(command, env=env)
         time.sleep(MITMDUMP_SLEEP)
         data = mitmproxy_proc.poll()
         if data is None:  # None value indicates process hasn't terminated
             LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid)
             return mitmproxy_proc
         # cannot continue as we won't be able to playback the pages
         LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data)
         sys.exit()
 
     def stop_mitmproxy_playback(self):
         """Stop the mitproxy server playback"""
         mitmproxy_proc = self.mitmproxy_proc
         LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid)
-        mitmproxy_proc.kill()
-
+        if mozinfo.os == 'win':
+            mitmproxy_proc.kill()
+        else:
+            mitmproxy_proc.terminate()
         time.sleep(MITMDUMP_SLEEP)
         status = mitmproxy_proc.poll()
         if status is None:  # None value indicates process hasn't terminated
             # I *think* we can still continue, as process will be automatically
             # killed anyway when mozharness is done (?) if not, we won't be able
             # to startup mitmxproy next time if it is already running
             LOG.error("Failed to kill the mitmproxy playback process")
             LOG.info(str(status))
@@ -281,18 +329,18 @@ class MitmproxyDesktop(Mitmproxy):
     def is_mitmproxy_cert_installed(self):
         """Verify mitmxproy CA cert was added to Firefox"""
         try:
             # read autoconfig file, confirm mitmproxy cert is in there
             contents = self.read_policies_json(self.policies_dir)
             LOG.info("Firefox policies file contents:")
             LOG.info(contents)
             if (POLICIES_CONTENT_ON % {
-                'cert': self.cert_path,
-                'host': self.config['host']}) in contents:
+                    'cert': self.cert_path,
+                    'host': self.config['host']}) in contents:
                 LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
             else:
 
                 return False
         except Exception as e:
             LOG.info("failed to read Firefox policies file, exeption: %s" % e)
             return False
         return True
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/python3.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 15380470,
+    "visibility": "public",
+    "digest": "cd78b88d95b69bef99d7192b71dd34118700f44db0a0069a13bfd4943b131e8d7fdac83859f8ac15d873d4b329eef69d8d75d0a6746d06fdcfc5d06da0c9784c",
+    "algorithm": "sha512",
+    "unpack": true,
+    "filename": "python3.6.zip"
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/python3_x64.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 16026760,
+    "visibility": "public",
+    "digest": "379428e3955671213a245ccd9ccf6f9d17d368db68c02da8baed7be629f2691127cd3e3f86807b25e2098d9840083fdc07946ab1bed0c14db4a5b628a47ed9ef",
+    "algorithm": "sha512",
+    "unpack": true,
+    "filename": "python3.6.amd64.zip"
+  }
+]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -166,16 +166,17 @@ class Raptor(object):
         self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
         self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
         _key = 'playback_binary_zip_%s' % self.config['platform']
         self.config['playback_binary_zip'] = test.get(_key, None)
         self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
         _key = 'playback_pageset_zip_%s' % self.config['platform']
         self.config['playback_pageset_zip'] = test.get(_key, None)
         self.config['playback_recordings'] = test.get('playback_recordings', None)
+        self.config['python3_win_manifest'] = test.get('python3_win_manifest', None)
 
     def run_test(self, test, timeout=None):
         self.log.info("starting raptor test: %s" % test['name'])
         self.log.info("test settings: %s" % str(test))
         self.log.info("raptor config: %s" % str(self.config))
 
         # benchmark-type tests require the benchmark test to be served out
         if test.get('type') == "benchmark":
--- a/testing/raptor/raptor/tests/raptor-tp6-1.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-1.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-1
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-10.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-10.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-10
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 60000
 gecko_profile_interval = 1
--- a/testing/raptor/raptor/tests/raptor-tp6-2.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-2.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-2
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-2.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 page_timeout = 60000
 gecko_profile_interval = 1
 gecko_profile_entries = 14000000
--- a/testing/raptor/raptor/tests/raptor-tp6-3.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-3.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-3
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-3.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-4.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-4.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-4
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-4.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-5.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-5.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-5
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-5.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-6.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-6.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-6
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-6.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-7.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-7.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-7
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-7.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 60000
--- a/testing/raptor/raptor/tests/raptor-tp6-8.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-8.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-8
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
--- a/testing/raptor/raptor/tests/raptor-tp6-9.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-9.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6-9
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
--- a/testing/raptor/raptor/tests/raptor-tp6m-1.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-1.ini
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6m-1
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy-android
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
 page_cycles = 15
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 page_timeout = 60000
 alert_on = fcp, loadtime
 
 [raptor-tp6m-amazon-geckoview]
--- a/testing/raptor/test/test_manifest.py
+++ b/testing/raptor/test/test_manifest.py
@@ -31,16 +31,17 @@ VALID_MANIFESTS = [{'apps': 'firefox',
                     'alert_on': 'fcp',
                     'unit': 'ms',
                     'lower_is_better': True,
                     'alert_threshold': 2.0,
                     'playback': 'mitmproxy',
                     'playback_binary_manifest': 'binary.manifest',
                     'playback_pageset_manifest': 'pageset.manifest',
                     'playback_recordings': 'recorded_site.mp',
+                    'python3_win_manifest': 'py3.manifest',
                     'manifest': 'valid_details_1'},
                    {'apps': 'chrome',
                     'type': 'benchmark',
                     'page_cycles': 5,
                     'test_url': 'http://www.test-url/goes/here',
                     'measure': 'fcp',
                     'unit': 'score',
                     'lower_is_better': False,
@@ -53,16 +54,17 @@ INVALID_MANIFESTS = [{'apps': 'firefox',
                       'test_url': 'http://www.test-url/goes/here',
                       'unit': 'ms',
                       'lower_is_better': True,
                       'alert_threshold': 2.0,
                       'playback': 'mitmproxy',
                       'playback_binary_manifest': 'binary.manifest',
                       'playback_pageset_manifest': 'pageset.manifest',
                       'playback_recordings': 'recorded_site.mp',
+                      'python3_win_manifest': 'py3.manifest',
                       'manifest': 'invalid_details_1'},
                      {'apps': 'chrome',
                       'type': 'pageload',
                       'page_cycles': 25,
                       'test_url': 'http://www.test-url/goes/here',
                       'measure': 'fnbpaint, fcp',
                       'unit': 'ms',
                       'lower_is_better': True,
@@ -77,16 +79,17 @@ INVALID_MANIFESTS = [{'apps': 'firefox',
                       'alert_on': 'nope',
                       'unit': 'ms',
                       'lower_is_better': True,
                       'alert_threshold': 2.0,
                       'playback': 'mitmproxy',
                       'playback_binary_manifest': 'binary.manifest',
                       'playback_pageset_manifest': 'pageset.manifest',
                       'playback_recordings': 'recorded_site.mp',
+                      'python3_win_manifest': 'py3.manifest',
                       'manifest': 'invalid_details_3'}]
 
 
 @pytest.mark.parametrize('app', ['firefox', 'chrome', 'geckoview'])
 def test_get_browser_test_list(app):
     test_list = get_browser_test_list(app)
     assert len(test_list) > 0