author | Cosmin Sabou <csabou@mozilla.com> |
Wed, 13 Feb 2019 17:04:19 +0200 | |
changeset 458898 | 23a24796db2b |
parent 458897 | 97dd82032154 |
child 458899 | 6540e3aba359 |
push id | 35551 |
push user | shindli@mozilla.com |
push date | Wed, 13 Feb 2019 21:34:09 +0000 |
treeherder | mozilla-central@08f794a4928e [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
bugs | 1520447 |
milestone | 67.0a1 |
backs out | a5ad662d5bbb |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/raptor/mach_commands.py +++ b/testing/raptor/mach_commands.py @@ -127,16 +127,20 @@ class RaptorRunner(MozbuildObject): 'base_work_dir': self.mozharness_dir, 'exes': { 'python': self.python_interp, 'virtualenv': [self.python_interp, self.virtualenv_script], }, 'title': socket.gethostname(), 'default_actions': default_actions, 'raptor_cmd_line_args': self.raptor_args, + 'python3_manifest': { + 'win32': 'python3.manifest', + 'win64': 'python3_x64.manifest', + }, 'host': self.host, 'power_test': self.power_test, 'is_release_build': self.is_release_build, } def make_args(self): self.args = { 'config': {},
--- a/testing/raptor/raptor/manifest.py +++ b/testing/raptor/raptor/manifest.py @@ -14,17 +14,17 @@ here = os.path.abspath(os.path.dirname(_ raptor_ini = os.path.join(here, 'raptor.ini') tests_dir = os.path.join(here, 'tests') LOG = get_proxy_logger(component="raptor-manifest") required_settings = ['apps', 'type', 'page_cycles', 'test_url', 'measure', 'unit', 'lower_is_better', 'alert_threshold'] playback_settings = ['playback_binary_manifest', 'playback_pageset_manifest', - 'playback_recordings'] + 'playback_recordings', 'python3_win_manifest'] def filter_app(tests, values): for test in tests: if values["app"] in test['apps']: yield test
deleted file mode 100644 --- a/testing/raptor/raptor/playback/mitmproxy-rel-bin-win.manifest +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "size": 13576786, - "visibility": "public", - "digest": "5d471e470369381f130de28a3c54db27c3724e1e9269c510a593d61c4cf41713c9424da62e46ae98fd1decce00ecca876209ed059487f5a02882ba16a50daed1", - "algorithm": "sha512", - "filename": "mitmdump-win-2.0.2.zip", - "unpack": true - } -] \ No newline at end of file
--- a/testing/raptor/raptor/playback/mitmproxy.py +++ b/testing/raptor/raptor/playback/mitmproxy.py @@ -7,32 +7,36 @@ from __future__ import absolute_import import os import subprocess import sys import time import mozinfo from mozlog import get_proxy_logger -from mozprocess import ProcessHandler from .base import Playback here = os.path.dirname(os.path.realpath(__file__)) LOG = get_proxy_logger(component='raptor-mitmproxy') # needed so unit tests can find their imports if os.environ.get('SCRIPTSPATH', None) is not None: # in production it is env SCRIPTS_PATH mozharness_dir = os.environ['SCRIPTSPATH'] else: # locally it's in source tree mozharness_dir = os.path.join(here, '../../../mozharness') sys.path.insert(0, mozharness_dir) +# required for using a python3 virtualenv on win for mitmproxy +from mozharness.base.python import Python3Virtualenv +from mozharness.mozilla.testing.testbase import TestingMixin +from mozharness.base.vcs.vcsbase import MercurialScript + raptor_dir = os.path.join(here, '..') sys.path.insert(0, raptor_dir) from utils import transform_platform, tooltool_download, download_file_from_url # path for mitmproxy certificate, generated auto after mitmdump is started # on local machine it is 'HOME', however it is different on production machines try: @@ -70,17 +74,17 @@ POLICIES_CONTENT_OFF = '''{ "Proxy": { "Mode": "none", "Locked": false } } }''' -class Mitmproxy(Playback): +class Mitmproxy(Playback, Python3Virtualenv, TestingMixin, MercurialScript): def __init__(self, config): self.config = config self.mitmproxy_proc = None self.mitmdump_path = None self.recordings = config.get('playback_recordings', None) self.browser_path = config.get('binary', None) @@ -97,41 +101,85 @@ class Mitmproxy(Playback): # add raptor to raptor_dir self.raptor_dir = os.path.join(self.raptor_dir, "testing", "raptor") self.recordings_path = self.raptor_dir LOG.info("raptor_dir used for mitmproxy downloads and exe files: %s" % self.raptor_dir) # go ahead and download and setup mitmproxy self.download() + # on windows we must use a python3 virtualen for mitmproxy + if 'win' in self.config['platform']: + self.setup_py3_virtualenv() + # mitmproxy must be started before setup, so that the CA cert is available self.start() self.setup() def download(self): """Download and unpack mitmproxy binary and pageset using tooltool""" if not os.path.exists(self.raptor_dir): os.makedirs(self.raptor_dir) - LOG.info("downloading mitmproxy binary") - _manifest = os.path.join(here, self.config['playback_binary_manifest']) - transformed_manifest = transform_platform(_manifest, self.config['platform']) - tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir) + if 'win' in self.config['platform']: + # on windows we need a python3 environment and use our own package from tooltool + self.py3_path = self.fetch_python3() + LOG.info("python3 path is: %s" % self.py3_path) + else: + # on osx and linux we use pre-built binaries + LOG.info("downloading mitmproxy binary") + _manifest = os.path.join(here, self.config['playback_binary_manifest']) + transformed_manifest = transform_platform(_manifest, self.config['platform']) + tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir) # we use one pageset for all platforms LOG.info("downloading mitmproxy pageset") _manifest = os.path.join(here, self.config['playback_pageset_manifest']) transformed_manifest = transform_platform(_manifest, self.config['platform']) tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir) return + def fetch_python3(self): + """Mitmproxy on windows needs Python 3.x""" + python3_path = os.path.join(self.raptor_dir, 'python3.6', 'python') + if not os.path.exists(os.path.dirname(python3_path)): + _manifest = os.path.join(here, self.config['python3_win_manifest']) + transformed_manifest = transform_platform(_manifest, self.config['platform'], + self.config['processor']) + LOG.info("downloading py3 package for mitmproxy windows: %s" % transformed_manifest) + tooltool_download(transformed_manifest, self.config['run_local'], self.raptor_dir) + cmd = [python3_path, '--version'] + # just want python3 ver printed in production log + subprocess.Popen(cmd, env=os.environ.copy()) + return python3_path + + def setup_py3_virtualenv(self): + """Mitmproxy on windows needs Python 3.x; set up a separate py 3.x env here""" + LOG.info("Setting up python 3.x virtualenv, required for mitmproxy on windows") + # these next two are required for py3_venv_configuration + self.abs_dirs = {'base_work_dir': mozharness_dir} + self.log_obj = None + # now create the py3 venv + venv_path = os.path.join(self.raptor_dir, 'py3venv') + self.py3_venv_configuration(python_path=self.py3_path, venv_path=venv_path) + self.py3_create_venv() + self.py3_install_modules(["cffi==1.10.0"]) + requirements = [os.path.join(here, "mitmproxy_requirements.txt")] + self.py3_install_requirement_files(requirements) + # add py3 executables path to system path + sys.path.insert(1, self.py3_path_to_executables()) + # install mitmproxy itself + self.py3_install_modules(modules=['mitmproxy']) + self.mitmdump_path = os.path.join(self.py3_path_to_executables(), 'mitmdump') + def start(self): - """Start playing back the mitmproxy recording.""" - - self.mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump') + """Start playing back the mitmproxy recording. If on windows, the mitmdump_path was + already set when creating py3 env""" + if self.mitmdump_path is None: + self.mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump') recordings_list = self.recordings.split() self.mitmproxy_proc = self.start_mitmproxy_playback(self.mitmdump_path, self.recordings_path, recordings_list, self.browser_path) return @@ -175,34 +223,34 @@ class Mitmproxy(Playback): env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"] command = [mitmdump_path, '-k', '-q', '-s', param2] LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"]) LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command)) # to turn off mitmproxy log output, use these params for Popen: # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) - mitmproxy_proc = ProcessHandler(command, env=env) - mitmproxy_proc.run() - + mitmproxy_proc = subprocess.Popen(command, env=env) time.sleep(MITMDUMP_SLEEP) data = mitmproxy_proc.poll() if data is None: # None value indicates process hasn't terminated LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid) return mitmproxy_proc # cannot continue as we won't be able to playback the pages LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data) sys.exit() def stop_mitmproxy_playback(self): """Stop the mitproxy server playback""" mitmproxy_proc = self.mitmproxy_proc LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid) - mitmproxy_proc.kill() - + if mozinfo.os == 'win': + mitmproxy_proc.kill() + else: + mitmproxy_proc.terminate() time.sleep(MITMDUMP_SLEEP) status = mitmproxy_proc.poll() if status is None: # None value indicates process hasn't terminated # I *think* we can still continue, as process will be automatically # killed anyway when mozharness is done (?) if not, we won't be able # to startup mitmxproy next time if it is already running LOG.error("Failed to kill the mitmproxy playback process") LOG.info(str(status)) @@ -281,18 +329,18 @@ class MitmproxyDesktop(Mitmproxy): def is_mitmproxy_cert_installed(self): """Verify mitmxproy CA cert was added to Firefox""" try: # read autoconfig file, confirm mitmproxy cert is in there contents = self.read_policies_json(self.policies_dir) LOG.info("Firefox policies file contents:") LOG.info(contents) if (POLICIES_CONTENT_ON % { - 'cert': self.cert_path, - 'host': self.config['host']}) in contents: + 'cert': self.cert_path, + 'host': self.config['host']}) in contents: LOG.info("Verified mitmproxy CA certificate is installed in Firefox") else: return False except Exception as e: LOG.info("failed to read Firefox policies file, exeption: %s" % e) return False return True
new file mode 100644 --- /dev/null +++ b/testing/raptor/raptor/playback/python3.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 15380470, + "visibility": "public", + "digest": "cd78b88d95b69bef99d7192b71dd34118700f44db0a0069a13bfd4943b131e8d7fdac83859f8ac15d873d4b329eef69d8d75d0a6746d06fdcfc5d06da0c9784c", + "algorithm": "sha512", + "unpack": true, + "filename": "python3.6.zip" + } +]
new file mode 100644 --- /dev/null +++ b/testing/raptor/raptor/playback/python3_x64.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 16026760, + "visibility": "public", + "digest": "379428e3955671213a245ccd9ccf6f9d17d368db68c02da8baed7be629f2691127cd3e3f86807b25e2098d9840083fdc07946ab1bed0c14db4a5b628a47ed9ef", + "algorithm": "sha512", + "unpack": true, + "filename": "python3.6.amd64.zip" + } +]
--- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -166,16 +166,17 @@ class Raptor(object): self.log.info("test uses playback tool: %s " % self.config['playback_tool']) self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None) _key = 'playback_binary_zip_%s' % self.config['platform'] self.config['playback_binary_zip'] = test.get(_key, None) self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None) _key = 'playback_pageset_zip_%s' % self.config['platform'] self.config['playback_pageset_zip'] = test.get(_key, None) self.config['playback_recordings'] = test.get('playback_recordings', None) + self.config['python3_win_manifest'] = test.get('python3_win_manifest', None) def run_test(self, test, timeout=None): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) # benchmark-type tests require the benchmark test to be served out if test.get('type') == "benchmark":
--- a/testing/raptor/raptor/tests/raptor-tp6-1.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-1.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-1 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-10.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-10.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-10 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 60000 gecko_profile_interval = 1
--- a/testing/raptor/raptor/tests/raptor-tp6-2.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-2.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-2 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-2.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 page_timeout = 60000 gecko_profile_interval = 1 gecko_profile_entries = 14000000
--- a/testing/raptor/raptor/tests/raptor-tp6-3.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-3.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-3 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-3.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-4.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-4.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-4 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-4.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-5.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-5.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-5 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-5.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-6.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-6.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-6 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-6.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 30000
--- a/testing/raptor/raptor/tests/raptor-tp6-7.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-7.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-7 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-7.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 60000
--- a/testing/raptor/raptor/tests/raptor-tp6-8.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-8.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-8 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 30000 gecko_profile_interval = 1
--- a/testing/raptor/raptor/tests/raptor-tp6-9.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-9.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6-9 [DEFAULT] type = pageload playback = mitmproxy playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest page_cycles = 25 unit = ms lower_is_better = true alert_threshold = 2.0 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds # beyond typical pageload time page_timeout = 30000 gecko_profile_interval = 1
--- a/testing/raptor/raptor/tests/raptor-tp6m-1.ini +++ b/testing/raptor/raptor/tests/raptor-tp6m-1.ini @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # raptor tp6m-1 [DEFAULT] type = pageload playback = mitmproxy-android playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest page_cycles = 15 unit = ms lower_is_better = true alert_threshold = 2.0 page_timeout = 60000 alert_on = fcp, loadtime [raptor-tp6m-amazon-geckoview]
--- a/testing/raptor/test/test_manifest.py +++ b/testing/raptor/test/test_manifest.py @@ -31,16 +31,17 @@ VALID_MANIFESTS = [{'apps': 'firefox', 'alert_on': 'fcp', 'unit': 'ms', 'lower_is_better': True, 'alert_threshold': 2.0, 'playback': 'mitmproxy', 'playback_binary_manifest': 'binary.manifest', 'playback_pageset_manifest': 'pageset.manifest', 'playback_recordings': 'recorded_site.mp', + 'python3_win_manifest': 'py3.manifest', 'manifest': 'valid_details_1'}, {'apps': 'chrome', 'type': 'benchmark', 'page_cycles': 5, 'test_url': 'http://www.test-url/goes/here', 'measure': 'fcp', 'unit': 'score', 'lower_is_better': False, @@ -53,16 +54,17 @@ INVALID_MANIFESTS = [{'apps': 'firefox', 'test_url': 'http://www.test-url/goes/here', 'unit': 'ms', 'lower_is_better': True, 'alert_threshold': 2.0, 'playback': 'mitmproxy', 'playback_binary_manifest': 'binary.manifest', 'playback_pageset_manifest': 'pageset.manifest', 'playback_recordings': 'recorded_site.mp', + 'python3_win_manifest': 'py3.manifest', 'manifest': 'invalid_details_1'}, {'apps': 'chrome', 'type': 'pageload', 'page_cycles': 25, 'test_url': 'http://www.test-url/goes/here', 'measure': 'fnbpaint, fcp', 'unit': 'ms', 'lower_is_better': True, @@ -77,16 +79,17 @@ INVALID_MANIFESTS = [{'apps': 'firefox', 'alert_on': 'nope', 'unit': 'ms', 'lower_is_better': True, 'alert_threshold': 2.0, 'playback': 'mitmproxy', 'playback_binary_manifest': 'binary.manifest', 'playback_pageset_manifest': 'pageset.manifest', 'playback_recordings': 'recorded_site.mp', + 'python3_win_manifest': 'py3.manifest', 'manifest': 'invalid_details_3'}] @pytest.mark.parametrize('app', ['firefox', 'chrome', 'geckoview']) def test_get_browser_test_list(app): test_list = get_browser_test_list(app) assert len(test_list) > 0