author | Henrik Skupin <mail@hskupin.info> |
Wed, 24 Apr 2019 09:31:37 +0000 | |
changeset 470742 | dbd92ad84948dd4e3b9b676a8c0d686ab06192b7 |
parent 470741 | 97795bbf4e35329511b1371da8f5a7c0c8a7d87a |
child 470743 | 34a50e2bcaebfe8beeef0ce2f09c14df1da2da6a |
push id | 83949 |
push user | hskupin@mozilla.com |
push date | Wed, 24 Apr 2019 19:43:28 +0000 |
treeherder | autoland@b2677f62c9d0 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | davehunt |
bugs | 1506928 |
milestone | 68.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -1,13 +1,14 @@ #!/usr/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. + from __future__ import absolute_import import json import os import posixpath import shutil import sys import tempfile @@ -15,16 +16,17 @@ import time import requests import mozcrash import mozinfo from mozdevice import ADBDevice from mozlog import commandline, get_default_logger from mozprofile import create_profile +from mozproxy import get_playback from mozrunner import runners # need this so raptor imports work both from /raptor and via mach here = os.path.abspath(os.path.dirname(__file__)) paths = [here] if os.environ.get('SCRIPTSPATH', None) is not None: # in production it is env SCRIPTS_PATH paths.append(os.environ['SCRIPTSPATH']) @@ -47,24 +49,24 @@ try: except ImportError: build = None from benchmark import Benchmark from cmdline import (parse_args, FIREFOX_ANDROID_APPS, CHROMIUM_DISTROS) from control_server import RaptorControlServer +from gecko_profile import GeckoProfile from gen_test_config import gen_test_config from outputhandler import OutputHandler from manifest import get_raptor_test_list +from memory import generate_android_memory_profile from mozproxy import get_playback +from power import init_android_power_test, finish_android_power_test from results import RaptorResultsHandler -from gecko_profile import GeckoProfile -from power import init_android_power_test, finish_android_power_test -from memory import generate_android_memory_profile from utils import view_gecko_profile class Raptor(object): """Container class for Raptor""" def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, @@ -122,65 +124,66 @@ class Raptor(object): @property def profile_data_dir(self): if 'MOZ_DEVELOPER_REPO_DIR' in os.environ: return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles') if build: return os.path.join(build.topsrcdir, 'testing', 'profiles') return os.path.join(here, 'profile_data') + def check_for_crashes(self): + raise NotImplementedError + def run_test_setup(self, test): self.log.info("starting raptor test: %s" % test['name']) self.log.info("test settings: %s" % str(test)) self.log.info("raptor config: %s" % str(self.config)) if test.get('type') == "benchmark": self.serve_benchmark_source(test) - gen_test_config(self.config['app'], - test['name'], - self.control_server.port, - self.post_startup_delay, - host=self.config['host'], - b_port=self.benchmark_port, - debug_mode=1 if self.debug_mode else 0, - browser_cycle=test.get('browser_cycle', 1)) + gen_test_config( + self.config['app'], + test['name'], + self.control_server.port, + self.post_startup_delay, + host=self.config['host'], + b_port=self.benchmark_port, + debug_mode=1 if self.debug_mode else 0, + browser_cycle=test.get('browser_cycle', 1), + ) self.install_raptor_webext() if test.get("preferences", None) is not None: self.set_browser_test_prefs(test['preferences']) # if 'alert_on' was provided in the test INI, add to our config for results/output self.config['subtest_alert_on'] = test.get('alert_on', None) - def set_browser_test_prefs(self, raw_prefs): - # add test specific preferences - if self.config['app'] == "firefox": - self.log.info("setting test-specific browser preferences") - self.profile.set_preferences(json.loads(raw_prefs)) - else: - self.log.info("preferences were configured for the test, however \ - we currently do not install them on non Firefox browsers.") - def run_test_teardown(self): self.check_for_crashes() if self.playback is not None: self.playback.stop() self.remove_raptor_webext() # gecko profiling symbolication if self.config['gecko_profile'] is True: self.gecko_profiler.symbolicate() # clean up the temp gecko profiling folders self.log.info("cleaning up after gecko profiling") self.gecko_profiler.clean() + def set_browser_test_prefs(self, raw_prefs): + # add test specific preferences + self.log.info("preferences were configured for the test, however \ + we currently do not install them on non Firefox browsers.") + def create_browser_profile(self): self.profile = create_profile(self.profile_class) # Merge in base profiles with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['raptor'] for name in base_profiles: @@ -193,50 +196,40 @@ class Raptor(object): def start_control_server(self): self.control_server = RaptorControlServer(self.results_handler, self.debug_mode) self.control_server.start() if self.config['enable_control_server_wait']: self.control_server_wait_set('webext_status/__raptor_shutdownBrowser') - # for android we must make the control server available to the device - if self.config['app'] in self.firefox_android_apps and \ - self.config['host'] in ('localhost', '127.0.0.1'): - self.log.info("making the raptor control server port available to device") - _tcp_port = "tcp:%s" % self.control_server.port - self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) + def get_playback_config(self, test): + platform = self.config['platform'] + playback_dir = os.path.join(here, 'playback') - def get_playback_config(self, test): - self.config['playback_tool'] = test.get('playback') - self.log.info("test uses playback tool: %s " % self.config['playback_tool']) - platform = self.config['platform'] - self.config['playback_version'] = test.get('playback_version', "2.0.2") - self.config['playback_binary_zip'] = test.get('playback_binary_zip_%s' % platform) - self.config['playback_pageset_zip'] = test.get('playback_pageset_zip_%s' % platform) - playback_dir = os.path.join(here, 'playback') - self.config['playback_binary_manifest'] = test.get('playback_binary_manifest') - self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest') + self.config.update({ + 'playback_tool': test.get('playback'), + 'playback_version': test.get('playback_version', "2.0.2"), + 'playback_binary_zip': test.get('playback_binary_zip_%s' % platform), + 'playback_pageset_zip': test.get('playback_pageset_zip_%s' % platform), + 'playback_binary_manifest': test.get('playback_binary_manifest'), + 'playback_pageset_manifest': test.get('playback_pageset_manifest'), + }) for key in ('playback_pageset_manifest', 'playback_pageset_zip'): if self.config.get(key) is None: continue self.config[key] = os.path.join(playback_dir, self.config[key]) + self.log.info("test uses playback tool: %s " % self.config['playback_tool']) + def serve_benchmark_source(self, test): # benchmark-type tests require the benchmark test to be served out self.benchmark = Benchmark(self.config, test) self.benchmark_port = int(self.benchmark.port) - # for android we must make the benchmarks server available to the device - if self.config['app'] in self.firefox_android_apps and \ - self.config['host'] in ('localhost', '127.0.0.1'): - self.log.info("making the raptor benchmarks server port available to device") - _tcp_port = "tcp:%s" % self.benchmark_port - self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) - def install_raptor_webext(self): # must intall raptor addon each time because we dynamically update some content # the webext is installed into the browser profile # note: for chrome the addon is just a list of paths that ultimately are added # to the chromium command line '--load-extension' argument self.raptor_webext = os.path.join(webext_dir, 'raptor') self.log.info("installing webext %s" % self.raptor_webext) self.profile.addons.install(self.raptor_webext) @@ -253,24 +246,16 @@ class Raptor(object): if self.config['app'] in ['firefox', 'geckoview', 'fennec', 'refbrow', 'fenix']: self.profile.addons.remove_addon(self.webext_id) # for chrome the addon is just a list (appended to cmd line) chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"] if self.config['app'] in chrome_apps: self.profile.addons.remove(self.raptor_webext) - def set_test_browser_prefs(self, test_prefs): - # if the test has any specific browser prefs specified, set those in browser profiile - if self.config['app'] == "firefox": - self.profile.set_preferences(json.loads(test_prefs)) - else: - self.log.info("preferences were configured for the test, \ - but we do not install them on non Firefox browsers.") - def get_proxy_command_for_mitm(self, test, version): # Generate Mitmproxy playback args script = os.path.join(here, "playback", "alternate-server-replay-{}.py".format(version)) recordings = test.get("playback_recordings") if recordings: recording_paths = [] proxy_dir = self.playback.mozproxy_dir for recording in recordings.split(): @@ -303,23 +288,16 @@ class Raptor(object): self.get_playback_config(test) self.playback = get_playback(self.config, self.device) self.get_proxy_command_for_mitm(test, self.config['playback_version']) # let's start it! self.playback.start() - # for android we must make the playback server available to the device - if self.config['app'] in self.firefox_android_apps and self.config['host'] \ - in ('localhost', '127.0.0.1'): - self.log.info("making the raptor playback server port available to device") - _tcp_port = "tcp:8080" - self.device.create_socket_connection('reverse', _tcp_port, _tcp_port) - def delete_proxy_settings_from_profile(self): # Must delete the proxy settings from the profile if running # the test with a host different from localhost. userjspath = os.path.join(self.profile.profile, 'user.js') with open(userjspath) as userjsfile: prefs = userjsfile.readlines() prefs = [pref for pref in prefs if 'network.proxy' not in pref] with open(userjspath, 'w') as userjsfile: @@ -381,23 +359,16 @@ class Raptor(object): def get_page_timeout_list(self): return self.results_handler.page_timeout_list def clean_up(self): if self.config['enable_control_server_wait']: self.control_server_wait_clear('all') self.control_server.stop() - if self.config['app'] not in self.firefox_android_apps: - self.runner.stop() - elif self.config['app'] in self.firefox_android_apps: - self.log.info('removing reverse socket connections') - self.device.remove_socket_connections('reverse') - else: - pass self.log.info("finished") def control_server_wait_set(self, state): response = requests.post("http://127.0.0.1:%s/" % self.control_server.port, json={"type": "wait-set", "data": state}) return response.content def control_server_wait_timeout(self, timeout): @@ -430,16 +401,19 @@ class RaptorDesktop(Raptor): process_args = { 'processOutputLine': [self.output_handler], } runner_cls = runners[self.config['app']] self.runner = runner_cls( self.config['binary'], profile=self.profile, process_args=process_args, symbols_path=self.config['symbols_path']) + def launch_desktop_browser(self, test): + raise NotImplementedError + def start_runner_proc(self): # launch the browser via our previously-created runner self.runner.start() proc = self.runner.process_handler self.output_handler.proc = proc # give our control server the browser process so it can shut it down later self.control_server.browser_proc = proc @@ -474,16 +448,21 @@ class RaptorDesktop(Raptor): self.runner.wait(timeout=None) def check_for_crashes(self): try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass + def clean_up(self): + self.runner.stop() + + super(RaptorDesktop, self).clean_up() + class RaptorDesktopFirefox(RaptorDesktop): def disable_non_local_connections(self): # For Firefox we need to set MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before startup # when testing release builds from mozilla-beta/release. This is because of restrictions # on release builds that require webextensions to be signed unless this env var is set self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1") @@ -511,16 +490,21 @@ class RaptorDesktopFirefox(RaptorDesktop # if geckoProfile is enabled, initialize it if self.config['gecko_profile'] is True: self._init_gecko_profiling(test) # tell the control server the gecko_profile dir; the control server will # receive the actual gecko profiles from the web ext and will write them # to disk; then profiles are picked up by gecko_profile.symbolicate self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir + def set_browser_test_prefs(self, raw_prefs): + # add test specific preferences + self.log.info("setting test-specific Firefox preferences") + self.profile.set_preferences(json.loads(raw_prefs)) + class RaptorDesktopChrome(RaptorDesktop): def setup_chrome_desktop_for_playback(self): # if running a pageload test on google chrome, add the cmd line options # to turn on the proxy and ignore security certificate errors # if using host localhost, 127.0.0.1. chrome_args = [ @@ -554,16 +538,44 @@ class RaptorAndroid(Raptor): # on android, when creating the browser profile, we want to use a 'firefox' type profile self.profile_class = "firefox" self.config.update({ 'activity': activity, 'intent': intent, }) + def set_reverse_port(self, port): + tcp_port = "tcp:{}".format(port) + self.device.create_socket_connection('reverse', tcp_port, tcp_port) + + def serve_benchmark_source(self, *args, **kwargs): + super(RaptorAndroid, self).serve_benchmark_source(*args, **kwargs) + + # for Android we must make the benchmarks server available to the device + if self.config['host'] in ('localhost', '127.0.0.1'): + self.log.info("making the raptor benchmarks server port available to device") + self.set_reverse_port(self.benchmark_port) + + def start_control_server(self): + super(RaptorAndroid, self).start_control_server() + + # for Android we must make the control server available to the device + if self.config['host'] in ('localhost', '127.0.0.1'): + self.log.info("making the raptor control server port available to device") + self.set_reverse_port(self.control_server.port) + + def start_playback(self, test): + super(RaptorAndroid, self).start_playback(test) + + # for Android we must make the playback server available to the device + if self.config['host'] in ('localhost', '127.0.0.1'): + self.log.info("making the raptor playback server port available to device") + self.set_reverse_port(8080) + def create_browser_handler(self): # create the android device handler; it gets initiated and sets up adb etc self.log.info("creating android device handler using mozdevice") self.device = ADBDevice(verbose=True) self.device.clear_logcat() self.clear_app_data() def clear_app_data(self): @@ -820,16 +832,22 @@ class RaptorAndroid(Raptor): self.device.pull(remote_dir, dump_dir) mozcrash.log_crashes(self.log, dump_dir, self.config['symbols_path']) finally: try: shutil.rmtree(dump_dir) except Exception: self.log.warning("unable to remove directory: %s" % dump_dir) + def clean_up(self): + self.log.info('removing reverse socket connections') + self.device.remove_socket_connections('reverse') + + super(RaptorAndroid, self).clean_up() + def main(args=sys.argv[1:]): args = parse_args() commandline.setup_logging('raptor', args, {'tbpl': sys.stdout}) LOG = get_default_logger(component='raptor-main') LOG.info("raptor-start")