author | Henrik Skupin <mail@hskupin.info> |
Wed, 24 Apr 2019 19:06:06 +0000 | |
changeset 470744 | b2677f62c9d0a8a99860a18c7f400874db82e55e |
parent 470743 | 34a50e2bcaebfe8beeef0ce2f09c14df1da2da6a |
child 470745 | 83021d773da65cf00f6021d17b59eaf7cf622e36 |
push id | 83949 |
push user | hskupin@mozilla.com |
push date | Wed, 24 Apr 2019 19:43:28 +0000 |
treeherder | autoland@b2677f62c9d0 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | rwood, tarek |
bugs | 1506928 |
milestone | 68.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -5,16 +5,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import import json import os import posixpath import shutil +import signal import sys import tempfile import time import requests import mozcrash import mozinfo @@ -60,23 +61,38 @@ from outputhandler import OutputHandler from manifest import get_raptor_test_list from memory import generate_android_memory_profile from mozproxy import get_playback from power import init_android_power_test, finish_android_power_test from results import RaptorResultsHandler from utils import view_gecko_profile +class SignalHandler: + + def __init__(self): + signal.signal(signal.SIGINT, self.handle_signal) + signal.signal(signal.SIGTERM, self.handle_signal) + + def handle_signal(self, signum, frame): + raise SignalHandlerException("Program aborted due to signal %s" % signum) + + +class SignalHandlerException(Exception): + pass + + class Raptor(object): """Container class for Raptor""" def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, symbols_path=None, host=None, power_test=False, memory_test=False, - is_release_build=False, debug_mode=False, post_startup_delay=None, **kwargs): + is_release_build=False, debug_mode=False, post_startup_delay=None, + interrupt_handler=None, **kwargs): # Override the magic --host HOST_IP with the value of the environment variable. if host == 'HOST_IP': host = os.environ['HOST_IP'] self.config = { 'app': app, 'binary': binary, @@ -101,16 +117,17 @@ class Raptor(object): self.playback = None self.benchmark = None self.benchmark_port = 0 self.gecko_profiler = None self.post_startup_delay = post_startup_delay self.device = None self.profile_class = app self.firefox_android_apps = FIREFOX_ANDROID_APPS + self.interrupt_handler = interrupt_handler # debug mode is currently only supported when running locally self.debug_mode = debug_mode if self.config['run_local'] else False # if running debug-mode reduce the pause after browser startup if self.debug_mode: self.post_startup_delay = min(self.post_startup_delay, 3000) self.log.info("debug-mode enabled, reducing post-browser startup pause to %d ms" @@ -144,17 +161,17 @@ class Raptor(object): self.config['app'], test['name'], self.control_server.port, self.post_startup_delay, host=self.config['host'], b_port=self.benchmark_port, debug_mode=1 if self.debug_mode else 0, browser_cycle=test.get('browser_cycle', 1), - ) + ) self.install_raptor_webext() if test.get("preferences") is not None: self.set_browser_test_prefs(test['preferences']) # if 'alert_on' was provided in the test INI, add to our config for results/output self.config['subtest_alert_on'] = test.get('alert_on') @@ -416,42 +433,47 @@ class RaptorDesktop(Raptor): self.output_handler.proc = proc # give our control server the browser process so it can shut it down later self.control_server.browser_proc = proc def run_test(self, test, timeout=None): self.run_test_setup(test) - if test.get('playback') is not None: - self.start_playback(test) + try: + if test.get('playback') is not None: + self.start_playback(test) - if self.config['host'] not in ('localhost', '127.0.0.1'): - self.delete_proxy_settings_from_profile() + if self.config['host'] not in ('localhost', '127.0.0.1'): + self.delete_proxy_settings_from_profile() + + # start the browser/app under test + self.launch_desktop_browser(test) - # now start the browser/app under test - self.launch_desktop_browser(test) + # set our control server flag to indicate we are running the browser/app + self.control_server._finished = False + + self.wait_for_test_finish(test, timeout) - # set our control server flag to indicate we are running the browser/app - self.control_server._finished = False + finally: + self.run_test_teardown() - self.wait_for_test_finish(test, timeout) - - self.run_test_teardown() - + def run_test_teardown(self): # browser should be closed by now but this is a backup-shutdown (if not in debug-mode) if not self.debug_mode: if self.runner.is_running(): self.runner.stop() else: # in debug mode, and running locally, leave the browser running if self.config['run_local']: self.log.info("* debug-mode enabled - please shutdown the browser manually...") self.runner.wait(timeout=None) + super(RaptorDesktop, self).run_test_teardown() + def check_for_crashes(self): try: self.runner.check_for_crashes() except NotImplementedError: # not implemented for Chrome pass def clean_up(self): self.runner.stop() @@ -671,20 +693,30 @@ class RaptorAndroid(Raptor): self.log.info("copying %s to %s" % (_source, _dest)) shutil.copyfile(_source, _dest) else: self.log.critical("unable to find ssl cert db file: %s" % _source) def run_test(self, test, timeout=None): # tests will be run warm (i.e. NO browser restart between page-cycles) # unless otheriwse specified in the test INI by using 'cold = true' - if test.get('cold', False) is True: - self.run_test_cold(test, timeout) - else: - self.run_test_warm(test, timeout) + try: + if test.get('cold', False) is True: + self.run_test_cold(test, timeout) + else: + self.run_test_warm(test, timeout) + + except SignalHandlerException: + self.device.stop_application(self.config['binary']) + + finally: + if self.config['power_test']: + finish_android_power_test(self, test['name']) + + self.run_test_teardown() def run_test_cold(self, test, timeout=None): ''' Run the Raptor test but restart the entire browser app between page-cycles. Note: For page-load tests, playback will only be started once - at the beginning of all browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs in the browser profile so those will need to be set again in each new profile/cycle. @@ -765,21 +797,16 @@ class RaptorAndroid(Raptor): self.wait_for_test_finish(test, timeout) # in debug mode, and running locally, leave the browser running if self.debug_mode and self.config['run_local']: self.log.info("* debug-mode enabled - please shutdown the browser manually...") self.runner.wait(timeout=None) - if self.config['power_test']: - finish_android_power_test(self, test['name']) - - self.run_test_teardown() - def run_test_warm(self, test, timeout=None): self.log.info("test %s is running in warm mode; browser will NOT be restarted between " "page cycles" % test['name']) if self.config['power_test']: init_android_power_test(self) self.run_test_setup(test) self.create_raptor_sdcard_folder() @@ -799,21 +826,16 @@ class RaptorAndroid(Raptor): # now start the browser/app under test self.launch_firefox_android_app(test['name']) # set our control server flag to indicate we are running the browser/app self.control_server._finished = False self.wait_for_test_finish(test, timeout) - if self.config['power_test']: - finish_android_power_test(self, test['name']) - - self.run_test_teardown() - # in debug mode, and running locally, leave the browser running if self.debug_mode and self.config['run_local']: self.log.info("* debug-mode enabled - please shutdown the browser manually...") self.runner.wait(timeout=None) def check_for_crashes(self): # Turn off verbose to prevent logcat from being inserted into the main log. verbose = self.device._verbose @@ -887,27 +909,32 @@ def main(args=sys.argv[1:]): symbols_path=args.symbols_path, host=args.host, power_test=args.power_test, memory_test=args.memory_test, is_release_build=args.is_release_build, debug_mode=args.debug_mode, post_startup_delay=args.post_startup_delay, activity=args.activity, - intent=args.intent) + intent=args.intent, + interrupt_handler=SignalHandler(), + ) raptor.create_browser_profile() raptor.create_browser_handler() raptor.start_control_server() - for next_test in raptor_test_list: - raptor.run_test(next_test, timeout=int(next_test['page_timeout'])) + try: + for next_test in raptor_test_list: + raptor.run_test(next_test, timeout=int(next_test['page_timeout'])) - success = raptor.process_results(raptor_test_names) - raptor.clean_up() + success = raptor.process_results(raptor_test_names) + + finally: + raptor.clean_up() if not success: # didn't get test results; test timed out or crashed, etc. we want job to fail LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found for %s" % ', '.join(raptor_test_names)) os.sys.exit(1) # if we have results but one test page timed out (i.e. one tp6 test page didn't load