Bug 1506928 - [raptor] Fix inheritance of methods in raptor classes. r=davehunt
☠☠ backed out by f478c61c1bc9 ☠ ☠
authorHenrik Skupin <mail@hskupin.info>
Mon, 15 Apr 2019 21:45:28 +0200
changeset 469764 120d7d8970e5ad9a891adf7ff0516f5aaed67d6c
parent 469763 9ca5edb93d09b69fa866e9ca42c7bdb38ecf2d65
child 469765 097b31242a5abad203858c7c0f3a2ea96e09ad8c
push id112819
push userhskupin@mozilla.com
push dateWed, 17 Apr 2019 11:09:58 +0000
treeherdermozilla-inbound@98c7524647ae [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1506928
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1506928 - [raptor] Fix inheritance of methods in raptor classes. r=davehunt Differential Revision: https://phabricator.services.mozilla.com/D27639
testing/raptor/raptor/raptor.py
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -1,28 +1,30 @@
 #!/usr/bin/env python
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 from __future__ import absolute_import
 
 import json
 import os
 import posixpath
 import shutil
 import sys
 import tempfile
 import time
 
 import mozcrash
 import mozinfo
 from mozdevice import ADBDevice
 from mozlog import commandline, get_default_logger
 from mozprofile import create_profile
+from mozproxy import get_playback
 from mozrunner import runners
 
 # need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 paths = [here]
 if os.environ.get('SCRIPTSPATH', None) is not None:
     # in production it is env SCRIPTS_PATH
     paths.append(os.environ['SCRIPTSPATH'])
@@ -45,23 +47,22 @@ try:
 except ImportError:
     build = None
 
 from benchmark import Benchmark
 from cmdline import (parse_args,
                      FIREFOX_ANDROID_APPS,
                      CHROMIUM_DISTROS)
 from control_server import RaptorControlServer
+from gecko_profile import GeckoProfile
 from gen_test_config import gen_test_config
 from outputhandler import OutputHandler
 from manifest import get_raptor_test_list
-from mozproxy import get_playback
+from power import init_android_power_test, finish_android_power_test
 from results import RaptorResultsHandler
-from gecko_profile import GeckoProfile
-from power import init_android_power_test, finish_android_power_test
 from utils import view_gecko_profile
 
 
 class Raptor(object):
     """Container class for Raptor"""
 
     def __init__(self, app, binary, run_local=False, obj_path=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
@@ -116,65 +117,66 @@ class Raptor(object):
     @property
     def profile_data_dir(self):
         if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
             return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
         if build:
             return os.path.join(build.topsrcdir, 'testing', 'profiles')
         return os.path.join(here, 'profile_data')
 
+    def check_for_crashes(self):
+        raise NotImplementedError
+
     def run_test_setup(self, test):
         self.log.info("starting raptor test: %s" % test['name'])
         self.log.info("test settings: %s" % str(test))
         self.log.info("raptor config: %s" % str(self.config))
 
         if test.get('type') == "benchmark":
             self.serve_benchmark_source(test)
 
-        gen_test_config(self.config['app'],
-                        test['name'],
-                        self.control_server.port,
-                        self.post_startup_delay,
-                        host=self.config['host'],
-                        b_port=self.benchmark_port,
-                        debug_mode=1 if self.debug_mode else 0,
-                        browser_cycle=test.get('browser_cycle', 1))
+        gen_test_config(
+            self.config['app'],
+            test['name'],
+            self.control_server.port,
+            self.post_startup_delay,
+            host=self.config['host'],
+            b_port=self.benchmark_port,
+            debug_mode=1 if self.debug_mode else 0,
+            browser_cycle=test.get('browser_cycle', 1),
+            )
 
         self.install_raptor_webext()
 
         if test.get("preferences", None) is not None:
             self.set_browser_test_prefs(test['preferences'])
 
         # if 'alert_on' was provided in the test INI, add to our config for results/output
         self.config['subtest_alert_on'] = test.get('alert_on', None)
 
-    def set_browser_test_prefs(self, raw_prefs):
-        # add test specific preferences
-        if self.config['app'] == "firefox":
-            self.log.info("setting test-specific browser preferences")
-            self.profile.set_preferences(json.loads(raw_prefs))
-        else:
-            self.log.info("preferences were configured for the test, however \
-                          we currently do not install them on non Firefox browsers.")
-
     def run_test_teardown(self):
         self.check_for_crashes()
 
         if self.playback is not None:
             self.playback.stop()
 
         self.remove_raptor_webext()
 
         # gecko profiling symbolication
         if self.config['gecko_profile'] is True:
             self.gecko_profiler.symbolicate()
             # clean up the temp gecko profiling folders
             self.log.info("cleaning up after gecko profiling")
             self.gecko_profiler.clean()
 
+    def set_browser_test_prefs(self, raw_prefs):
+        # add test specific preferences
+        self.log.info("preferences were configured for the test, however \
+                        we currently do not install them on non Firefox browsers.")
+
     def create_browser_profile(self):
         self.profile = create_profile(self.profile_class)
 
         # Merge in base profiles
         with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
             base_profiles = json.load(fh)['raptor']
 
         for name in base_profiles:
@@ -184,50 +186,40 @@ class Raptor(object):
 
         # add profile dir to our config
         self.config['local_profile_dir'] = self.profile.profile
 
     def start_control_server(self):
         self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
         self.control_server.start()
 
-        # for android we must make the control server available to the device
-        if self.config['app'] in self.firefox_android_apps and \
-                self.config['host'] in ('localhost', '127.0.0.1'):
-            self.log.info("making the raptor control server port available to device")
-            _tcp_port = "tcp:%s" % self.control_server.port
-            self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
+    def get_playback_config(self, test):
+        platform = self.config['platform']
+        playback_dir = os.path.join(here, 'playback')
 
-    def get_playback_config(self, test):
-        self.config['playback_tool'] = test.get('playback')
-        self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
-        platform = self.config['platform']
-        self.config['playback_version'] = test.get('playback_version', "2.0.2")
-        self.config['playback_binary_zip'] = test.get('playback_binary_zip_%s' % platform)
-        self.config['playback_pageset_zip'] = test.get('playback_pageset_zip_%s' % platform)
-        playback_dir = os.path.join(here, 'playback')
-        self.config['playback_binary_manifest'] = test.get('playback_binary_manifest')
-        self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest')
+        self.config.update({
+            'playback_tool': test.get('playback'),
+            'playback_version': test.get('playback_version', "2.0.2"),
+            'playback_binary_zip': test.get('playback_binary_zip_%s' % platform),
+            'playback_pageset_zip': test.get('playback_pageset_zip_%s' % platform),
+            'playback_binary_manifest': test.get('playback_binary_manifest'),
+            'playback_pageset_manifest': test.get('playback_pageset_manifest'),
+        })
         for key in ('playback_pageset_manifest', 'playback_pageset_zip'):
             if self.config.get(key) is None:
                 continue
             self.config[key] = os.path.join(playback_dir, self.config[key])
 
+        self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
+
     def serve_benchmark_source(self, test):
         # benchmark-type tests require the benchmark test to be served out
         self.benchmark = Benchmark(self.config, test)
         self.benchmark_port = int(self.benchmark.port)
 
-        # for android we must make the benchmarks server available to the device
-        if self.config['app'] in self.firefox_android_apps and \
-                self.config['host'] in ('localhost', '127.0.0.1'):
-            self.log.info("making the raptor benchmarks server port available to device")
-            _tcp_port = "tcp:%s" % self.benchmark_port
-            self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
-
     def install_raptor_webext(self):
         # must intall raptor addon each time because we dynamically update some content
         # the webext is installed into the browser profile
         # note: for chrome the addon is just a list of paths that ultimately are added
         # to the chromium command line '--load-extension' argument
         self.raptor_webext = os.path.join(webext_dir, 'raptor')
         self.log.info("installing webext %s" % self.raptor_webext)
         self.profile.addons.install(self.raptor_webext)
@@ -244,24 +236,16 @@ class Raptor(object):
         if self.config['app'] in ['firefox', 'geckoview', 'fennec', 'refbrow', 'fenix']:
             self.profile.addons.remove_addon(self.webext_id)
 
         # for chrome the addon is just a list (appended to cmd line)
         chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"]
         if self.config['app'] in chrome_apps:
             self.profile.addons.remove(self.raptor_webext)
 
-    def set_test_browser_prefs(self, test_prefs):
-        # if the test has any specific browser prefs specified, set those in browser profiile
-        if self.config['app'] == "firefox":
-            self.profile.set_preferences(json.loads(test_prefs))
-        else:
-            self.log.info("preferences were configured for the test, \
-                          but we do not install them on non Firefox browsers.")
-
     def get_proxy_command_for_mitm(self, test, version):
         # Generate Mitmproxy playback args
         script = os.path.join(here, "playback", "alternate-server-replay-{}.py".format(version))
         recordings = test.get("playback_recordings")
         if recordings:
             recording_paths = []
             proxy_dir = self.playback.mozproxy_dir
             for recording in recordings.split():
@@ -294,23 +278,16 @@ class Raptor(object):
         self.get_playback_config(test)
         self.playback = get_playback(self.config, self.device)
 
         self.get_proxy_command_for_mitm(test, self.config['playback_version'])
 
         # let's start it!
         self.playback.start()
 
-        # for android we must make the playback server available to the device
-        if self.config['app'] in self.firefox_android_apps and self.config['host'] \
-                in ('localhost', '127.0.0.1'):
-            self.log.info("making the raptor playback server port available to device")
-            _tcp_port = "tcp:8080"
-            self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
-
     def delete_proxy_settings_from_profile(self):
         # Must delete the proxy settings from the profile if running
         # the test with a host different from localhost.
         userjspath = os.path.join(self.profile.profile, 'user.js')
         with open(userjspath) as userjsfile:
             prefs = userjsfile.readlines()
         prefs = [pref for pref in prefs if 'network.proxy' not in pref]
         with open(userjspath, 'w') as userjsfile:
@@ -363,23 +340,16 @@ class Raptor(object):
         self.config['raptor_json_path'] = raptor_json_path
         return self.results_handler.summarize_and_output(self.config, test_names)
 
     def get_page_timeout_list(self):
         return self.results_handler.page_timeout_list
 
     def clean_up(self):
         self.control_server.stop()
-        if self.config['app'] not in self.firefox_android_apps:
-            self.runner.stop()
-        elif self.config['app'] in self.firefox_android_apps:
-            self.log.info('removing reverse socket connections')
-            self.device.remove_socket_connections('reverse')
-        else:
-            pass
         self.log.info("finished")
 
 
 class RaptorDesktop(Raptor):
 
     def create_browser_handler(self):
         # create the desktop browser runner
         self.log.info("creating browser runner using mozrunner")
@@ -387,16 +357,19 @@ class RaptorDesktop(Raptor):
         process_args = {
             'processOutputLine': [self.output_handler],
         }
         runner_cls = runners[self.config['app']]
         self.runner = runner_cls(
             self.config['binary'], profile=self.profile, process_args=process_args,
             symbols_path=self.config['symbols_path'])
 
+    def launch_desktop_browser(self, test):
+        raise NotImplementedError
+
     def start_runner_proc(self):
         # launch the browser via our previously-created runner
         self.runner.start()
         proc = self.runner.process_handler
         self.output_handler.proc = proc
 
         # give our control server the browser process so it can shut it down later
         self.control_server.browser_proc = proc
@@ -431,16 +404,21 @@ class RaptorDesktop(Raptor):
                 self.runner.wait(timeout=None)
 
     def check_for_crashes(self):
         try:
             self.runner.check_for_crashes()
         except NotImplementedError:  # not implemented for Chrome
             pass
 
+    def clean_up(self):
+        self.runner.stop()
+
+        super(RaptorDesktop, self).clean_up()
+
 
 class RaptorDesktopFirefox(RaptorDesktop):
 
     def disable_non_local_connections(self):
         # For Firefox we need to set MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before startup
         # when testing release builds from mozilla-beta/release. This is because of restrictions
         # on release builds that require webextensions to be signed unless this env var is set
         self.log.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
@@ -468,16 +446,21 @@ class RaptorDesktopFirefox(RaptorDesktop
         # if geckoProfile is enabled, initialize it
         if self.config['gecko_profile'] is True:
             self._init_gecko_profiling(test)
             # tell the control server the gecko_profile dir; the control server will
             # receive the actual gecko profiles from the web ext and will write them
             # to disk; then profiles are picked up by gecko_profile.symbolicate
             self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir
 
+    def set_browser_test_prefs(self, raw_prefs):
+        # add test specific preferences
+        self.log.info("setting test-specific Firefox preferences")
+        self.profile.set_preferences(json.loads(raw_prefs))
+
 
 class RaptorDesktopChrome(RaptorDesktop):
 
     def setup_chrome_desktop_for_playback(self):
         # if running a pageload test on google chrome, add the cmd line options
         # to turn on the proxy and ignore security certificate errors
         # if using host localhost, 127.0.0.1.
         chrome_args = [
@@ -508,16 +491,44 @@ class RaptorDesktopChrome(RaptorDesktop)
 class RaptorAndroid(Raptor):
     def __init__(self, app, binary, activity=None, **kwargs):
         super(RaptorAndroid, self).__init__(app, binary, **kwargs)
 
         # on android, when creating the browser profile, we want to use a 'firefox' type profile
         self.profile_class = "firefox"
         self.config['activity'] = activity
 
+    def set_reverse_port(self, port):
+        tcp_port = "tcp:{}".format(port)
+        self.device.create_socket_connection('reverse', tcp_port, tcp_port)
+
+    def serve_benchmark_source(self, *args, **kwargs):
+        super(RaptorAndroid, self).serve_benchmark_source(*args, **kwargs)
+
+        # for Android we must make the benchmarks server available to the device
+        if self.config['host'] in ('localhost', '127.0.0.1'):
+            self.log.info("making the raptor benchmarks server port available to device")
+            self.set_reverse_port(self.benchmark_port)
+
+    def start_control_server(self):
+        super(RaptorAndroid, self).start_control_server()
+
+        # for Android we must make the control server available to the device
+        if self.config['host'] in ('localhost', '127.0.0.1'):
+            self.log.info("making the raptor control server port available to device")
+            self.set_reverse_port(self.control_server.port)
+
+    def start_playback(self, test):
+        super(RaptorAndroid, self).start_playback(test)
+
+        # for Android we must make the playback server available to the device
+        if self.config['host'] in ('localhost', '127.0.0.1'):
+            self.log.info("making the raptor playback server port available to device")
+            self.set_reverse_port(8080)
+
     def create_browser_handler(self):
         # create the android device handler; it gets initiated and sets up adb etc
         self.log.info("creating android device handler using mozdevice")
         self.device = ADBDevice(verbose=True)
         self.device.clear_logcat()
         self.clear_app_data()
 
     def clear_app_data(self):
@@ -771,16 +782,22 @@ class RaptorAndroid(Raptor):
             self.device.pull(remote_dir, dump_dir)
             mozcrash.log_crashes(self.log, dump_dir, self.config['symbols_path'])
         finally:
             try:
                 shutil.rmtree(dump_dir)
             except Exception:
                 self.log.warning("unable to remove directory: %s" % dump_dir)
 
+    def clean_up(self):
+        self.log.info('removing reverse socket connections')
+        self.device.remove_socket_connections('reverse')
+
+        super(RaptorAndroid, self).clean_up()
+
 
 def main(args=sys.argv[1:]):
     args = parse_args()
     commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
     LOG = get_default_logger(component='raptor-main')
 
     LOG.info("raptor-start")