Bug 1528236 - [raptor] Use constructor for initialization of raptor instances. r=perftest-reviewers,rwood
authorHenrik Skupin <mail@hskupin.info>
Tue, 14 May 2019 19:32:11 +0000
changeset 532641 09eea8c87cd551a490ac92ebdc99cbb2041f07ee
parent 532640 e8d6d453337d96fb354184b2045e6b5b456a97b7
child 532642 ed679c1e266d2beb10cd491f710fb6021e37c158
push id11270
push userrgurzau@mozilla.com
push dateWed, 15 May 2019 15:07:19 +0000
treeherdermozilla-beta@571bc76da583 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, rwood
bugs1528236
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1528236 - [raptor] Use constructor for initialization of raptor instances. r=perftest-reviewers,rwood The patch tries to reduce the amount of initialization code to call when creating a new Raptor class. Initializing the ADBDevice already in the constructor of the RaptorAndroid class is currently not possible because it would mean that a device is immediately created but the "adb" binary is not available for python test jobs. This has to be fixed later. Differential Revision: https://phabricator.services.mozilla.com/D29331
testing/raptor/raptor/raptor.py
testing/raptor/test/conftest.py
testing/raptor/test/test_raptor.py
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -78,17 +78,17 @@ class SignalHandler:
 
 class SignalHandlerException(Exception):
     pass
 
 
 class Raptor(object):
     """Container class for Raptor"""
 
-    def __init__(self, app, binary, run_local=False, obj_path=None,
+    def __init__(self, app, binary, run_local=False, obj_path=None, profile_class=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
                  symbols_path=None, host=None, power_test=False, memory_test=False,
                  is_release_build=False, debug_mode=False, post_startup_delay=None,
                  interrupt_handler=None, e10s=True, **kwargs):
 
         # Override the magic --host HOST_IP with the value of the environment variable.
         if host == 'HOST_IP':
             host = os.environ['HOST_IP']
@@ -116,17 +116,17 @@ class Raptor(object):
         self.log = get_default_logger(component='raptor-main')
         self.control_server = None
         self.playback = None
         self.benchmark = None
         self.benchmark_port = 0
         self.gecko_profiler = None
         self.post_startup_delay = post_startup_delay
         self.device = None
-        self.profile_class = app
+        self.profile_class = profile_class or app
         self.firefox_android_apps = FIREFOX_ANDROID_APPS
         self.interrupt_handler = interrupt_handler
 
         # debug mode is currently only supported when running locally
         self.debug_mode = debug_mode if self.config['run_local'] else False
 
         # if running debug-mode reduce the pause after browser startup
         if self.debug_mode:
@@ -134,16 +134,19 @@ class Raptor(object):
             self.log.info("debug-mode enabled, reducing post-browser startup pause to %d ms"
                           % self.post_startup_delay)
 
         self.log.info("main raptor init, config is: %s" % str(self.config))
 
         # create results holder
         self.results_handler = RaptorResultsHandler()
 
+        self.create_browser_profile()
+        self.start_control_server()
+
     @property
     def profile_data_dir(self):
         if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
             return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
         if build:
             return os.path.join(build.topsrcdir, 'testing', 'profiles')
         return os.path.join(here, 'profile_data')
 
@@ -421,17 +424,19 @@ class Raptor(object):
     def control_server_wait_clear(self, state):
         response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
                                  json={"type": "wait-clear", "data": state})
         return response.content
 
 
 class RaptorDesktop(Raptor):
 
-    def create_browser_handler(self):
+    def __init__(self, *args, **kwargs):
+        super(RaptorDesktop, self).__init__(*args, **kwargs)
+
         # create the desktop browser runner
         self.log.info("creating browser runner using mozrunner")
         self.output_handler = OutputHandler()
         process_args = {
             'processOutputLine': [self.output_handler],
         }
         runner_cls = runners[self.config['app']]
         self.runner = runner_cls(
@@ -630,67 +635,58 @@ class RaptorDesktopChrome(RaptorDesktop)
             self.setup_chrome_desktop_for_playback()
 
         self.start_runner_proc()
 
 
 class RaptorAndroid(Raptor):
 
     def __init__(self, app, binary, activity=None, intent=None, **kwargs):
-        super(RaptorAndroid, self).__init__(app, binary, **kwargs)
+        super(RaptorAndroid, self).__init__(app, binary, profile_class="firefox", **kwargs)
 
         self.config.update({
             'activity': activity,
             'intent': intent,
         })
 
-        self.profile_class = "firefox"
-
         self.remote_test_root = os.path.abspath(os.path.join(os.sep, 'sdcard', 'raptor'))
         self.remote_profile = os.path.join(self.remote_test_root, "profile")
 
     def set_reverse_port(self, port):
         tcp_port = "tcp:{}".format(port)
         self.device.create_socket_connection('reverse', tcp_port, tcp_port)
 
-    def serve_benchmark_source(self, *args, **kwargs):
-        super(RaptorAndroid, self).serve_benchmark_source(*args, **kwargs)
-
-        # for Android we must make the benchmarks server available to the device
-        if self.config['host'] in ('localhost', '127.0.0.1'):
-            self.log.info("making the raptor benchmarks server port available to device")
-            self.set_reverse_port(self.benchmark_port)
-
-    def start_control_server(self):
-        super(RaptorAndroid, self).start_control_server()
-
-        # for Android we must make the control server available to the device
+    def set_reverse_ports(self, is_benchmark=False):
+        # Make services running on the host available to the device
         if self.config['host'] in ('localhost', '127.0.0.1'):
             self.log.info("making the raptor control server port available to device")
             self.set_reverse_port(self.control_server.port)
 
-    def start_playback(self, test):
-        super(RaptorAndroid, self).start_playback(test)
-
-        # for Android we must make the playback server available to the device
         if self.config['host'] in ('localhost', '127.0.0.1'):
             self.log.info("making the raptor playback server port available to device")
             self.set_reverse_port(8080)
 
-    def create_browser_handler(self):
-        # create the android device handler; it gets initiated and sets up adb etc
-        self.log.info("creating android device handler using mozdevice")
-        self.device = ADBDevice(verbose=True)
-        self.device.clear_logcat()
+        if is_benchmark and self.config['host'] in ('localhost', '127.0.0.1'):
+            self.log.info("making the raptor benchmarks server port available to device")
+            self.set_reverse_port(self.benchmark_port)
+
+    def setup_adb_device(self):
+        if self.device is None:
+            self.device = ADBDevice(verbose=True)
+            self.tune_performance()
+
+        self.log.info("creating remote root folder for raptor: %s" % self.remote_test_root)
+        self.device.rm(self.remote_test_root, force=True, recursive=True)
+        self.device.mkdir(self.remote_test_root)
+        self.device.chmod(self.remote_test_root, recursive=True, root=True)
 
         self.clear_app_data()
-        self.create_raptor_sdcard_folder()
 
     def tune_performance(self):
-        """Sets various performance-oriented parameters, to reduce jitter.
+        """Set various performance-oriented parameters, to reduce jitter.
 
         For more information, see https://bugzilla.mozilla.org/show_bug.cgi?id=1547135.
         """
         self.log.info("tuning android device performance")
         self.set_scheduler()
         self.set_svc_power_stayon()
         device_name = self.device.shell_output('getprop ro.product.model')
         if (self.device._have_su or self.device._have_android_su):
@@ -868,22 +864,16 @@ class RaptorAndroid(Raptor):
             # self.device.mkdir(self.remote_profile)
             self.device.push(self.profile.profile, self.remote_profile)
             self.device.chmod(self.remote_profile, recursive=True, root=True)
 
         except Exception:
             self.log.error("Unable to copy profile to device.")
             raise
 
-    def create_raptor_sdcard_folder(self):
-        self.log.info("creating test folder for raptor: %s" % self.remote_test_root)
-        self.device.rm(self.remote_test_root, force=True, recursive=True)
-        self.device.mkdir(self.remote_test_root)
-        self.device.chmod(self.remote_test_root, recursive=True, root=True)
-
     def turn_on_android_app_proxy(self):
         # for geckoview/android pageload playback we can't use a policy to turn on the
         # proxy; we need to set prefs instead; note that the 'host' may be different
         # than '127.0.0.1' so we must set the prefs accordingly
         self.log.info("setting profile prefs to turn on the android app proxy")
         proxy_prefs = {}
         proxy_prefs["network.proxy.type"] = 1
         proxy_prefs["network.proxy.http"] = self.config['host']
@@ -949,16 +939,33 @@ class RaptorAndroid(Raptor):
             _source = os.path.join(source_dir, next_file)
             _dest = os.path.join(target_dir, next_file)
             if os.path.exists(_source):
                 self.log.info("copying %s to %s" % (_source, _dest))
                 shutil.copyfile(_source, _dest)
             else:
                 self.log.critical("unable to find ssl cert db file: %s" % _source)
 
+    def run_tests(self, tests, test_names):
+        self.setup_adb_device()
+
+        return super(RaptorAndroid, self).run_tests(tests, test_names)
+
+    def run_test_setup(self, test):
+        super(RaptorAndroid, self).run_test_setup(test)
+
+        is_benchmark = test.get('type') == "benchmark"
+        self.set_reverse_ports(is_benchmark=is_benchmark)
+
+    def run_test_teardown(self):
+        self.log.info('removing reverse socket connections')
+        self.device.remove_socket_connections('reverse')
+
+        super(RaptorAndroid, self).run_test_teardown()
+
     def run_test(self, test, timeout=None):
         # tests will be run warm (i.e. NO browser restart between page-cycles)
         # unless otheriwse specified in the test INI by using 'cold = true'
         try:
             if test.get('cold', False) is True:
                 self.run_test_cold(test, timeout)
             else:
                 self.run_test_warm(test, timeout)
@@ -1115,19 +1122,16 @@ class RaptorAndroid(Raptor):
             mozcrash.log_crashes(self.log, dump_dir, self.config['symbols_path'])
         finally:
             try:
                 shutil.rmtree(dump_dir)
             except Exception:
                 self.log.warning("unable to remove directory: %s" % dump_dir)
 
     def clean_up(self):
-        self.log.info('removing reverse socket connections')
-        self.device.remove_socket_connections('reverse')
-
         self.log.info("removing test folder for raptor: %s" % self.remote_test_root)
         self.device.rm(self.remote_test_root, force=True, recursive=True)
 
         super(RaptorAndroid, self).clean_up()
 
 
 def main(args=sys.argv[1:]):
     args = parse_args()
@@ -1176,23 +1180,16 @@ def main(args=sys.argv[1:]):
                           is_release_build=args.is_release_build,
                           debug_mode=args.debug_mode,
                           post_startup_delay=args.post_startup_delay,
                           activity=args.activity,
                           intent=args.intent,
                           interrupt_handler=SignalHandler(),
                           )
 
-    raptor.create_browser_profile()
-    raptor.create_browser_handler()
-    if type(raptor) == RaptorAndroid:
-        # only Raptor Android supports device performance tuning
-        raptor.tune_performance()
-    raptor.start_control_server()
-
     success = raptor.run_tests(raptor_test_list, raptor_test_names)
 
     if not success:
         # didn't get test results; test timed out or crashed, etc. we want job to fail
         LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found for %s" %
                      ', '.join(raptor_test_names))
         os.sys.exit(1)
 
--- a/testing/raptor/test/conftest.py
+++ b/testing/raptor/test/conftest.py
@@ -5,17 +5,17 @@ import os
 import sys
 
 import pytest
 
 from argparse import Namespace
 
 # need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
-if os.environ.get('SCRIPTSPATH', None) is not None:
+if os.environ.get('SCRIPTSPATH') is not None:
     # in production it is env SCRIPTS_PATH
     mozharness_dir = os.environ['SCRIPTSPATH']
 else:
     # locally it's in source tree
     mozharness_dir = os.path.join(here, '../../mozharness')
 sys.path.insert(0, mozharness_dir)
 
 from raptor.raptor import RaptorDesktopFirefox
--- a/testing/raptor/test/test_raptor.py
+++ b/testing/raptor/test/test_raptor.py
@@ -20,45 +20,41 @@ else:
     # locally it's in source tree
     mozharness_dir = os.path.join(here, '../../mozharness')
 sys.path.insert(0, mozharness_dir)
 
 from raptor.raptor import RaptorDesktopFirefox, RaptorDesktopChrome, RaptorAndroid
 
 
 class TestBrowserThread(threading.Thread):
-        def __init__(self, raptor_instance, test, timeout=None):
+        def __init__(self, raptor_instance, tests, names):
             super(TestBrowserThread, self).__init__()
             self.raptor_instance = raptor_instance
-            self.test = test
-            self.timeout = timeout
+            self.tests = tests
+            self.names = names
             self.exc = None
 
         def run(self):
             try:
-                self.raptor_instance.run_test(self.test, self.timeout)
+                self.raptor_instance.run_tests(self.tests, self.names)
             except BaseException:
                 self.exc = sys.exc_info()
 
 
 @pytest.mark.parametrize("raptor_class, app_name", [
                          [RaptorDesktopFirefox, "firefox"],
                          [RaptorDesktopChrome, "chrome"],
                          [RaptorDesktopChrome, "chromium"],
                          [RaptorAndroid, "fennec"],
                          [RaptorAndroid, "geckoview"],
                          ])
-def test_create_profile(options, raptor_class, app_name, get_prefs):
+def test_build_profile(options, raptor_class, app_name, get_prefs):
     options['app'] = app_name
     raptor = raptor_class(**options)
 
-    if app_name in ["fennec", "geckoview"]:
-        raptor.profile_class = "firefox"
-    raptor.create_browser_profile()
-
     assert isinstance(raptor.profile, BaseProfile)
     if app_name != 'firefox':
         return
 
     # These prefs are set in mozprofile
     firefox_prefs = [
         'user_pref("app.update.checkInstallTime", false);',
         'user_pref("app.update.disabledForTesting", true);',
@@ -72,22 +68,16 @@ def test_create_profile(options, raptor_
     with open(prefs_file, 'r') as fh:
         prefs = fh.read()
         for firefox_pref in firefox_prefs:
             assert firefox_pref in prefs
         assert raptor_pref in prefs
 
 
 def test_start_and_stop_server(raptor):
-    assert raptor.control_server is None
-
-    raptor.create_browser_profile()
-    raptor.create_browser_handler()
-    raptor.start_control_server()
-
     assert raptor.control_server._server_thread.is_alive()
     assert raptor.control_server.port is not None
     assert raptor.control_server.server is not None
 
     raptor.clean_up()
     assert not raptor.control_server._server_thread.is_alive()
 
 
@@ -96,22 +86,16 @@ def test_server_wait_states(raptor):
 
     import requests
 
     def post_state():
         requests.post("http://127.0.0.1:%s/" % raptor.control_server.port,
                       json={"type": "webext_status",
                             "data": "test status"})
 
-    assert raptor.control_server is None
-
-    raptor.create_browser_profile()
-    raptor.create_browser_handler()
-    raptor.start_control_server()
-
     wait_time = 5
     message_state = 'webext_status/test status'
     rhc = raptor.control_server.server.RequestHandlerClass
 
     # Test initial state
     assert rhc.wait_after_messages == {}
     assert rhc.waiting_in_state is None
     assert rhc.wait_timeout == 60
@@ -139,17 +123,17 @@ def test_server_wait_states(raptor):
     # Test wait timeout
     # Block on post request
     assert raptor.control_server_wait_set(message_state) == ''
     assert rhc.wait_after_messages[message_state]
     assert raptor.control_server_wait_timeout(wait_time) == ''
     assert rhc.wait_timeout == wait_time
     start = datetime.datetime.now()
     post_state()
-    assert datetime.datetime.now() - start < datetime.timedelta(seconds=wait_time+2)
+    assert datetime.datetime.now() - start < datetime.timedelta(seconds=wait_time + 2)
     assert raptor.control_server_wait_get() == 'None'
     assert message_state not in rhc.wait_after_messages
 
     raptor.clean_up()
     assert not raptor.control_server._server_thread.is_alive()
 
 
 @pytest.mark.parametrize('app', [
@@ -157,24 +141,24 @@ def test_server_wait_states(raptor):
     pytest.mark.xfail('chrome'),
     pytest.mark.xfail('chromium'),
 ])
 def test_start_browser(get_binary, app):
     binary = get_binary(app)
     assert binary
 
     raptor = RaptorDesktopFirefox(app, binary, post_startup_delay=0)
-    raptor.create_browser_profile()
-    raptor.create_browser_handler()
-    raptor.start_control_server()
 
-    test = {}
-    test['name'] = 'raptor-{}-tp6'.format(app)
+    tests = [{
+        'name': 'raptor-{}-tp6'.format(app),
+        'page_timeout': 1000
+    }]
+    test_names = [test['name'] for test in tests]
 
-    thread = TestBrowserThread(raptor, test, timeout=0)
+    thread = TestBrowserThread(raptor, tests, test_names)
     thread.start()
 
     timeout = time.time() + 5  # seconds
     while time.time() < timeout:
         try:
             is_running = raptor.runner.is_running()
             assert is_running
             break