Backed out 2 changesets (bug 1561939) as per nalexander`s request.
authorNarcis Beleuzu <nbeleuzu@mozilla.com>
Wed, 24 Jul 2019 00:41:43 +0300
changeset 483922 c2a690d8461ca420f0ac9e864c07157bd89ea52a
parent 483921 35c99c4a67383e4b78a84c8d744ee27a3bb0af25
child 483923 7e617d055160592022f71cf21f9bea7c2f7a3937
push id90698
push usernbeleuzu@mozilla.com
push dateTue, 23 Jul 2019 21:42:18 +0000
treeherderautoland@c2a690d8461c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1561939
milestone70.0a1
backs out2833ba67a5b97cbe597667d035fd3360ebf33995
3125e12eee29b9a660a9b9767399ee71e38cfea6
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 2 changesets (bug 1561939) as per nalexander`s request. Backed out changeset 2833ba67a5b9 (bug 1561939) Backed out changeset 3125e12eee29 (bug 1561939)
testing/config/mozbase_source_requirements.txt
testing/raptor/raptor/raptor.py
--- a/testing/config/mozbase_source_requirements.txt
+++ b/testing/config/mozbase_source_requirements.txt
@@ -1,20 +1,20 @@
---editable ../../python/mozterm
+../../python/mozterm
 
---editable ../mozbase/manifestparser
---editable ../mozbase/mozcrash
---editable ../mozbase/mozdebug
---editable ../mozbase/mozdevice
---editable ../mozbase/mozfile
---editable ../mozbase/mozhttpd
---editable ../mozbase/mozinfo
---editable ../mozbase/mozinstall
---editable ../mozbase/mozleak
---editable ../mozbase/mozlog
---editable ../mozbase/moznetwork
---editable ../mozbase/mozprocess
---editable ../mozbase/mozprofile
---editable ../mozbase/mozproxy
---editable ../mozbase/mozrunner
---editable ../mozbase/mozscreenshot
---editable ../mozbase/moztest
---editable ../mozbase/mozversion
+../mozbase/manifestparser
+../mozbase/mozcrash
+../mozbase/mozdebug
+../mozbase/mozdevice
+../mozbase/mozfile
+../mozbase/mozhttpd
+../mozbase/mozinfo
+../mozbase/mozinstall
+../mozbase/mozleak
+../mozbase/mozlog
+../mozbase/moznetwork
+../mozbase/mozprocess
+../mozbase/mozprofile
+../mozbase/mozproxy
+../mozbase/mozrunner
+../mozbase/mozscreenshot
+../mozbase/moztest
+../mozbase/mozversion
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -1,17 +1,16 @@
 #!/usr/bin/env python
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
-from abc import ABCMeta, abstractmethod
 import json
 import os
 import posixpath
 import shutil
 import signal
 import sys
 import tempfile
 import time
@@ -74,21 +73,18 @@ class SignalHandler:
     def handle_signal(self, signum, frame):
         raise SignalHandlerException("Program aborted due to signal %s" % signum)
 
 
 class SignalHandlerException(Exception):
     pass
 
 
-class Perftest(object):
-    """Abstract base class for perftests that execute via a subharness,
-either Raptor or browsertime."""
-
-    __metaclass__ = ABCMeta
+class Raptor(object):
+    """Container class for Raptor"""
 
     def __init__(self, app, binary, run_local=False, obj_path=None, profile_class=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
                  symbols_path=None, host=None, power_test=False, cpu_test=False, memory_test=False,
                  is_release_build=False, debug_mode=False, post_startup_delay=None,
                  interrupt_handler=None, e10s=True, enable_webrender=False, **kwargs):
 
         # Override the magic --host HOST_IP with the value of the environment variable.
@@ -115,16 +111,18 @@ either Raptor or browsertime."""
             'e10s': e10s,
             'enable_webrender': enable_webrender,
         }
         # We can never use e10s on fennec
         if self.config['app'] == 'fennec':
             self.config['e10s'] = False
 
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
+        self.raptor_webext = None
+        self.control_server = None
         self.playback = None
         self.benchmark = None
         self.benchmark_port = 0
         self.gecko_profiler = None
         self.post_startup_delay = post_startup_delay
         self.device = None
         self.profile_class = profile_class or app
         self.firefox_android_apps = FIREFOX_ANDROID_APPS
@@ -138,114 +136,32 @@ either Raptor or browsertime."""
             self.post_startup_delay = min(self.post_startup_delay, 3000)
             LOG.info("debug-mode enabled, reducing post-browser startup pause to %d ms"
                      % self.post_startup_delay)
 
         LOG.info("main raptor init, config is: %s" % str(self.config))
 
         # setup the control server
         self.results_handler = RaptorResultsHandler(self.config)
+        self.start_control_server()
 
         self.build_browser_profile()
 
-    def build_browser_profile(self):
-        self.profile = create_profile(self.profile_class)
-
-        # Merge extra profile data from testing/profiles
-        with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
-            base_profiles = json.load(fh)['raptor']
-
-        for profile in base_profiles:
-            path = os.path.join(self.profile_data_dir, profile)
-            LOG.info("Merging profile: {}".format(path))
-            self.profile.merge(path)
-
-        # share the profile dir with the config and the control server
-        self.config['local_profile_dir'] = self.profile.profile
-
     @property
     def profile_data_dir(self):
         if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
             return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
         if build:
             return os.path.join(build.topsrcdir, 'testing', 'profiles')
         return os.path.join(here, 'profile_data')
 
-    @abstractmethod
     def check_for_crashes(self):
-        pass
-
-    @abstractmethod
-    def run_test_setup(self, test):
-        LOG.info("starting test: %s" % test['name'])
-
-    def run_tests(self, tests, test_names):
-        try:
-            for test in tests:
-                try:
-                    self.run_test(test, timeout=int(test.get('page_timeout')))
-                finally:
-                    self.run_test_teardown(test)
-
-            return self.process_results(test_names)
-
-        finally:
-            self.clean_up()
-
-    @abstractmethod
-    def run_test(self, test, timeout):
-        raise NotImplementedError()
-
-    @abstractmethod
-    def run_test_teardown(self, test):
-        self.check_for_crashes()
-
-        # gecko profiling symbolication
-        if self.config['gecko_profile'] is True:
-            self.gecko_profiler.symbolicate()
-            # clean up the temp gecko profiling folders
-            LOG.info("cleaning up after gecko profiling")
-            self.gecko_profiler.clean()
-
-    def process_results(self, test_names):
-        # when running locally output results in build/raptor.json; when running
-        # in production output to a local.json to be turned into tc job artifact
-        if self.config.get('run_local', False):
-            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
-                raptor_json_path = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
-                                                'testing', 'mozharness', 'build', 'raptor.json')
-            else:
-                raptor_json_path = os.path.join(here, 'raptor.json')
-        else:
-            raptor_json_path = os.path.join(os.getcwd(), 'local.json')
-
-        self.config['raptor_json_path'] = raptor_json_path
-        return self.results_handler.summarize_and_output(self.config, test_names)
-
-    @abstractmethod
-    def clean_up(self):
-        pass
-
-    def get_page_timeout_list(self):
-        return self.results_handler.page_timeout_list
-
-
-class Raptor(Perftest):
-    """Container class for Raptor"""
-
-    def __init__(self, *args, **kwargs):
-        super(Raptor, self).__init__(*args, **kwargs)
-
-        self.raptor_webext = None
-        self.control_server = None
-        self.start_control_server()
+        raise NotImplementedError
 
     def run_test_setup(self, test):
-        super(Raptor, self).run_test_setup(test)
-
         LOG.info("starting raptor test: %s" % test['name'])
         LOG.info("test settings: %s" % str(test))
         LOG.info("raptor config: %s" % str(self.config))
 
         if test.get('type') == "benchmark":
             self.serve_benchmark_source(test)
 
         gen_test_config(
@@ -320,32 +236,50 @@ class Raptor(Perftest):
             # in debug-mode we leave the browser running (require manual shutdown)
             if not self.debug_mode:
                 elapsed_time += 1
                 if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                     self.control_server.wait_for_quit()
                     raise RuntimeError("Test failed to finish. "
                                        "Application timed out after {} seconds".format(timeout))
 
-    def run_test_teardown(self, test):
-        super(Raptor, self).run_test_teardown(test)
+    def run_test_teardown(self):
+        self.check_for_crashes()
 
         if self.playback is not None:
             self.playback.stop()
 
         self.remove_raptor_webext()
 
+        # gecko profiling symbolication
+        if self.config['gecko_profile'] is True:
+            self.gecko_profiler.symbolicate()
+            # clean up the temp gecko profiling folders
+            LOG.info("cleaning up after gecko profiling")
+            self.gecko_profiler.clean()
+
     def set_browser_test_prefs(self, raw_prefs):
         # add test specific preferences
         LOG.info("setting test-specific Firefox preferences")
         self.profile.set_preferences(json.loads(raw_prefs))
 
     def build_browser_profile(self):
-        super(Raptor, self).build_browser_profile(self)
+        self.profile = create_profile(self.profile_class)
+
+        # Merge extra profile data from testing/profiles
+        with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
+            base_profiles = json.load(fh)['raptor']
 
+        for profile in base_profiles:
+            path = os.path.join(self.profile_data_dir, profile)
+            LOG.info("Merging profile: {}".format(path))
+            self.profile.merge(path)
+
+        # share the profile dir with the config and the control server
+        self.config['local_profile_dir'] = self.profile.profile
         self.control_server.user_profile = self.profile
 
     def start_control_server(self):
         self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
         self.control_server.start()
 
         if self.config['enable_control_server_wait']:
             self.control_server_wait_set('webext_status/__raptor_shutdownBrowser')
@@ -508,19 +442,35 @@ class Raptor(Perftest):
         upload_dir = os.getenv('MOZ_UPLOAD_DIR')
         if not upload_dir:
             LOG.critical("Profiling ignored because MOZ_UPLOAD_DIR was not set")
         else:
             self.gecko_profiler = GeckoProfile(upload_dir,
                                                self.config,
                                                test)
 
+    def process_results(self, test_names):
+        # when running locally output results in build/raptor.json; when running
+        # in production output to a local.json to be turned into tc job artifact
+        if self.config.get('run_local', False):
+            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
+                raptor_json_path = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
+                                                'testing', 'mozharness', 'build', 'raptor.json')
+            else:
+                raptor_json_path = os.path.join(here, 'raptor.json')
+        else:
+            raptor_json_path = os.path.join(os.getcwd(), 'local.json')
+
+        self.config['raptor_json_path'] = raptor_json_path
+        return self.results_handler.summarize_and_output(self.config, test_names)
+
+    def get_page_timeout_list(self):
+        return self.results_handler.page_timeout_list
+
     def clean_up(self):
-        super(Raptor, self).clean_up()
-
         if self.config['enable_control_server_wait']:
             self.control_server_wait_clear('all')
 
         self.control_server.stop()
         LOG.info("finished")
 
     def control_server_wait_set(self, state):
         response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
@@ -639,49 +589,53 @@ class RaptorDesktop(Raptor):
             # now start the browser/app under test
             self.launch_desktop_browser(test)
 
             # set our control server flag to indicate we are running the browser/app
             self.control_server._finished = False
 
             self.wait_for_test_finish(test, timeout)
 
+        self.run_test_teardown()
+
     def __run_test_warm(self, test, timeout):
         self.run_test_setup(test)
 
-        if test.get('playback') is not None:
-            self.start_playback(test)
+        try:
+            if test.get('playback') is not None:
+                self.start_playback(test)
 
-        if self.config['host'] not in ('localhost', '127.0.0.1'):
-            self.delete_proxy_settings_from_profile()
+            if self.config['host'] not in ('localhost', '127.0.0.1'):
+                self.delete_proxy_settings_from_profile()
+
+            # start the browser/app under test
+            self.launch_desktop_browser(test)
 
-        # start the browser/app under test
-        self.launch_desktop_browser(test)
+            # set our control server flag to indicate we are running the browser/app
+            self.control_server._finished = False
+
+            self.wait_for_test_finish(test, timeout)
 
-        # set our control server flag to indicate we are running the browser/app
-        self.control_server._finished = False
+        finally:
+            self.run_test_teardown()
 
-        self.wait_for_test_finish(test, timeout)
-
-    def run_test_teardown(self, test):
+    def run_test_teardown(self):
         # browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
         if not self.debug_mode:
             if self.runner.is_running():
                 self.runner.stop()
         else:
             # in debug mode, and running locally, leave the browser running
             if self.config['run_local']:
                 LOG.info("* debug-mode enabled - please shutdown the browser manually...")
                 self.runner.wait(timeout=None)
 
-        super(RaptorDesktop, self).run_test_teardown(test)
+        super(RaptorDesktop, self).run_test_teardown()
 
     def check_for_crashes(self):
-        super(RaptorDesktop, self).check_for_crashes()
-
         try:
             self.runner.check_for_crashes()
         except NotImplementedError:  # not implemented for Chrome
             pass
 
     def clean_up(self):
         self.runner.stop()
 
@@ -969,21 +923,21 @@ class RaptorAndroid(Raptor):
         return super(RaptorAndroid, self).run_tests(tests, test_names)
 
     def run_test_setup(self, test):
         super(RaptorAndroid, self).run_test_setup(test)
 
         is_benchmark = test.get('type') == "benchmark"
         self.set_reverse_ports(is_benchmark=is_benchmark)
 
-    def run_test_teardown(self, test):
+    def run_test_teardown(self):
         LOG.info('removing reverse socket connections')
         self.device.remove_socket_connections('reverse')
 
-        super(RaptorAndroid, self).run_test_teardown(test)
+        super(RaptorAndroid, self).run_test_teardown()
 
     def run_test(self, test, timeout):
         # tests will be run warm (i.e. NO browser restart between page-cycles)
         # unless otheriwse specified in the test INI by using 'cold = true'
         try:
 
             if self.config['power_test']:
                 # gather OS baseline data
@@ -1001,16 +955,17 @@ class RaptorAndroid(Raptor):
                 self.__run_test_warm(test, timeout)
 
         except SignalHandlerException:
             self.device.stop_application(self.config['binary'])
 
         finally:
             if self.config['power_test']:
                 finish_android_power_test(self, test['name'])
+            self.run_test_teardown()
 
     def __run_test_cold(self, test, timeout):
         '''
         Run the Raptor test but restart the entire browser app between page-cycles.
 
         Note: For page-load tests, playback will only be started once - at the beginning of all
         browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs
         in the browser profile so those will need to be set again in each new profile/cycle.
@@ -1139,23 +1094,20 @@ class RaptorAndroid(Raptor):
         self.wait_for_test_finish(test, timeout)
 
         # in debug mode, and running locally, leave the browser running
         if self.debug_mode and self.config['run_local']:
             LOG.info("* debug-mode enabled - please shutdown the browser manually...")
             self.runner.wait(timeout=None)
 
     def check_for_crashes(self):
-        super(RaptorAndroid, self).check_for_crashes()
-
         if not self.app_launched:
             LOG.info("skipping check_for_crashes: application has not been launched")
             return
         self.app_launched = False
-
         # Turn off verbose to prevent logcat from being inserted into the main log.
         verbose = self.device._verbose
         self.device._verbose = False
         logcat = self.device.get_logcat()
         self.device._verbose = verbose
         if logcat:
             if mozcrash.check_for_java_exception(logcat, "raptor"):
                 return