Bug 1528236 - [raptor] Allow Raptor to run the tests all at once. r=perftest-reviewers,stephendonner,rwood
authorHenrik Skupin <mail@hskupin.info>
Tue, 14 May 2019 19:32:01 +0000
changeset 535707 e8d6d453337d96fb354184b2045e6b5b456a97b7
parent 535706 9eb4d019ee371b8b4d37c6e3f2f6ad582038c1dc
child 535708 09eea8c87cd551a490ac92ebdc99cbb2041f07ee
push id2082
push userffxbld-merge
push dateMon, 01 Jul 2019 08:34:18 +0000
treeherdermozilla-release@2fb19d0466d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, stephendonner, rwood
bugs1528236
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1528236 - [raptor] Allow Raptor to run the tests all at once. r=perftest-reviewers,stephendonner,rwood Differential Revision: https://phabricator.services.mozilla.com/D31047
testing/raptor/raptor/raptor.py
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -172,16 +172,58 @@ class Raptor(object):
         self.install_raptor_webext()
 
         if test.get("preferences") is not None:
             self.set_browser_test_prefs(test['preferences'])
 
         # if 'alert_on' was provided in the test INI, add to our config for results/output
         self.config['subtest_alert_on'] = test.get('alert_on')
 
+    def run_tests(self, tests, test_names):
+        try:
+            for test in tests:
+                self.run_test(test, timeout=int(test['page_timeout']))
+
+            return self.process_results(test_names)
+
+        finally:
+            self.clean_up()
+
+    def run_test(self, test, timeout=None):
+        raise NotImplementedError()
+
+    def wait_for_test_finish(self, test, timeout):
+        # convert timeout to seconds and account for page cycles
+        timeout = int(timeout / 1000) * int(test.get('page_cycles', 1))
+        # account for the pause the raptor webext runner takes after browser startup
+        # and the time an exception is propagated through the framework
+        timeout += (int(self.post_startup_delay / 1000) + 10)
+
+        # if geckoProfile enabled, give browser more time for profiling
+        if self.config['gecko_profile'] is True:
+            timeout += 5 * 60
+
+        elapsed_time = 0
+        while not self.control_server._finished:
+            if self.config['enable_control_server_wait']:
+                response = self.control_server_wait_get()
+                if response == 'webext_status/__raptor_shutdownBrowser':
+                    if self.config['memory_test']:
+                        generate_android_memory_profile(self, test['name'])
+                    self.control_server_wait_continue()
+            time.sleep(1)
+            # we only want to force browser-shutdown on timeout if not in debug mode;
+            # in debug-mode we leave the browser running (require manual shutdown)
+            if not self.debug_mode:
+                elapsed_time += 1
+                if elapsed_time > (timeout) - 5:  # stop 5 seconds early
+                    self.log.info("application timed out after {} seconds".format(timeout))
+                    self.control_server.wait_for_quit()
+                    break
+
     def run_test_teardown(self):
         self.check_for_crashes()
 
         if self.playback is not None:
             self.playback.stop()
 
         self.remove_raptor_webext()
 
@@ -326,45 +368,16 @@ class Raptor(object):
         upload_dir = os.getenv('MOZ_UPLOAD_DIR')
         if not upload_dir:
             self.log.critical("Profiling ignored because MOZ_UPLOAD_DIR was not set")
         else:
             self.gecko_profiler = GeckoProfile(upload_dir,
                                                self.config,
                                                test)
 
-    def wait_for_test_finish(self, test, timeout):
-        # convert timeout to seconds and account for page cycles
-        timeout = int(timeout / 1000) * int(test.get('page_cycles', 1))
-        # account for the pause the raptor webext runner takes after browser startup
-        # and the time an exception is propagated through the framework
-        timeout += (int(self.post_startup_delay / 1000) + 10)
-
-        # if geckoProfile enabled, give browser more time for profiling
-        if self.config['gecko_profile'] is True:
-            timeout += 5 * 60
-
-        elapsed_time = 0
-        while not self.control_server._finished:
-            if self.config['enable_control_server_wait']:
-                response = self.control_server_wait_get()
-                if response == 'webext_status/__raptor_shutdownBrowser':
-                    if self.config['memory_test']:
-                        generate_android_memory_profile(self, test['name'])
-                    self.control_server_wait_continue()
-            time.sleep(1)
-            # we only want to force browser-shutdown on timeout if not in debug mode;
-            # in debug-mode we leave the browser running (require manual shutdown)
-            if not self.debug_mode:
-                elapsed_time += 1
-                if elapsed_time > (timeout) - 5:  # stop 5 seconds early
-                    self.log.info("application timed out after {} seconds".format(timeout))
-                    self.control_server.wait_for_quit()
-                    break
-
     def process_results(self, test_names):
         # when running locally output results in build/raptor.json; when running
         # in production output to a local.json to be turned into tc job artifact
         if self.config.get('run_local', False):
             if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
                 raptor_json_path = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
                                                 'testing', 'mozharness', 'build', 'raptor.json')
             else:
@@ -1170,24 +1183,17 @@ def main(args=sys.argv[1:]):
 
     raptor.create_browser_profile()
     raptor.create_browser_handler()
     if type(raptor) == RaptorAndroid:
         # only Raptor Android supports device performance tuning
         raptor.tune_performance()
     raptor.start_control_server()
 
-    try:
-        for next_test in raptor_test_list:
-            raptor.run_test(next_test, timeout=int(next_test['page_timeout']))
-
-        success = raptor.process_results(raptor_test_names)
-
-    finally:
-        raptor.clean_up()
+    success = raptor.run_tests(raptor_test_list, raptor_test_names)
 
     if not success:
         # didn't get test results; test timed out or crashed, etc. we want job to fail
         LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found for %s" %
                      ', '.join(raptor_test_names))
         os.sys.exit(1)
 
     # if we have results but one test page timed out (i.e. one tp6 test page didn't load