Bug 1501809 - Fix Raptor tp6 'main application timed out'; r=jmaher
authorRob Wood <rwood@mozilla.com>
Wed, 24 Oct 2018 22:30:50 +0000
changeset 491343 8c16d6956679cac8745372056217ddb257f05171
parent 491342 5e471c480b8a6ba4939a6c159d86a687a24bb5d8
child 491344 f62d1135e3d83f1d73e4cb5a6e90b3200673d89c
push id247
push userfmarier@mozilla.com
push dateSat, 27 Oct 2018 01:06:44 +0000
reviewersjmaher
bugs1501809
milestone65.0a1
Bug 1501809 - Fix Raptor tp6 'main application timed out'; r=jmaher Differential Revision: https://phabricator.services.mozilla.com/D9714
testing/raptor/raptor/gen_test_config.py
testing/raptor/raptor/raptor.py
testing/raptor/webext/raptor/runner.js
--- a/testing/raptor/raptor/gen_test_config.py
+++ b/testing/raptor/raptor/gen_test_config.py
@@ -8,29 +8,30 @@ import os
 from mozlog import get_proxy_logger
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext', 'raptor')
 LOG = get_proxy_logger(component="raptor-gen-test-config")
 
 
-def gen_test_config(browser, test, cs_port, b_port=0):
+def gen_test_config(browser, test, cs_port, post_startup_delay, b_port=0):
     LOG.info("writing test settings into background js, so webext can get it")
 
     data = """// this file is auto-generated by raptor, do not edit directly
 function getTestConfig() {
     return {"browser": "%s",
             "cs_port": "%d",
             "test_name": "%s",
             "test_settings_url": "http://127.0.0.1:%d/%s.json",
+            "post_startup_delay": "%s",
             "benchmark_port": "%d"};
 }
 
-""" % (browser, cs_port, test, cs_port, test, b_port)
+""" % (browser, cs_port, test, cs_port, test, post_startup_delay, b_port)
 
     webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
 
     file = open(webext_background_script, "w")
     file.write(data)
     file.close()
 
     LOG.info("finished writing test config to %s" % webext_background_script)
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -49,16 +49,17 @@ class Raptor(object):
         self.config['processor'] = mozinfo.processor
         self.config['run_local'] = run_local
         self.config['obj_path'] = obj_path
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.log = get_default_logger(component='raptor-main')
         self.control_server = None
         self.playback = None
         self.benchmark = None
+        self.post_startup_delay = 30000  # raptor webext pause time after browser startup
 
         # Create the profile; for geckoview we want a firefox profile type
         if self.config['app'] == 'geckoview':
             self.profile = create_profile('firefox')
         else:
             self.profile = create_profile(self.config['app'])
 
         # Merge in base profiles
@@ -134,16 +135,17 @@ class Raptor(object):
             self.benchmark = Benchmark(self.config, test)
             benchmark_port = int(self.benchmark.port)
         else:
             benchmark_port = 0
 
         gen_test_config(self.config['app'],
                         test['name'],
                         self.control_server.port,
+                        self.post_startup_delay,
                         benchmark_port)
 
         # for android we must make the benchmarks server available to the device
         if self.config['app'] == "geckoview":
             self.log.info("making the raptor benchmarks server port available to device")
             _tcp_port = "tcp:%s" % benchmark_port
             self.device.create_socket_connection('reverse', _tcp_port, _tcp_port)
 
@@ -227,16 +229,19 @@ class Raptor(object):
 
             self.control_server.browser_proc = proc
 
         # set our cs flag to indicate we are running the browser/app
         self.control_server._finished = False
 
         # convert to seconds and account for page cycles
         timeout = int(timeout / 1000) * int(test['page_cycles'])
+        # account for the pause the raptor webext runner takes after browser startup
+        timeout += int(self.post_startup_delay / 1000)
+
         try:
             elapsed_time = 0
             while not self.control_server._finished:
                 time.sleep(1)
                 elapsed_time += 1
                 if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                     self.log.info("application timed out after {} seconds".format(timeout))
                     self.control_server.wait_for_quit()
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -12,17 +12,17 @@
 // inside the 'talos-pagesets' dir or 'heroes' dir (tarek's github
 // repo) or 'webkit/PerformanceTests' dir (for benchmarks) first run:
 // 'python -m SimpleHTTPServer 8081'
 // to serve out the pages that we want to prototype with. Also
 // update the manifest content 'matches' accordingly
 
 // when the browser starts this webext runner will start automatically; we
 // want to give the browser some time (ms) to settle before starting tests
-var postStartupDelay = 30000;
+var postStartupDelay;
 
 // delay (ms) between pageload cycles
 var pageCycleDelay = 1000;
 
 var newTabDelay = 1000;
 var reuseTab = false;
 
 var browserName;
@@ -409,16 +409,17 @@ function runner() {
   let config = getTestConfig();
   console.log("test name is: " + config.test_name);
   console.log("test settings url is: " + config.test_settings_url);
   testName = config.test_name;
   settingsURL = config.test_settings_url;
   csPort = config.cs_port;
   browserName = config.browser;
   benchmarkPort = config.benchmark_port;
+  postStartupDelay = config.post_startup_delay;
 
   getBrowserInfo().then(function() {
     getTestSettings().then(function() {
       if (testType == "benchmark") {
         // webkit benchmark type of test
         console.log("benchmark test start");
       } else if (testType == "pageload") {
         // standard pageload test