Bug 1491114 - Add ability to specify page_cycles and page_timeout on the command line; r=jmaher
authorRob Wood <rwood@mozilla.com>
Thu, 01 Nov 2018 19:54:37 +0000
changeset 444038 1dbaae27c584753c9d22f33265c76e66d044cb2e
parent 444037 0b39b9d22436f3f463c6c62c69e5ee1737fd2e60
child 444039 09e5201164c98469f6e50ddf2fe73a7e288123d8
push id109501
push userccoroiu@mozilla.com
push dateFri, 02 Nov 2018 05:19:20 +0000
treeherdermozilla-inbound@2be891b25f69 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1491114
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1491114 - Add ability to specify page_cycles and page_timeout on the command line; r=jmaher Differential Revision: https://phabricator.services.mozilla.com/D10599
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/manifest.py
testing/raptor/raptor/raptor.py
testing/raptor/test/test_cmdline.py
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -105,16 +105,28 @@ class Raptor(TestingMixin, MercurialScri
             "type": "int",
             "help": "The interval between samples taken by the profiler (milliseconds)"
         }],
         [["--gecko-profile-entries"], {
             "dest": "gecko_profile_entries",
             "type": "int",
             "help": "How many samples to take with the profiler"
         }],
+        [["--page-cycles"], {
+            "dest": "page_cycles",
+            "type": "int",
+            "help": "How many times to repeat loading the test page (for page load tests); "
+                    "for benchmark tests this is how many times the benchmark test will be run"
+        }],
+        [["--page-timeout"], {
+            "dest": "page_timeout",
+            "type": "int",
+            "help": "How long to wait (ms) for one page_cycle to complete, before timing out"
+        }],
+
     ] + testing_config_options + copy.deepcopy(code_coverage_config_options)
 
     def __init__(self, **kwargs):
         kwargs.setdefault('config_options', self.config_options)
         kwargs.setdefault('all_actions', ['clobber',
                                           'download-and-extract',
                                           'populate-webroot',
                                           'install-chrome',
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -32,16 +32,21 @@ def create_parser(mach_interface=False):
             "can analyze the local profiles. To disable auto-launching of perf-html.io "
             "set the DISABLE_PROFILE_LAUNCH=1 env var.")
     add_arg('--gecko-profile-interval', dest='gecko_profile_interval', type=float,
             help="How frequently to take samples (milliseconds)")
     add_arg('--gecko-profile-entries', dest="gecko_profile_entries", type=int,
             help="How many samples to take with the profiler")
     add_arg('--symbolsPath', dest='symbols_path',
             help="Path to the symbols for the build we are testing")
+    add_arg('--page-cycles', dest="page_cycles", type=int,
+            help="How many times to repeat loading the test page (for page load tests); "
+                 "for benchmark tests this is how many times the benchmark test will be run")
+    add_arg('--page-timeout', dest="page_timeout", type=int,
+            help="How long to wait (ms) for one page_cycle to complete, before timing out")
     if not mach_interface:
         add_arg('--branchName', dest="branch_name", default='',
                 help="Name of the branch we are testing on")
         add_arg('--run-local', dest="run_local", default=False, action="store_true",
                 help="Flag that indicates if raptor is running locally or in production")
         add_arg('--obj-path', dest="obj_path", default=None,
                 help="Browser build obj_path (received when running in production)")
 
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -61,16 +61,17 @@ def validate_test_ini(test_details):
 
     return valid_settings
 
 
 def write_test_settings_json(test_details, oskey):
     # write test settings json file with test details that the control
     # server will provide for the web ext
     test_url = transform_platform(test_details['test_url'], oskey)
+
     test_settings = {
         "raptor-options": {
             "type": test_details['type'],
             "test_url": test_url,
             "page_cycles": int(test_details['page_cycles'])
         }
     }
 
@@ -166,16 +167,28 @@ def get_raptor_test_list(args, oskey):
     if len(tests_to_run) == 0:
         _ini = args.test + ".ini"
         for next_test in available_tests:
             head, tail = os.path.split(next_test['manifest'])
             if tail == _ini:
                 # subtest comes from matching test ini file name, so add it
                 tests_to_run.append(next_test)
 
+    # if --page-cycles command line arg was provided, override the page_cycles value
+    # that was in the manifest/test INI with the command line arg value instead
+    if args.page_cycles is not None:
+        LOG.info("setting page-cycles to %d as specified on the command line" % args.page_cycles)
+        next_test['page_cycles'] = args.page_cycles
+
+    # if --page-timeout command line arg was provided, override the page_timeout value
+    # that was in the manifest/test INI with the command line arg value instead
+    if args.page_timeout is not None:
+        LOG.info("setting page-timeout to %d as specified on the command line" % args.page_timeout)
+        next_test['page_timeout'] = args.page_timeout
+
     # if geckoProfile is enabled, turn it on in test settings and limit pagecycles to 2
     if args.gecko_profile is True:
         for next_test in tests_to_run:
             next_test['gecko_profile'] = True
             if next_test['page_cycles'] > 2:
                 LOG.info("gecko profiling enabled, limiting pagecycles "
                          "to 2 for test %s" % next_test['name'])
                 next_test['page_cycles'] = 2
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -441,16 +441,17 @@ def main(args=sys.argv[1:]):
 
     raptor.start_control_server()
 
     for next_test in raptor_test_list:
         if 'page_timeout' not in next_test.keys():
             next_test['page_timeout'] = 120000
         if 'page_cycles' not in next_test.keys():
             next_test['page_cycles'] = 1
+
         raptor.run_test(next_test, timeout=int(next_test['page_timeout']))
 
     success = raptor.process_results()
     raptor.clean_up()
 
     if not success:
         # didn't get test results; test timed out or crashed, etc. we want job to fail
         LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found")
--- a/testing/raptor/test/test_cmdline.py
+++ b/testing/raptor/test/test_cmdline.py
@@ -5,17 +5,21 @@ import pytest
 
 import mozunit
 
 from argparse import ArgumentParser, Namespace
 from raptor.cmdline import verify_options
 
 
 def test_verify_options(filedir):
-    args = Namespace(app='firefox', binary='invalid/path', gecko_profile='False')
+    args = Namespace(app='firefox',
+                     binary='invalid/path',
+                     gecko_profile='False',
+                     page_cycles=1,
+                     page_timeout=60000)
     parser = ArgumentParser()
 
     with pytest.raises(SystemExit):
         verify_options(parser, args)
 
     args.binary = os.path.join(filedir, 'fake_binary.exe')
     verify_options(parser, args)  # assert no exception