Bug 1544686 - [raptor] Allow CLI arguments to override Gecko profiler settings from test manifest. r=perftest reviewers,rwood
authorHenrik Skupin <mail@hskupin.info>
Mon, 29 Apr 2019 13:35:51 +0000
changeset 530614 50b7f764f4afc39d21c1b96c8c11505daaf6e4a4
parent 530613 201fb70499ef00904a0a728f5934b1ea66bddd7b
child 530615 7ba49ce135e667a8e97f00889a434965f331bfb8
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest
bugs1544686
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1544686 - [raptor] Allow CLI arguments to override Gecko profiler settings from test manifest. r=perftest reviewers,rwood Differential Revision: https://phabricator.services.mozilla.com/D28047
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/manifest.py
testing/raptor/test/test_manifest.py
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -97,20 +97,22 @@ def create_parser(mach_interface=False):
             help=argparse.SUPPRESS)
     add_arg('--geckoProfileEntries', dest="gecko_profile_entries", type=int,
             help=argparse.SUPPRESS)
     add_arg('--gecko-profile', action="store_true", dest="gecko_profile",
             help="Profile the run and output the results in $MOZ_UPLOAD_DIR. "
             "After talos is finished, profiler.firefox.com will be launched in Firefox "
             "so you can analyze the local profiles. To disable auto-launching of "
             "profiler.firefox.com set the DISABLE_PROFILE_LAUNCH=1 env var.")
-    add_arg('--gecko-profile-interval', dest='gecko_profile_interval', type=float,
-            help="How frequently to take samples (milliseconds)")
     add_arg('--gecko-profile-entries', dest="gecko_profile_entries", type=int,
-            help="How many samples to take with the profiler")
+            help='How many samples to take with the profiler')
+    add_arg('--gecko-profile-interval', dest='gecko_profile_interval', type=int,
+            help='How frequently to take samples (milliseconds)')
+    add_arg('--gecko-profile-thread', dest='gecko_profile_threads', action='append',
+            help='Name of the extra thread to be profiled')
     add_arg('--symbolsPath', dest='symbols_path',
             help="Path to the symbols for the build we are testing")
     add_arg('--page-cycles', dest="page_cycles", type=int,
             help="How many times to repeat loading the test page (for page load tests); "
                  "for benchmark tests this is how many times the benchmark test will be run")
     add_arg('--page-timeout', dest="page_timeout", type=int,
             help="How long to wait (ms) for one page_cycle to complete, before timing out")
     add_arg('--post-startup-delay',
@@ -147,17 +149,17 @@ def verify_options(parser, args):
 
     # if geckoProfile specified but not running on Firefox, not supported
     if args.gecko_profile is True and args.app != "firefox":
         parser.error("Gecko profiling is only supported when running raptor on Firefox!")
 
     # if --power-test specified, must be on geckoview/android with --host specified.
     if args.power_test:
         if args.app not in ["fennec", "geckoview", "refbrow", "fenix"] \
-          or args.host in ('localhost', '127.0.0.1'):
+                or args.host in ('localhost', '127.0.0.1'):
             parser.error("Power test is only supported when running raptor on Firefox Android "
                          "browsers when host is specified!")
 
     # if running on geckoview/refbrow/fenix, we need an activity and intent
     if args.app in ["geckoview", "refbrow", "fenix"]:
         if not args.activity:
             # if we have a default activity specified in APPS above, use that
             if APPS[args.app].get("default_activity", None) is not None:
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -164,30 +164,31 @@ def write_test_settings_json(args, test_
     if test_details.get("alert_threshold", None) is not None:
         test_settings['raptor-options']['alert_threshold'] = float(test_details['alert_threshold'])
 
     if test_details.get("screen_capture", None) is not None:
         test_settings['raptor-options']['screen_capture'] = test_details.get("screen_capture")
 
     # if Gecko profiling is enabled, write profiling settings for webext
     if test_details.get("gecko_profile", False):
-        threads = 'GeckoMain,Compositor'
+        threads = ['GeckoMain', 'Compositor']
 
         # With WebRender enabled profile some extra threads
         if os.getenv('MOZ_WEBRENDER') == '1':
-            threads = '{},Renderer,WR'.format(threads)
+            threads.extend(['Renderer', 'WR'])
 
         if test_details.get('gecko_profile_threads'):
-            threads = '{0},{1}'.format(threads, test_details.get('gecko_profile_threads'))
+            test_threads = filter(None, test_details['gecko_profile_threads'].split(','))
+            threads.extend(test_threads)
 
         test_settings['raptor-options'].update({
             'gecko_profile': True,
             'gecko_profile_entries': int(test_details.get('gecko_profile_entries')),
             'gecko_profile_interval': int(test_details.get('gecko_profile_interval')),
-            'gecko_profile_threads': threads,
+            'gecko_profile_threads': ','.join(set(threads)),
         })
 
     if test_details.get("newtab_per_cycle", None) is not None:
         test_settings['raptor-options']['newtab_per_cycle'] = \
             bool(test_details['newtab_per_cycle'])
 
     settings_file = os.path.join(tests_dir, test_details['name'] + '.json')
     try:
@@ -242,35 +243,60 @@ def get_raptor_test_list(args, oskey):
                 tests_to_run.append(next_test)
 
     # go through each test and set the page-cycles and page-timeout, and some config flags
     # the page-cycles value in the INI can be overriden when debug-mode enabled, when
     # gecko-profiling enabled, or when --page-cycles cmd line arg was used (that overrides all)
     for next_test in tests_to_run:
         LOG.info("configuring settings for test %s" % next_test['name'])
         max_page_cycles = next_test.get('page_cycles', 1)
+
         if args.gecko_profile is True:
             next_test['gecko_profile'] = True
-            LOG.info("gecko-profiling enabled")
+            LOG.info('gecko-profiling enabled')
             max_page_cycles = 3
+
+            if 'gecko_profile_entries' in args and args.gecko_profile_entries is not None:
+                next_test['gecko_profile_entries'] = str(args.gecko_profile_entries)
+                LOG.info('gecko-profiling entries set to %s' % args.gecko_profile_entries)
+
+            if 'gecko_profile_interval' in args and args.gecko_profile_interval is not None:
+                next_test['gecko_profile_interval'] = str(args.gecko_profile_interval)
+                LOG.info('gecko-profiling interval set to %s' % args.gecko_profile_interval)
+
+            if 'gecko_profile_threads' in args and args.gecko_profile_threads is not None:
+                threads = filter(None, next_test.get('gecko_profile_threads', '').split(','))
+                threads.extend(args.gecko_profile_threads)
+                next_test['gecko_profile_threads'] = ','.join(threads)
+                LOG.info('gecko-profiling extra threads %s' % args.gecko_profile_threads)
+
+        else:
+            # if the gecko profiler is not enabled, ignore all of it's settings
+            next_test.pop('gecko_profile_entries', None)
+            next_test.pop('gecko_profile_interval', None)
+            next_test.pop('gecko_profile_threads', None)
+
         if args.debug_mode is True:
             next_test['debug_mode'] = True
             LOG.info("debug-mode enabled")
             max_page_cycles = 2
+
         if args.page_cycles is not None:
             next_test['page_cycles'] = args.page_cycles
             LOG.info("set page-cycles to %d as specified on cmd line" % args.page_cycles)
         else:
             if int(next_test.get('page_cycles', 1)) > max_page_cycles:
                 next_test['page_cycles'] = max_page_cycles
                 LOG.info("page-cycles set to %d" % next_test['page_cycles'])
+
         # if --page-timeout was provided on the command line, use that instead of INI
         if args.page_timeout is not None:
             LOG.info("setting page-timeout to %d as specified on cmd line" % args.page_timeout)
             next_test['page_timeout'] = args.page_timeout
+
         # if --browser-cycles was provided on the command line, use that instead of INI
         if args.browser_cycles is not None:
             LOG.info("setting browser-cycles to %d as specified on cmd line" % args.browser_cycles)
             next_test['browser_cycles'] = args.browser_cycles
 
         if next_test.get("cold", "false") == "true":
             # when running in cold mode, set browser-cycles to the page-cycles value; as we want
             # the browser to restart between page-cycles; and set page-cycles to 1 as we only
--- a/testing/raptor/test/test_manifest.py
+++ b/testing/raptor/test/test_manifest.py
@@ -187,26 +187,79 @@ def test_get_raptor_test_list_geckoview(
                        test="raptor-unity-webgl",
                        browser_cycles=1)
 
     test_list = get_raptor_test_list(args, mozinfo.os)
     assert len(test_list) == 1
     assert test_list[0]['name'] == 'raptor-unity-webgl-geckoview'
 
 
-def test_get_raptor_test_list_gecko_profiling(create_args):
+def test_get_raptor_test_list_gecko_profiling_enabled(create_args):
     args = create_args(test="raptor-tp6-google-firefox",
                        gecko_profile=True,
                        browser_cycles=1)
 
     test_list = get_raptor_test_list(args, mozinfo.os)
     assert len(test_list) == 1
     assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
     assert test_list[0]['gecko_profile'] is True
-    assert test_list[0]['page_cycles'] == 3
+    assert test_list[0].get('gecko_profile_entries') == '14000000'
+    assert test_list[0].get('gecko_profile_interval') == '1'
+    assert test_list[0].get('gecko_profile_threads') is None
+
+
+def test_get_raptor_test_list_gecko_profiling_enabled_args_override(create_args):
+    args = create_args(test="raptor-tp6-google-firefox",
+                       gecko_profile=True,
+                       gecko_profile_entries=42,
+                       gecko_profile_interval=100,
+                       gecko_profile_threads=['Foo'],
+                       browser_cycles=1)
+
+    test_list = get_raptor_test_list(args, mozinfo.os)
+    assert len(test_list) == 1
+    assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
+    assert test_list[0]['gecko_profile'] is True
+    assert test_list[0]['gecko_profile_entries'] == '42'
+    assert test_list[0]['gecko_profile_interval'] == '100'
+    assert test_list[0]['gecko_profile_threads'] == 'Foo'
+
+
+def test_get_raptor_test_list_gecko_profiling_disabled(create_args):
+    args = create_args(test="raptor-tp6-google-firefox",
+                       gecko_profile=False,
+                       gecko_profile_entries=42,
+                       gecko_profile_interval=100,
+                       gecko_profile_threads=['Foo'],
+                       browser_cycles=1)
+
+    test_list = get_raptor_test_list(args, mozinfo.os)
+    assert len(test_list) == 1
+    assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
+    assert test_list[0].get('gecko_profile') is None
+    assert test_list[0].get('gecko_profile_entries') is None
+    assert test_list[0].get('gecko_profile_interval') is None
+    assert test_list[0].get('gecko_profile_threads') is None
+
+
+def test_get_raptor_test_list_gecko_profiling_disabled_args_override(create_args):
+    args = create_args(test="raptor-tp6-google-firefox",
+                       gecko_profile=False,
+                       gecko_profile_entries=42,
+                       gecko_profile_interval=100,
+                       gecko_profile_threads=['Foo'],
+                       browser_cycles=1)
+
+    test_list = get_raptor_test_list(args, mozinfo.os)
+    assert len(test_list) == 1
+    assert test_list[0]['name'] == 'raptor-tp6-google-firefox'
+    assert test_list[0].get('gecko_profile') is None
+    assert test_list[0].get('gecko_profile_entries') is None
+    assert test_list[0].get('gecko_profile_interval') is None
+    assert test_list[0].get('gecko_profile_threads') is None
 
 
 def test_get_raptor_test_list_debug_mode(create_args):
     args = create_args(test="raptor-tp6-google-firefox",
                        debug_mode=True,
                        browser_cycles=1)
 
     test_list = get_raptor_test_list(args, mozinfo.os)