author | Rob Wood <rwood@mozilla.com> |
Wed, 04 Dec 2019 21:23:35 +0000 | |
changeset 505557 | ebc95c785899006c401fd36b5eae32b9b55eeae2 |
parent 505556 | ae2503b461f3c46365f1caf18781a9515fa388d6 |
child 505558 | bd5c387d7822ca3f02ad39ad9741f1e0c722c3c4 |
push id | 102340 |
push user | rwood@mozilla.com |
push date | Wed, 04 Dec 2019 22:29:19 +0000 |
treeherder | autoland@ebc95c785899 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | perftest-reviewers, ahal, sparky, jlorenzo |
bugs | 1590561 |
milestone | 73.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/taskcluster/ci/test/raptor.yml +++ b/taskcluster/ci/test/raptor.yml @@ -978,36 +978,38 @@ raptor-jetstream2-firefox-profiling: mozharness: extra-options: - --test=raptor-jetstream2 - --gecko-profile fetches: fetch: - jetstream2 -browsertime-tp6-1: - description: "Raptor (browsertime) tp6-1" - raptor-test: raptor-tp6-1 - run-on-projects: ['mozilla-central','integration', 'try'] +browsertime-tp6: + description: "Raptor (browsertime) tp6 page-load tests" + raptor-test: tp6 + raptor-subtests: ['amazon', 'bing-search', 'facebook', 'google-search', + 'google-slides', 'wikipedia', 'yahoo-news', 'youtube'] + run-on-projects: ['mozilla-central', 'integration', 'try'] apps: ['firefox', 'chrome'] variants: by-app: firefox: ['fission'] default: [] pageload: by-app: firefox: 'both' chrome: 'cold' default: 'warm' limit-platforms: by-app: chrome: - .*shippable[^-qr].* default: [] - treeherder-symbol: Btime(tp6-1) + treeherder-symbol: Btime(tp6) max-run-time: 4000 tier: 3 condprof: True run-visual-metrics: by-app: chrome: false default: true mozharness:
--- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -305,17 +305,17 @@ raptor-fetch-geckoview: raptor-fetch-chromium: - raptor-unity-webgl-chromium - raptor-wasm-misc-chromium - raptor-assorted-dom-chromium - raptor-jetstream2-chromium browsertime: - - browsertime-tp6-1 + - browsertime-tp6 - browsertime-speedometer # Raptor desktop power tests # (requires Intel Power Gadget to be pre-installed) raptor-firefox-power: - raptor-youtube-playback-v9-power-firefox - raptor-youtube-playback-h264-power-firefox
--- a/taskcluster/taskgraph/transforms/raptor.py +++ b/taskcluster/taskgraph/transforms/raptor.py @@ -23,16 +23,17 @@ transforms = TransformSequence() raptor_description_schema = Schema({ # Raptor specific configs. Optional('apps'): optionally_keyed_by( 'test-platform', [basestring] ), Optional('raptor-test'): basestring, + Optional('raptor-subtests'): [basestring], Optional('activity'): optionally_keyed_by( 'app', basestring ), Optional('binary-path'): optionally_keyed_by( 'app', basestring ), @@ -152,17 +153,25 @@ def split_pageload(config, tests): continue if pageload == 'both': orig = deepcopy(test) yield orig assert 'raptor-test' in test test['description'] += " using cold pageload" - test['raptor-test'] += '-cold' + + # for raptor-webext to run cold we just call the corresponding '-cold' test name; but + # for raptor browsertime we leave the raptor test name as/is and will set the '--cold' + # command line argument instead via settting test['cold'] to true + if test['test-name'].startswith('browsertime-tp6'): + test['cold'] = True + else: + test['raptor-test'] += '-cold' + test['max-run-time'] = 3000 test['test-name'] += '-cold' test['try-name'] += '-cold' group, symbol = split_symbol(test['treeherder-symbol']) symbol += '-c' test['treeherder-symbol'] = join_symbol(group, symbol) yield test @@ -195,27 +204,66 @@ def build_condprof_tests(config, tests): condprof_test['try-name'] += '-condprof' condprof_test['test-name'] += '-condprof' condprof_test['treeherder-symbol'] = join_symbol(group, symbol) yield condprof_test @transforms.add +def split_browsertime_page_load_by_url(config, tests): + + for test in tests: + + # for tests that have 'raptor-subtests' listed, we want to create a separate + # test job for every subtest (i.e. split out each page-load URL into its own job) + subtests = test.pop('raptor-subtests', None) + if not subtests: + yield test + continue + + chunk_number = 0 + + for subtest in subtests: + + # create new test job + chunked = deepcopy(test) + + # only run the subtest/single URL + chunked['test-name'] += "-{}".format(subtest) + chunked['try-name'] += "-{}".format(subtest) + chunked['raptor-test'] = subtest + + # set treeherder symbol and description + chunk_number += 1 + group, symbol = split_symbol(test['treeherder-symbol']) + symbol += "-{}".format(chunk_number) + chunked['treeherder-symbol'] = join_symbol(group, symbol) + chunked['description'] += "-{}".format(subtest) + + yield chunked + + +@transforms.add def add_extra_options(config, tests): for test in tests: extra_options = test.setdefault('mozharness', {}).setdefault('extra-options', []) if test.pop('run-visual-metrics', False): extra_options.append('--browsertime-video') test['attributes']['run-visual-metrics'] = True if 'app' in test: extra_options.append('--app={}'.format(test.pop('app'))) + # for browsertime tp6 cold page-load jobs we need to set the '--cold' cmd line arg + if test['test-name'].startswith('browsertime-tp6'): + if test.pop('cold', False) is True: + extra_options.append('--cold') + if 'activity' in test: extra_options.append('--activity={}'.format(test.pop('activity'))) if 'binary-path' in test: extra_options.append('--binary-path={}'.format(test.pop('binary-path'))) if 'raptor-test' in test: extra_options.append('--test={}'.format(test.pop('raptor-test')))
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -243,16 +243,22 @@ class Raptor(TestingMixin, MercurialScri "help": "Enable Fission (site isolation) in Gecko.", }], [["--setpref"], { "action": "append", "dest": "extra_prefs", "default": [], "help": "A preference to set. Must be a key-value pair separated by a ':'." }], + [["--cold"], { + "action": "store_true", + "dest": "cold", + "default": False, + "help": "Enable cold page-load for browsertime tp6", + }], ] + testing_config_options + \ copy.deepcopy(code_coverage_config_options) + \ browsertime_options def __init__(self, **kwargs): kwargs.setdefault('config_options', self.config_options) kwargs.setdefault('all_actions', ['clobber', @@ -502,16 +508,18 @@ class Raptor(TestingMixin, MercurialScri if self.config.get('is_release_build', False): options.extend(['--is-release-build']) if self.config.get('power_test', False): options.extend(['--power-test']) if self.config.get('memory_test', False): options.extend(['--memory-test']) if self.config.get('cpu_test', False): options.extend(['--cpu-test']) + if self.config.get('cold', False): + options.extend(['--cold']) if self.config.get('enable_webrender', False): options.extend(['--enable-webrender']) if self.config.get('with_conditioned_profile', False): options.extend(['--with-conditioned-profile']) if self.config.get('enable_fission', False): options.extend(['--enable-fission']) if self.config.get('extra_prefs'): options.extend(['--setpref={}'.format(i) for i in self.config.get('extra_prefs')])
--- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -151,16 +151,19 @@ def create_parser(mach_interface=False): help="Flag which indicates if Raptor is running locally or in production") add_arg('--obj-path', dest="obj_path", default=None, help="Browser-build obj_path (received when running in production)") add_arg('--noinstall', dest="noinstall", default=False, action="store_true", help="Flag which indicates if Raptor should not offer to install Android APK.") add_arg('--installerpath', dest="installerpath", default=None, type=str, help="Location where Android browser APK was extracted to before installation.") + # for browsertime jobs, cold page load is determined by a '--cold' cmd line argument + add_arg('--cold', dest="cold", action="store_true", + help="Enable cold page-load for browsertime tp6") # Arguments for invoking browsertime. add_arg('--browsertime', dest='browsertime', default=False, action="store_true", help="Whether to use browsertime to execute pageload tests") add_arg('--browsertime-node', dest='browsertime_node', help="path to Node.js executable") add_arg('--browsertime-browsertimejs', dest='browsertime_browsertimejs', help="path to browsertime.js script") add_arg('--browsertime-chromedriver', dest='browsertime_chromedriver',
--- a/testing/raptor/raptor/manifest.py +++ b/testing/raptor/raptor/manifest.py @@ -5,17 +5,17 @@ from __future__ import absolute_import import json import os from six.moves.urllib.parse import parse_qs, urlsplit, urlunsplit, urlencode, unquote from logger.logger import RaptorLogger from manifestparser import TestManifest -from utils import transform_platform +from utils import transform_platform, transform_subtest from constants.raptor_tests_constants import YOUTUBE_PLAYBACK_MEASURE here = os.path.abspath(os.path.dirname(__file__)) raptor_ini = os.path.join(here, 'raptor.ini') tests_dir = os.path.join(here, 'tests') LOG = RaptorLogger(component='raptor-manifest') LIVE_SITE_TIMEOUT_MULTIPLIER = 1.2 @@ -308,16 +308,25 @@ def get_raptor_test_list(args, oskey): # go through each test and set the page-cycles and page-timeout, and some config flags # the page-cycles value in the INI can be overriden when debug-mode enabled, when # gecko-profiling enabled, or when --page-cycles cmd line arg was used (that overrides all) for next_test in tests_to_run: LOG.info("configuring settings for test %s" % next_test['name']) max_page_cycles = next_test.get('page_cycles', 1) + # if using playback, the playback recording info may need to be transformed + if next_test.get('playback') is not None: + next_test['playback_pageset_manifest'] = \ + transform_subtest(next_test['playback_pageset_manifest'], + next_test['name']) + next_test['playback_recordings'] = \ + transform_subtest(next_test['playback_recordings'], + next_test['name']) + if args.gecko_profile is True: next_test['gecko_profile'] = True LOG.info('gecko-profiling enabled') max_page_cycles = 3 if 'gecko_profile_entries' in args and args.gecko_profile_entries is not None: next_test['gecko_profile_entries'] = str(args.gecko_profile_entries) LOG.info('gecko-profiling entries set to %s' % args.gecko_profile_entries) @@ -356,17 +365,30 @@ def get_raptor_test_list(args, oskey): LOG.info("setting page-timeout to %d as specified on cmd line" % args.page_timeout) next_test['page_timeout'] = args.page_timeout # if --browser-cycles was provided on the command line, use that instead of INI if args.browser_cycles is not None: LOG.info("setting browser-cycles to %d as specified on cmd line" % args.browser_cycles) next_test['browser_cycles'] = args.browser_cycles - if next_test.get("cold", "false") == "true": + # for browsertime jobs, cold page-load mode is determined by command line argument; for + # raptor-webext jobs cold page-load is determined by the 'cold' key in test manifest INI + _running_cold = False + if args.browsertime is True: + if args.cold is True: + _running_cold = True + else: + # running warm page-load so ignore browser-cycles if it was provided (set to 1) + next_test['browser_cycles'] = 1 + else: + if next_test.get("cold", "false") == "true": + _running_cold = True + + if _running_cold: # when running in cold mode, set browser-cycles to the page-cycles value; as we want # the browser to restart between page-cycles; and set page-cycles to 1 as we only # want 1 single page-load for every browser-cycle next_test['cold'] = True next_test['expected_browser_cycles'] = int(next_test['browser_cycles']) next_test['page_cycles'] = 1 # also ensure '-cold' is in test name so perfherder results indicate warm cold-load if "-cold" not in next_test['name']:
--- a/testing/raptor/raptor/raptor.ini +++ b/testing/raptor/raptor/raptor.ini @@ -136,8 +136,11 @@ # raptor scenario tests [include:tests/scenario/raptor-scn-power-idle.ini] [include:tests/scenario/raptor-scn-power-idle-bg.ini] # raptor unit tests # this is required for the manifest unit tests [include:tests/unittests/raptor-tp6-unittest.ini] + +# raptor-browsertime desktop tests +[include:tests/tp6/desktop/browsertime-tp6.ini]
--- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -620,16 +620,17 @@ class Browsertime(Perftest): userjsfile.writelines(lines) except Exception as e: LOG.critical("Exception {} while removing mozprofile delimiters".format(e)) def set_browser_test_prefs(self, raw_prefs): # add test specific preferences LOG.info("setting test-specific Firefox preferences") self.profile.set_preferences(json.loads(raw_prefs)) + self.remove_mozprofile_delimiters_from_profile() def run_test_setup(self, test): super(Browsertime, self).run_test_setup(test) if test.get('type') == "benchmark": # benchmark-type tests require the benchmark test to be served out self.benchmark = Benchmark(self.config, test) test['test_url'] = test['test_url'].replace('<host>', self.benchmark.host)
new file mode 100644 --- /dev/null +++ b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# raptor-browsertime tp6 desktop page-load test specification + +[DEFAULT] +alert_on = fcp, loadtime +alert_threshold = 2.0 +apps = firefox, chrome +browser_cycles = 25 +gecko_profile_entries = 14000000 +gecko_profile_interval = 1 +lower_is_better = true +measure = fnbpaint, fcp, dcf, loadtime +page_cycles = 25 +page_timeout = 60000 +playback = mitmproxy +playback_pageset_manifest = mitm4-linux-firefox-{subtest}.manifest +playback_recordings = {subtest}.mp +type = pageload +unit = ms +use_live_sites = false + +[amazon] +test_url = https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1 + +[bing-search] +test_url = https://www.bing.com/search?q=barack+obama +playback_recordings = bing.mp + +[facebook] +test_url = https://www.facebook.com + +[google-search] +test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h + +[google-slides] +test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing + +[wikipedia] +test_url = https://en.wikipedia.org/wiki/Barack_Obama + +[yahoo-news] +test_url = https://www.yahoo.com/lifestyle/police-respond-noise-complaint-end-playing-video-games-respectful-tenants-002329963.html + +[youtube] +test_url = https://www.youtube.com
--- a/testing/raptor/raptor/utils.py +++ b/testing/raptor/raptor/utils.py @@ -44,16 +44,25 @@ def transform_platform(str_to_transform, if 'x86_64' in config_processor: str_to_transform = str_to_transform.replace('{x64}', '_x64') else: str_to_transform = str_to_transform.replace('{x64}', '') return str_to_transform +def transform_subtest(str_to_transform, subtest_name): + """Transform subtest name i.e. 'mitm4-linux-firefox-{subtest}.manifest' + transforms to 'mitm4-linux-firefox-amazon.manifest'.""" + if '{subtest}' not in str_to_transform: + return str_to_transform + + return str_to_transform.replace('{subtest}', subtest_name) + + def view_gecko_profile(ffox_bin): # automatically load the latest talos gecko-profile archive in profiler.firefox.com LOG_GECKO = RaptorLogger(component='raptor-view-gecko-profile') if sys.platform.startswith('win') and not ffox_bin.endswith(".exe"): ffox_bin = ffox_bin + ".exe" if not os.path.exists(ffox_bin):
--- a/testing/raptor/test/conftest.py +++ b/testing/raptor/test/conftest.py @@ -85,17 +85,19 @@ def create_args(): test='raptor-tp6-unittest', binary='path/to/binary', gecko_profile=False, debug_mode=False, page_cycles=None, page_timeout=None, test_url_params=None, host=None, - run_local=True) + run_local=True, + browsertime=True, + cold=False) def inner(**kwargs): for next_arg in kwargs: print(next_arg) print(kwargs[next_arg]) setattr(args, next_arg, kwargs[next_arg]) return args
--- a/testing/raptor/test/test_manifest.py +++ b/testing/raptor/test/test_manifest.py @@ -54,17 +54,16 @@ VALID_MANIFESTS = [{ 'alert_change_type': None, 'alert_on': None, 'playback': None, }, { # page load test for geckoview 'alert_threshold': 2.0, 'apps': 'geckoview', 'browser_cycles': 10, - 'cold': True, 'lower_is_better': False, 'manifest': 'valid_details_2', 'measure': 'fcp', 'page_cycles': 1, 'test_url': 'http://www.test-url/goes/here', 'type': 'pageload', 'unit': 'score', }, {