Bug 1590561 - Use a single test INI and taskcluster configuration file for all raptor-browsertime desktop page-load tests r=perftest-reviewers,ahal,sparky,jlorenzo
authorRob Wood <rwood@mozilla.com>
Wed, 04 Dec 2019 21:23:35 +0000
changeset 505557 ebc95c785899006c401fd36b5eae32b9b55eeae2
parent 505556 ae2503b461f3c46365f1caf18781a9515fa388d6
child 505558 bd5c387d7822ca3f02ad39ad9741f1e0c722c3c4
push id102340
push userrwood@mozilla.com
push dateWed, 04 Dec 2019 22:29:19 +0000
treeherderautoland@ebc95c785899 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, ahal, sparky, jlorenzo
bugs1590561
milestone73.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1590561 - Use a single test INI and taskcluster configuration file for all raptor-browsertime desktop page-load tests r=perftest-reviewers,ahal,sparky,jlorenzo Differential Revision: https://phabricator.services.mozilla.com/D52185
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
taskcluster/taskgraph/transforms/raptor.py
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/manifest.py
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini
testing/raptor/raptor/utils.py
testing/raptor/test/conftest.py
testing/raptor/test/test_manifest.py
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -978,36 +978,38 @@ raptor-jetstream2-firefox-profiling:
     mozharness:
         extra-options:
             - --test=raptor-jetstream2
             - --gecko-profile
     fetches:
         fetch:
             - jetstream2
 
-browsertime-tp6-1:
-    description: "Raptor (browsertime) tp6-1"
-    raptor-test: raptor-tp6-1
-    run-on-projects: ['mozilla-central','integration', 'try']
+browsertime-tp6:
+    description: "Raptor (browsertime) tp6 page-load tests"
+    raptor-test: tp6
+    raptor-subtests: ['amazon', 'bing-search', 'facebook', 'google-search',
+                      'google-slides', 'wikipedia', 'yahoo-news', 'youtube']
+    run-on-projects: ['mozilla-central', 'integration', 'try']
     apps: ['firefox', 'chrome']
     variants:
         by-app:
             firefox: ['fission']
             default: []
     pageload:
         by-app:
             firefox: 'both'
             chrome: 'cold'
             default: 'warm'
     limit-platforms:
         by-app:
             chrome:
                 - .*shippable[^-qr].*
             default: []
-    treeherder-symbol: Btime(tp6-1)
+    treeherder-symbol: Btime(tp6)
     max-run-time: 4000
     tier: 3
     condprof: True
     run-visual-metrics:
         by-app:
             chrome: false
             default: true
     mozharness:
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -305,17 +305,17 @@ raptor-fetch-geckoview:
 
 raptor-fetch-chromium:
     - raptor-unity-webgl-chromium
     - raptor-wasm-misc-chromium
     - raptor-assorted-dom-chromium
     - raptor-jetstream2-chromium
 
 browsertime:
-    - browsertime-tp6-1
+    - browsertime-tp6
     - browsertime-speedometer
 
 # Raptor desktop power tests
 # (requires Intel Power Gadget to be pre-installed)
 raptor-firefox-power:
     - raptor-youtube-playback-v9-power-firefox
     - raptor-youtube-playback-h264-power-firefox
 
--- a/taskcluster/taskgraph/transforms/raptor.py
+++ b/taskcluster/taskgraph/transforms/raptor.py
@@ -23,16 +23,17 @@ transforms = TransformSequence()
 
 raptor_description_schema = Schema({
     # Raptor specific configs.
     Optional('apps'): optionally_keyed_by(
         'test-platform',
         [basestring]
     ),
     Optional('raptor-test'): basestring,
+    Optional('raptor-subtests'): [basestring],
     Optional('activity'): optionally_keyed_by(
         'app',
         basestring
     ),
     Optional('binary-path'): optionally_keyed_by(
         'app',
         basestring
     ),
@@ -152,17 +153,25 @@ def split_pageload(config, tests):
             continue
 
         if pageload == 'both':
             orig = deepcopy(test)
             yield orig
 
         assert 'raptor-test' in test
         test['description'] += " using cold pageload"
-        test['raptor-test'] += '-cold'
+
+        # for raptor-webext to run cold we just call the corresponding '-cold' test name; but
+        # for raptor browsertime we leave the raptor test name as/is and will set the '--cold'
+        # command line argument instead via settting test['cold'] to true
+        if test['test-name'].startswith('browsertime-tp6'):
+            test['cold'] = True
+        else:
+            test['raptor-test'] += '-cold'
+
         test['max-run-time'] = 3000
         test['test-name'] += '-cold'
         test['try-name'] += '-cold'
 
         group, symbol = split_symbol(test['treeherder-symbol'])
         symbol += '-c'
         test['treeherder-symbol'] = join_symbol(group, symbol)
         yield test
@@ -195,27 +204,66 @@ def build_condprof_tests(config, tests):
         condprof_test['try-name'] += '-condprof'
         condprof_test['test-name'] += '-condprof'
         condprof_test['treeherder-symbol'] = join_symbol(group, symbol)
 
         yield condprof_test
 
 
 @transforms.add
+def split_browsertime_page_load_by_url(config, tests):
+
+    for test in tests:
+
+        # for tests that have 'raptor-subtests' listed, we want to create a separate
+        # test job for every subtest (i.e. split out each page-load URL into its own job)
+        subtests = test.pop('raptor-subtests', None)
+        if not subtests:
+            yield test
+            continue
+
+        chunk_number = 0
+
+        for subtest in subtests:
+
+            # create new test job
+            chunked = deepcopy(test)
+
+            # only run the subtest/single URL
+            chunked['test-name'] += "-{}".format(subtest)
+            chunked['try-name'] += "-{}".format(subtest)
+            chunked['raptor-test'] = subtest
+
+            # set treeherder symbol and description
+            chunk_number += 1
+            group, symbol = split_symbol(test['treeherder-symbol'])
+            symbol += "-{}".format(chunk_number)
+            chunked['treeherder-symbol'] = join_symbol(group, symbol)
+            chunked['description'] += "-{}".format(subtest)
+
+            yield chunked
+
+
+@transforms.add
 def add_extra_options(config, tests):
     for test in tests:
         extra_options = test.setdefault('mozharness', {}).setdefault('extra-options', [])
 
         if test.pop('run-visual-metrics', False):
             extra_options.append('--browsertime-video')
             test['attributes']['run-visual-metrics'] = True
 
         if 'app' in test:
             extra_options.append('--app={}'.format(test.pop('app')))
 
+        # for browsertime tp6 cold page-load jobs we need to set the '--cold' cmd line arg
+        if test['test-name'].startswith('browsertime-tp6'):
+            if test.pop('cold', False) is True:
+                extra_options.append('--cold')
+
         if 'activity' in test:
             extra_options.append('--activity={}'.format(test.pop('activity')))
 
         if 'binary-path' in test:
             extra_options.append('--binary-path={}'.format(test.pop('binary-path')))
 
         if 'raptor-test' in test:
             extra_options.append('--test={}'.format(test.pop('raptor-test')))
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -243,16 +243,22 @@ class Raptor(TestingMixin, MercurialScri
             "help": "Enable Fission (site isolation) in Gecko.",
         }],
         [["--setpref"], {
             "action": "append",
             "dest": "extra_prefs",
             "default": [],
             "help": "A preference to set. Must be a key-value pair separated by a ':'."
         }],
+        [["--cold"], {
+            "action": "store_true",
+            "dest": "cold",
+            "default": False,
+            "help": "Enable cold page-load for browsertime tp6",
+        }],
 
     ] + testing_config_options + \
         copy.deepcopy(code_coverage_config_options) + \
         browsertime_options
 
     def __init__(self, **kwargs):
         kwargs.setdefault('config_options', self.config_options)
         kwargs.setdefault('all_actions', ['clobber',
@@ -502,16 +508,18 @@ class Raptor(TestingMixin, MercurialScri
         if self.config.get('is_release_build', False):
             options.extend(['--is-release-build'])
         if self.config.get('power_test', False):
             options.extend(['--power-test'])
         if self.config.get('memory_test', False):
             options.extend(['--memory-test'])
         if self.config.get('cpu_test', False):
             options.extend(['--cpu-test'])
+        if self.config.get('cold', False):
+            options.extend(['--cold'])
         if self.config.get('enable_webrender', False):
             options.extend(['--enable-webrender'])
         if self.config.get('with_conditioned_profile', False):
             options.extend(['--with-conditioned-profile'])
         if self.config.get('enable_fission', False):
             options.extend(['--enable-fission'])
         if self.config.get('extra_prefs'):
             options.extend(['--setpref={}'.format(i) for i in self.config.get('extra_prefs')])
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -151,16 +151,19 @@ def create_parser(mach_interface=False):
                 help="Flag which indicates if Raptor is running locally or in production")
         add_arg('--obj-path', dest="obj_path", default=None,
                 help="Browser-build obj_path (received when running in production)")
     add_arg('--noinstall', dest="noinstall", default=False, action="store_true",
             help="Flag which indicates if Raptor should not offer to install Android APK.")
     add_arg('--installerpath', dest="installerpath", default=None, type=str,
             help="Location where Android browser APK was extracted to before installation.")
 
+    # for browsertime jobs, cold page load is determined by a '--cold' cmd line argument
+    add_arg('--cold', dest="cold", action="store_true",
+            help="Enable cold page-load for browsertime tp6")
     # Arguments for invoking browsertime.
     add_arg('--browsertime', dest='browsertime', default=False, action="store_true",
             help="Whether to use browsertime to execute pageload tests")
     add_arg('--browsertime-node', dest='browsertime_node',
             help="path to Node.js executable")
     add_arg('--browsertime-browsertimejs', dest='browsertime_browsertimejs',
             help="path to browsertime.js script")
     add_arg('--browsertime-chromedriver', dest='browsertime_chromedriver',
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -5,17 +5,17 @@ from __future__ import absolute_import
 
 import json
 import os
 
 from six.moves.urllib.parse import parse_qs, urlsplit, urlunsplit, urlencode, unquote
 
 from logger.logger import RaptorLogger
 from manifestparser import TestManifest
-from utils import transform_platform
+from utils import transform_platform, transform_subtest
 from constants.raptor_tests_constants import YOUTUBE_PLAYBACK_MEASURE
 
 here = os.path.abspath(os.path.dirname(__file__))
 raptor_ini = os.path.join(here, 'raptor.ini')
 tests_dir = os.path.join(here, 'tests')
 LOG = RaptorLogger(component='raptor-manifest')
 
 LIVE_SITE_TIMEOUT_MULTIPLIER = 1.2
@@ -308,16 +308,25 @@ def get_raptor_test_list(args, oskey):
 
     # go through each test and set the page-cycles and page-timeout, and some config flags
     # the page-cycles value in the INI can be overriden when debug-mode enabled, when
     # gecko-profiling enabled, or when --page-cycles cmd line arg was used (that overrides all)
     for next_test in tests_to_run:
         LOG.info("configuring settings for test %s" % next_test['name'])
         max_page_cycles = next_test.get('page_cycles', 1)
 
+        # if using playback, the playback recording info may need to be transformed
+        if next_test.get('playback') is not None:
+            next_test['playback_pageset_manifest'] = \
+                transform_subtest(next_test['playback_pageset_manifest'],
+                                  next_test['name'])
+            next_test['playback_recordings'] = \
+                transform_subtest(next_test['playback_recordings'],
+                                  next_test['name'])
+
         if args.gecko_profile is True:
             next_test['gecko_profile'] = True
             LOG.info('gecko-profiling enabled')
             max_page_cycles = 3
 
             if 'gecko_profile_entries' in args and args.gecko_profile_entries is not None:
                 next_test['gecko_profile_entries'] = str(args.gecko_profile_entries)
                 LOG.info('gecko-profiling entries set to %s' % args.gecko_profile_entries)
@@ -356,17 +365,30 @@ def get_raptor_test_list(args, oskey):
             LOG.info("setting page-timeout to %d as specified on cmd line" % args.page_timeout)
             next_test['page_timeout'] = args.page_timeout
 
         # if --browser-cycles was provided on the command line, use that instead of INI
         if args.browser_cycles is not None:
             LOG.info("setting browser-cycles to %d as specified on cmd line" % args.browser_cycles)
             next_test['browser_cycles'] = args.browser_cycles
 
-        if next_test.get("cold", "false") == "true":
+        # for browsertime jobs, cold page-load mode is determined by command line argument; for
+        # raptor-webext jobs cold page-load is determined by the 'cold' key in test manifest INI
+        _running_cold = False
+        if args.browsertime is True:
+            if args.cold is True:
+                _running_cold = True
+            else:
+                # running warm page-load so ignore browser-cycles if it was provided (set to 1)
+                next_test['browser_cycles'] = 1
+        else:
+            if next_test.get("cold", "false") == "true":
+                _running_cold = True
+
+        if _running_cold:
             # when running in cold mode, set browser-cycles to the page-cycles value; as we want
             # the browser to restart between page-cycles; and set page-cycles to 1 as we only
             # want 1 single page-load for every browser-cycle
             next_test['cold'] = True
             next_test['expected_browser_cycles'] = int(next_test['browser_cycles'])
             next_test['page_cycles'] = 1
             # also ensure '-cold' is in test name so perfherder results indicate warm cold-load
             if "-cold" not in next_test['name']:
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -136,8 +136,11 @@
 
 # raptor scenario tests
 [include:tests/scenario/raptor-scn-power-idle.ini]
 [include:tests/scenario/raptor-scn-power-idle-bg.ini]
 
 # raptor unit tests
 # this is required for the manifest unit tests
 [include:tests/unittests/raptor-tp6-unittest.ini]
+
+# raptor-browsertime desktop tests
+[include:tests/tp6/desktop/browsertime-tp6.ini]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -620,16 +620,17 @@ class Browsertime(Perftest):
                 userjsfile.writelines(lines)
         except Exception as e:
             LOG.critical("Exception {} while removing mozprofile delimiters".format(e))
 
     def set_browser_test_prefs(self, raw_prefs):
         # add test specific preferences
         LOG.info("setting test-specific Firefox preferences")
         self.profile.set_preferences(json.loads(raw_prefs))
+        self.remove_mozprofile_delimiters_from_profile()
 
     def run_test_setup(self, test):
         super(Browsertime, self).run_test_setup(test)
 
         if test.get('type') == "benchmark":
             # benchmark-type tests require the benchmark test to be served out
             self.benchmark = Benchmark(self.config, test)
             test['test_url'] = test['test_url'].replace('<host>', self.benchmark.host)
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor-browsertime tp6 desktop page-load test specification
+
+[DEFAULT]
+alert_on = fcp, loadtime
+alert_threshold = 2.0
+apps = firefox, chrome
+browser_cycles = 25
+gecko_profile_entries = 14000000
+gecko_profile_interval = 1
+lower_is_better = true
+measure = fnbpaint, fcp, dcf, loadtime
+page_cycles = 25
+page_timeout = 60000
+playback = mitmproxy
+playback_pageset_manifest = mitm4-linux-firefox-{subtest}.manifest
+playback_recordings = {subtest}.mp
+type = pageload
+unit = ms
+use_live_sites = false
+
+[amazon]
+test_url = https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1
+
+[bing-search]
+test_url = https://www.bing.com/search?q=barack+obama
+playback_recordings = bing.mp
+
+[facebook]
+test_url = https://www.facebook.com
+
+[google-search]
+test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
+
+[google-slides]
+test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing
+
+[wikipedia]
+test_url = https://en.wikipedia.org/wiki/Barack_Obama
+
+[yahoo-news]
+test_url = https://www.yahoo.com/lifestyle/police-respond-noise-complaint-end-playing-video-games-respectful-tenants-002329963.html
+
+[youtube]
+test_url = https://www.youtube.com
--- a/testing/raptor/raptor/utils.py
+++ b/testing/raptor/raptor/utils.py
@@ -44,16 +44,25 @@ def transform_platform(str_to_transform,
         if 'x86_64' in config_processor:
             str_to_transform = str_to_transform.replace('{x64}', '_x64')
         else:
             str_to_transform = str_to_transform.replace('{x64}', '')
 
     return str_to_transform
 
 
+def transform_subtest(str_to_transform, subtest_name):
+    """Transform subtest name i.e. 'mitm4-linux-firefox-{subtest}.manifest'
+    transforms to 'mitm4-linux-firefox-amazon.manifest'."""
+    if '{subtest}' not in str_to_transform:
+        return str_to_transform
+
+    return str_to_transform.replace('{subtest}', subtest_name)
+
+
 def view_gecko_profile(ffox_bin):
     # automatically load the latest talos gecko-profile archive in profiler.firefox.com
     LOG_GECKO = RaptorLogger(component='raptor-view-gecko-profile')
 
     if sys.platform.startswith('win') and not ffox_bin.endswith(".exe"):
         ffox_bin = ffox_bin + ".exe"
 
     if not os.path.exists(ffox_bin):
--- a/testing/raptor/test/conftest.py
+++ b/testing/raptor/test/conftest.py
@@ -85,17 +85,19 @@ def create_args():
                      test='raptor-tp6-unittest',
                      binary='path/to/binary',
                      gecko_profile=False,
                      debug_mode=False,
                      page_cycles=None,
                      page_timeout=None,
                      test_url_params=None,
                      host=None,
-                     run_local=True)
+                     run_local=True,
+                     browsertime=True,
+                     cold=False)
 
     def inner(**kwargs):
         for next_arg in kwargs:
             print(next_arg)
             print(kwargs[next_arg])
             setattr(args, next_arg, kwargs[next_arg])
         return args
 
--- a/testing/raptor/test/test_manifest.py
+++ b/testing/raptor/test/test_manifest.py
@@ -54,17 +54,16 @@ VALID_MANIFESTS = [{
     'alert_change_type': None,
     'alert_on': None,
     'playback': None,
 }, {
     # page load test for geckoview
     'alert_threshold': 2.0,
     'apps': 'geckoview',
     'browser_cycles': 10,
-    'cold': True,
     'lower_is_better': False,
     'manifest': 'valid_details_2',
     'measure': 'fcp',
     'page_cycles': 1,
     'test_url': 'http://www.test-url/goes/here',
     'type': 'pageload',
     'unit': 'score',
 }, {