Bug 1575089 - Add a new Talos test that measures warm startup time with a number of real-world WebExtensions installed. r=rwood
authorMike Conley <mconley@mozilla.com>
Sat, 31 Aug 2019 06:58:00 +0000
changeset 551554 72edc19d4181e2086bba17f616d001771d65f098
parent 551553 a63410d10c7a9d06504c66338ccdeaac945a8d00
child 551555 8e2dfb26c66ce8f6a48bb1ba2f9d2a4ca6d884fe
push id11865
push userbtara@mozilla.com
push dateMon, 02 Sep 2019 08:54:37 +0000
treeherdermozilla-beta@37f59c4671b3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1575089
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1575089 - Add a new Talos test that measures warm startup time with a number of real-world WebExtensions installed. r=rwood Differential Revision: https://phabricator.services.mozilla.com/D43309
.hgignore
taskcluster/ci/test/talos.yml
taskcluster/ci/test/test-sets.yml
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/talos.json
testing/talos/talos/ffsetup.py
testing/talos/talos/getinfooffline/api.js
testing/talos/talos/getinfooffline/background.js
testing/talos/talos/getinfooffline/manifest.json
testing/talos/talos/getinfooffline/schema.json
testing/talos/talos/test.py
testing/talos/webextensions.manifest
--- a/.hgignore
+++ b/.hgignore
@@ -151,16 +151,18 @@ compile_commands\.json
 ^testing/talos/.Python
 ^testing/talos/bin/
 ^testing/talos/include/
 ^testing/talos/lib/
 ^testing/talos/talos/tests/tp5n.zip
 ^testing/talos/talos/tests/tp5n.tar.gz
 ^testing/talos/talos/tests/tp5n
 ^testing/talos/talos/tests/devtools/damp.manifest.develop
+^testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint.manifest.develop
+^testing/talos/talos/webextensions/
 ^talos-venv
 ^py3venv
 ^testing/talos/talos/mitmproxy/mitmdump
 ^testing/talos/talos/mitmproxy/mitmproxy
 ^testing/talos/talos/mitmproxy/mitmweb
 
 # Ignore talos webkit benchmark files; source is copied from in-tree /third_party
 # into testing/talos/talos/tests/webkit/PerformanceTests/ when run locally
--- a/taskcluster/ci/test/talos.yml
+++ b/taskcluster/ci/test/talos.yml
@@ -400,8 +400,25 @@ talos-xperf:
             default: default
     mozharness:
         extra-options:
             - --suite=xperf
         config:
             by-test-platform:
                 windows.*:
                     - talos/windows_vm_config.py
+
+talos-realworld-webextensions:
+    description: "Talos tests with realworld WebExtensions"
+    try-name: "realworld-webextensions"
+    treeherder-symbol: T(realwebext)
+    run-on-projects:
+        by-test-platform:
+            linux64-ccov/.*: ['try']
+            windows10-64-ccov/.*: ['try']
+            (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
+            windows10-aarch64/opt: ['mozilla-central', 'try']
+            macosx.*64(?:-shippable)?(?:-qr)?/opt: ['mozilla-central', 'try']
+            default: ['mozilla-beta', 'trunk', 'try']
+    max-run-time: 1500
+    mozharness:
+        extra-options:
+            - --suite=realworld-webextensions
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -68,16 +68,17 @@ talos:
     - talos-g5
     - talos-other
     - talos-sessionrestore-many-windows
     - talos-svgr
     - talos-tp5o
     - talos-perf-reftest
     - talos-perf-reftest-singletons
     - talos-tabswitch
+    - talos-realworld-webextensions
     # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions
     # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions
 
 talos-ref-hw:
     - talos-g4
 
 raptor-firefox:
     - raptor-tp6-1-firefox
@@ -316,16 +317,17 @@ windows-talos:
     - talos-other
     - talos-sessionrestore-many-windows
     - talos-perf-reftest
     - talos-perf-reftest-singletons
     - talos-svgr
     - talos-tp5o
     - talos-xperf
     - talos-tabswitch
+    - talos-realworld-webextensions
     # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions
 
 marionette-gpu-tests:
     - marionette-gpu
 
 marionette-media-tests:
     - marionette-media
 
@@ -374,16 +376,17 @@ macosx64-talos:
     - talos-g4
     - talos-g5
     - talos-other
     - talos-sessionrestore-many-windows
     - talos-svgr
     - talos-tp5o
     - talos-perf-reftest
     - talos-perf-reftest-singletons
+    - talos-realworld-webextensions
     # - talos-tabswitch # Bug 1453007 times out
     # - talos-h1 # too long to unpack profile- Bug 1442893
 
 linux32-tests:
     - web-platform-tests
 
 desktop-screenshot-capture:
     - browser-screenshots
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -202,16 +202,17 @@ class Talos(TestingMixin, MercurialScrip
         self.obj_path = self.config.get("obj_path")
         self.tests = None
         self.gecko_profile = self.config.get('gecko_profile') or \
             "--geckoProfile" in self.config.get("talos_extra_options", []) or \
             "--gecko-profile" in self.config.get("talos_extra_options", [])
         self.gecko_profile_interval = self.config.get('gecko_profile_interval')
         self.pagesets_name = None
         self.benchmark_zip = None
+        self.webextensions_zip = None
 
     # We accept some configuration options from the try commit message in the format
     # mozharness: <options>
     # Example try commit message:
     #   mozharness: --gecko-profile try: <stuff>
     def query_gecko_profile_options(self):
         gecko_results = []
         # finally, if gecko_profile is set, we add that to the talos options
@@ -260,16 +261,28 @@ class Talos(TestingMixin, MercurialScrip
         """
         if self.benchmark_zip:
             return self.benchmark_zip
         if self.query_talos_json_config() and self.suite is not None:
             self.benchmark_zip = self.talos_json_config['suites'][self.suite].get('benchmark_zip')
             self.benchmark_zip_manifest = 'jetstream-benchmark.manifest'
             return self.benchmark_zip
 
+    def query_webextensions_zip(self):
+        """Certain suites require external WebExtension sets to be downloaded and
+        extracted.
+        """
+        if self.webextensions_zip:
+            return self.webextensions_zip
+        if self.query_talos_json_config() and self.suite is not None:
+            self.webextensions_zip = \
+                self.talos_json_config['suites'][self.suite].get('webextensions_zip')
+            self.webextensions_zip_manifest = 'webextensions.manifest'
+            return self.webextensions_zip
+
     def get_suite_from_test(self):
         """ Retrieve the talos suite name from a given talos test name."""
         # running locally, single test name provided instead of suite; go through tests and
         # find suite name
         suite_name = None
         if self.query_talos_json_config():
             if '-a' in self.config['talos_extra_options']:
                 test_name_index = self.config['talos_extra_options'].index('-a') + 1
@@ -362,41 +375,50 @@ class Talos(TestingMixin, MercurialScrip
                     self.validate_suite()
                 else:
                     self.fatal("Suite name not provided")
         else:
             # talos initiated in production via mozharness
             self.suite = self.config['suite']
 
         tooltool_artifacts = []
+        src_talos_pageset_dest = os.path.join(self.talos_path, 'talos', 'tests')
+        webextension_dest = os.path.join(self.talos_path, 'talos', 'webextensions')
+
         if self.query_pagesets_name():
             tooltool_artifacts.append({'name': self.pagesets_name,
-                                       'manifest': self.pagesets_name_manifest})
+                                       'manifest': self.pagesets_name_manifest,
+                                       'dest': src_talos_pageset_dest})
 
         if self.query_benchmark_zip():
             tooltool_artifacts.append({'name': self.benchmark_zip,
-                                       'manifest': self.benchmark_zip_manifest})
+                                       'manifest': self.benchmark_zip_manifest,
+                                       'dest': src_talos_pageset_dest})
+
+        if self.query_webextensions_zip():
+            tooltool_artifacts.append({'name': self.webextensions_zip,
+                                       'manifest': self.webextensions_zip_manifest,
+                                       'dest': webextension_dest})
 
         # now that have the suite name, check if artifact is required, if so download it
         # the --no-download option will override this
         for artifact in tooltool_artifacts:
             if '--no-download' not in self.config.get('talos_extra_options', []):
                 self.info("Downloading %s with tooltool..." % artifact)
-                self.src_talos_webdir = os.path.join(self.talos_path, 'talos')
-                src_talos_pageset = os.path.join(self.src_talos_webdir, 'tests')
-                if not os.path.exists(os.path.join(src_talos_pageset, artifact['name'])):
+
+                if not os.path.exists(os.path.join(artifact['dest'], artifact['name'])):
                     manifest_file = os.path.join(self.talos_path, artifact['manifest'])
                     self.tooltool_fetch(
                         manifest_file,
-                        output_dir=src_talos_pageset,
+                        output_dir=artifact['dest'],
                         cache=self.config.get('tooltool_cache')
                     )
-                    archive = os.path.join(src_talos_pageset, artifact['name'])
+                    archive = os.path.join(artifact['dest'], artifact['name'])
                     unzip = self.query_exe('unzip')
-                    unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
+                    unzip_cmd = [unzip, '-q', '-o', archive, '-d', artifact['dest']]
                     self.run_command(unzip_cmd, halt_on_failure=True)
                 else:
                     self.info("%s already available" % artifact)
             else:
                 self.info("Not downloading %s because the no-download option was specified" %
                           artifact)
 
         # if running webkit tests locally, need to copy webkit source into talos/tests
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -62,11 +62,15 @@
             "pagesets_name": "tp5n.zip",
             "talos_options": [
                 "--xperf_path",
                 "\"c:/Program Files (x86)/Windows Kits/10/Windows Performance Toolkit/xperf.exe\""
             ]
         },
         "h1": {
             "tests": ["ts_paint_heavy"]
+        },
+        "realworld-webextensions": {
+            "tests": ["startup_about_home_paint_realworld_webextensions"],
+            "webextensions_zip": "webextensions.zip"
         }
     }
 }
--- a/testing/talos/talos/ffsetup.py
+++ b/testing/talos/talos/ffsetup.py
@@ -146,23 +146,31 @@ class FFSetup(object):
         profile.set_preferences(preferences)
 
         # installing addons
         LOG.info("Installing Add-ons:")
         LOG.info(extensions)
         profile.addons.install(extensions)
 
         # installing webextensions
+        webextensions_to_install = []
+        webextensions_folder = self.test_config.get('webextensions_folder', None)
+        if isinstance(webextensions_folder, basestring):
+            folder = utils.interpolate(webextensions_folder)
+            for file in os.listdir(folder):
+                if file.endswith(".xpi"):
+                    webextensions_to_install.append(os.path.join(folder, file))
+
         webextensions = self.test_config.get('webextensions', None)
         if isinstance(webextensions, basestring):
-            webextensions = [webextensions]
+            webextensions_to_install.append(webextensions)
 
-        if webextensions is not None:
+        if webextensions_to_install is not None:
             LOG.info("Installing Webextensions:")
-            for webext in webextensions:
+            for webext in webextensions_to_install:
                 filename = utils.interpolate(webext)
                 if mozinfo.os == 'win':
                     filename = filename.replace('/', '\\')
                 if not filename.endswith('.xpi'):
                     continue
                 if not os.path.exists(filename):
                     continue
                 LOG.info(filename)
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/getinfooffline/api.js
@@ -0,0 +1,17 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80 filetype=javascript: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* globals ExtensionAPI */
+
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+this.getinfooffline = class extends ExtensionAPI {
+  onStartup() {
+    Services.io.offline = true;
+  }
+};
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/getinfooffline/background.js
@@ -0,0 +1,6 @@
+// We've run once, and hopefully bypassed any network connections that add-ons
+// might have tried to make after first install. Now let's uninstall ourselves
+// so that subsequent starts run in online mode.
+
+/* globals browser */
+browser.management.uninstallSelf({});
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/getinfooffline/manifest.json
@@ -0,0 +1,27 @@
+{
+  "manifest_version": 2,
+  "name": "Make the getinfo step happen offline for Talos",
+  "description": "Puts the browser in offline mode during the getinfo step and then uninstalls itself so that the browser runs in online mode for subsequent starts. This is useful when running startup tests that instal add-ons that like to open up network requests after being installed for the first time, which normally would cause the test runner to crash.",
+  "version": "0.1",
+
+  "applications": {
+    "gecko": {
+      "id": "getinfooffline@mozilla.org"
+    }
+  },
+
+  "background": {
+    "scripts": ["background.js"]
+  },
+
+  "experiment_apis": {
+    "getinfooffline": {
+      "schema": "schema.json",
+      "parent": {
+        "scopes": ["addon_parent"],
+        "script": "api.js",
+        "events": ["startup"]
+      }
+    }
+  }
+}
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/getinfooffline/schema.json
@@ -0,0 +1,1 @@
+[]
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -114,16 +114,17 @@ class TsBase(Test):
         'userready',
         'testeventmap',
         'base_vs_ref',
         'extensions',
         'filters',
         'setup',
         'cleanup',
         'webextensions',
+        'webextensions_folder',
         'reinstall',     # A list of files from the profile directory that
                          # should be copied to the temporary profile prior to
                          # running each cycle, to avoid one cycle overwriting
                          # the data used by the next another cycle (may be used
                          # e.g. for sessionstore.js to ensure that all cycles
                          # use the exact same sessionstore.js, rather than a
                          # more recent copy).
     ]
@@ -173,16 +174,28 @@ class ts_paint_flex(ts_paint):
 class startup_about_home_paint(ts_paint):
     url = None
     cycles = 20
     extensions = ['${talos}/startup_test/startup_about_home_paint/addon']
     tpmanifest = '${talos}/startup_test/startup_about_home_paint/startup_about_home_paint.manifest'
 
 
 @register_test()
+class startup_about_home_paint_realworld_webextensions(ts_paint):
+    url = None
+    cycles = 20
+    extensions = [
+        '${talos}/startup_test/startup_about_home_paint/addon',
+        '${talos}/getinfooffline'
+    ]
+    tpmanifest = '${talos}/startup_test/startup_about_home_paint/startup_about_home_paint.manifest'
+    webextensions_folder = '${talos}/webextensions'
+
+
+@register_test()
 class sessionrestore(TsBase):
     """
     A start up test measuring the time it takes to load a sessionstore.js file.
 
     1. Set up Firefox to restore from a given sessionstore.js file.
     2. Launch Firefox.
     3. Measure the delta between firstPaint and sessionRestored.
     """
@@ -248,17 +261,17 @@ class PageloaderTest(Test):
             'tpmozafterpaint', 'fnbpaint', 'tphero', 'tploadnocache', 'firstpaint',
             'userready', 'testeventmap', 'base_vs_ref', 'mainthread', 'resolution',
             'cycles', 'gecko_profile', 'gecko_profile_interval', 'gecko_profile_entries',
             'tptimeout', 'win_counters', 'w7_counters', 'linux_counters', 'mac_counters',
             'tpscrolltest', 'xperf_counters', 'timeout', 'responsiveness',
             'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
             'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup',
             'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile',
-            'subtest_alerts', 'perfherder_framework', 'pdfpaint']
+            'subtest_alerts', 'perfherder_framework', 'pdfpaint', 'webextensions_folder']
 
 
 class QuantumPageloadTest(PageloaderTest):
     """
     Base class for a Quantum Pageload test
     """
     tpcycles = 1
     tppagecycles = 25
new file mode 100644
--- /dev/null
+++ b/testing/talos/webextensions.manifest
@@ -0,0 +1,9 @@
+[
+  {
+    "size": 2608427,
+    "visibility": "public",
+    "digest": "f3198257ca82043bb9fe9a7085cff7af6f0d11685a0416ebb73e8edec547b970dd044666120e44898e2de36372add88ef93cd08af79dcd03e32fa1e9ef2053c8",
+    "algorithm": "sha512",
+    "filename": "webextensions.zip"
+  }
+]
\ No newline at end of file