Bug 1364941 - add new talos tests with an active, but blank web extension to measure ts_paint and tp5o. r=rwood
authorJoel Maher <jmaher@mozilla.com>
Tue, 23 May 2017 12:38:05 -0400
changeset 360240 b403f00d125ebb7303338e31d1010ceb95b3d694
parent 360239 5a0655f29081298bcad111da0a8fd02655a2abc9
child 360241 62c3f811e599a947b1c65863f60ba1c202de4969
push id31872
push userryanvm@gmail.com
push dateTue, 23 May 2017 22:11:58 +0000
treeherdermozilla-central@96e18bec9fc8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1364941
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1364941 - add new talos tests with an active, but blank web extension to measure ts_paint and tp5o. r=rwood MozReview-Commit-ID: 7NUhFgj6sF1
testing/talos/requirements.txt
testing/talos/talos.json
testing/talos/talos/ffsetup.py
testing/talos/talos/test.py
testing/talos/talos/webextensions/dummy/background.js
testing/talos/talos/webextensions/dummy/content.js
testing/talos/talos/webextensions/dummy/dummy.xpi
testing/talos/talos/webextensions/dummy/manifest.json
--- a/testing/talos/requirements.txt
+++ b/testing/talos/requirements.txt
@@ -2,8 +2,9 @@ mozlog>=3.1
 mozcrash>=0.15
 mozfile>=1.2
 mozhttpd>=0.7
 mozinfo>=0.8
 mozprocess>=0.22
 mozversion>=1.3
 mozprofile>=0.25
 psutil>=3.1.1
+marionette_driver>=1.4.0
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -70,17 +70,17 @@
         "g4": {
             "tests": ["basic_compositor_video", "glvideo"],
             "talos_options": ["--disable-e10s"]
         },
         "g4-e10s": {
             "tests": ["basic_compositor_video", "glvideo"]
         },
         "g5-e10s": {
-            "tests": ["ts_paint", "tp5o"],
+            "tests": ["ts_paint_webext", "tp5o_webext"],
             "pagesets_name": "tp5n.zip"
         },
         "svgr": {
             "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart", "tsvg_static"],
             "talos_options": ["--disable-e10s"]
         },
         "svgr-e10s": {
             "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart", "tsvg_static"]
--- a/testing/talos/talos/ffsetup.py
+++ b/testing/talos/talos/ffsetup.py
@@ -5,19 +5,23 @@
 """
 Set up a browser environment before running a test.
 """
 
 import os
 import re
 import tempfile
 import mozfile
-from mozprocess import ProcessHandler
+import mozinfo
+import mozrunner
+
+from mozprocess import ProcessHandlerMixin
 from mozprofile.profile import Profile
-
+from marionette_driver.marionette import Marionette
+from marionette_driver.addons import Addons
 from mozlog import get_proxy_logger
 
 from talos import utils
 from talos.utils import TalosError
 from talos.gecko_profile import GeckoProfile
 
 
 LOG = get_proxy_logger()
@@ -42,19 +46,16 @@ class FFSetup(object):
     This class should be used as a context manager::
 
       with FFSetup(browser_config, test_config) as setup:
           # setup.env is initialized, and setup.profile_dir created
           pass
       # here the profile is removed
     """
 
-    PROFILE_REGEX = re.compile('__metrics(.*)__metrics',
-                               re.DOTALL | re.MULTILINE)
-
     def __init__(self, browser_config, test_config):
         self.browser_config, self.test_config = browser_config, test_config
         self._tmp_dir = tempfile.mkdtemp()
         self.env = None
         # The profile dir must be named 'profile' because of xperf analysis
         # (in etlparser.py). TODO fix that ?
         self.profile_dir = os.path.join(self._tmp_dir, 'profile')
         self.gecko_profile = None
@@ -103,58 +104,100 @@ class FFSetup(object):
             os.path.normpath(self.test_config['profile_path']),
             self.profile_dir,
             restore=False)
 
         profile.set_preferences(preferences)
         profile.addon_manager.install_addons(extensions)
 
     def _run_profile(self):
-        command_args = utils.GenerateBrowserCommandLine(
-            self.browser_config["browser_path"],
-            self.browser_config["extra_args"],
-            self.profile_dir,
-            self.browser_config["init_url"]
-        )
+        runner_cls = mozrunner.runners.get(
+            mozinfo.info.get(
+                'appname',
+                'firefox'),
+            mozrunner.Runner)
 
-        def browser_log(line):
-            LOG.process_output(browser.pid, line)
+        if self.test_config.get('webextensions', None):
+            args = [self.browser_config["extra_args"], "-marionette"]
+            runner = runner_cls(profile=self.profile_dir,
+                                binary=self.browser_config["browser_path"],
+                                cmdargs=args,
+                                process_class=ProcessHandlerMixin,
+                                process_args={})
+
+            runner.start(outputTimeout=30)
+            proc = runner.process_handler
+            LOG.process_start(proc.pid, "%s %s" % (self.browser_config["browser_path"],
+                                                   ' '.join(args)))
+
+            try:
+                client = Marionette(host='localhost', port=2828)
+                client.start_session()
+                addons = Addons(client)
+            except Exception, e:
+                print e
+                raise TalosError("Failed to initialize Talos profile with Marionette")
 
-        browser = ProcessHandler(command_args, env=self.env,
-                                 processOutputLine=browser_log)
-        browser.run()
-        LOG.process_start(browser.pid, ' '.join(command_args))
+            extensions = self.test_config.get('webextensions', None)
+            if isinstance(extensions, str):
+                extensions = [self.test_config.get('webextensions', None)]
+            for ext in extensions:
+                filename = utils.interpolate(ext)
+                if mozinfo.os == 'win':
+                    filename = filename.replace('/', '\\')
+
+                if not filename.endswith('.xpi'):
+                    continue
+                if not os.path.exists(filename):
+                    continue
+
+                addons.install(filename)
+
+            # browse to init_url which will close the browser
+            client.navigate(self.browser_config["init_url"])
+            client.close()
+        else:
+            args = [self.browser_config["extra_args"], self.browser_config["init_url"]]
+            runner = runner_cls(profile=self.profile_dir,
+                                binary=self.browser_config["browser_path"],
+                                cmdargs=args,
+                                process_class=ProcessHandlerMixin,
+                                process_args={})
+
+            runner.start(outputTimeout=30)
+            proc = runner.process_handler
+            LOG.process_start(proc.pid, "%s %s" % (self.browser_config["browser_path"],
+                                                   ' '.join(args)))
+
         try:
-            exit_code = browser.wait()
-        except KeyboardInterrupt:
-            browser.kill()
-            raise
+            exit_code = proc.wait()
+        except Exception, e:
+            proc.kill()
+            raise TalosError("Browser Failed to close properly during warmup")
 
-        LOG.process_exit(browser.pid, exit_code)
-        results_raw = '\n'.join(browser.output)
-
-        if not self.PROFILE_REGEX.search(results_raw):
-            LOG.info("Could not find %s in browser output"
-                     % self.PROFILE_REGEX.pattern)
-            LOG.info("Raw results:%s" % results_raw)
-            raise TalosError("browser failed to close after being initialized")
+        LOG.process_exit(proc.pid, exit_code)
 
     def _init_gecko_profile(self):
         upload_dir = os.getenv('MOZ_UPLOAD_DIR')
         if self.test_config.get('gecko_profile') and not upload_dir:
             LOG.critical("Profiling ignored because MOZ_UPLOAD_DIR was not"
                          " set")
         if upload_dir and self.test_config.get('gecko_profile'):
             self.gecko_profile = GeckoProfile(upload_dir,
                                               self.browser_config,
                                               self.test_config)
             self.gecko_profile.update_env(self.env)
 
     def clean(self):
-        mozfile.remove(self._tmp_dir)
+        try:
+            mozfile.remove(self._tmp_dir)
+        except Exception, e:
+            print "Exception while removing profile directory: %s" % self._tmp_dir
+            print e
+
         if self.gecko_profile:
             self.gecko_profile.clean()
 
     def __enter__(self):
         LOG.info('Initialising browser for %s test...'
                  % self.test_config['name'])
         self._init_env()
         self._init_profile()
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -103,16 +103,17 @@ class TsBase(Test):
         'xperf_providers',
         'xperf_user_providers',
         'xperf_stackwalk',
         'tpmozafterpaint',
         'extensions',
         'filters',
         'setup',
         'cleanup',
+        'webextensions',
         'reinstall',     # A list of files from the profile directory that
                          # should be copied to the temporary profile prior to
                          # running each cycle, to avoid one cycle overwriting
                          # the data used by the next another cycle (may be used
                          # e.g. for sessionstore.js to ensure that all cycles
                          # use the exact same sessionstore.js, rather than a
                          # more recent copy).
     ]
@@ -137,16 +138,22 @@ class ts_paint(TsBase):
     tpmozafterpaint = True
     rss = False
     mainthread = False
     responsiveness = False
     unit = 'ms'
 
 
 @register_test()
+class ts_paint_webext(ts_paint):
+    webextensions = '${talos}/webextensions/dummy/dummy.xpi'
+    preferences = {'xpinstall.signatures.required': False}
+
+
+@register_test()
 class sessionrestore(TsBase):
     """
     A start up test measuring the time it takes to load a sessionstore.js file.
 
     1. Set up Firefox to restore from a given sessionstore.js file.
     2. Launch Firefox.
     3. Measure the delta between firstPaint and sessionRestored.
     """
@@ -230,17 +237,17 @@ class PageloaderTest(Test):
     keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
             'tpmozafterpaint', 'tploadnocache', 'rss', 'mainthread',
             'resolution', 'cycles', 'gecko_profile', 'gecko_profile_interval',
             'gecko_profile_entries', 'tptimeout', 'win_counters', 'w7_counters',
             'linux_counters', 'mac_counters', 'tpscrolltest', 'xperf_counters',
             'timeout', 'shutdown', 'responsiveness', 'profile_path',
             'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
             'filters', 'preferences', 'extensions', 'setup', 'cleanup',
-            'lower_is_better', 'alert_threshold', 'unit']
+            'lower_is_better', 'alert_threshold', 'unit', 'webextensions']
 
 
 @register_test()
 class tabpaint(PageloaderTest):
     """
     Tests the amount of time it takes to open new tabs, triggered from
     both the parent process and the content process.
     """
@@ -497,16 +504,22 @@ class tp5o(PageloaderTest):
     gecko_profile_interval = 2
     gecko_profile_entries = 4000000
     filters = filter.ignore_first.prepare(5) + filter.median.prepare()
     timeout = 1800
     unit = 'ms'
 
 
 @register_test()
+class tp5o_webext(tp5o):
+    webextensions = '${talos}/webextensions/dummy/dummy.xpi'
+    preferences = {'xpinstall.signatures.required': False}
+
+
+@register_test()
 class tp5o_scroll(PageloaderTest):
     """
     Tests scroll (like tscrollx does, including ASAP) but on the tp5o pageset.
     """
     tpmanifest = '${talos}/tests/tp5n/tp5o.manifest'
     tpcycles = 1
     tppagecycles = 12
     gecko_profile_interval = 2
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/webextensions/dummy/background.js
@@ -0,0 +1,3 @@
+"use strict";
+
+// Empty background script
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/webextensions/dummy/content.js
@@ -0,0 +1,3 @@
+"use strict";
+
+// Empty content script
new file mode 100644
index 0000000000000000000000000000000000000000..119b715e12b8e0944c3a14fe7f652050c06ca4c6
GIT binary patch
literal 963
zc$^FHW@h1H00F^*&0gYvmd?=uvO$=aA%`I;F*!TED8Do>MK7y3*d?4{2}mUe!};$Y
z^OI8&G6fP75)2X=7~~bwQdo^6T^n7lm@n;6uCcX$wD0BL+LJx;eVvL;jgmULhd#^?
zoYy!}a8VeCkEg?t845{U4o-|`!Wugs8;Uc8hVU}5A2gDWgkh!9;#7s=lA_Gy5+!S1
zE?zEueFfLtf|5!Fgp(AClZ!G7O1J`0ykPOIY^o-Z4Z>VNFC^#Zm89mC3>x=C-Gl6S
z5wM#~fR0btnVq`|$YWvzg?~9iZem_$T52)K_4#?hE=deuAnrfgsjmidzaSCrPh@45
zj)-b`XusHoSy|a-t-y>)4IPG#M?@N0m}abH5<H%)*UTXHnWbQcy2A|{h*wyQ<s(7Z
zEKN4jx9>F9Ap-%|?@j*=Qr5c0?g~1nA5fs^x-3&?W97uoMbrN6_6%OJI`Ld>!S0_t
z4GW9Cig+$bB}Kmw>e(T<XX|D8&NYuuGiiBS=k5>E)LD}sVfN+am+pl@izk*lKKv#$
zFDd^_?xv)G?kh!px&;9lQ9C-g`ra$}Y&!JP*)ikB^{+;3#aXKoS0~vzsqdNQ^7u{m
z%*@t3xqE77nQTiwvbQAB;E(6g<_Jml@Ad)Sj7)M2xKgtqSe%hTfdK;0(l!e?ZG?uf
zLLw5~EYxI$Y}R2QMowf{4Z;=0$Obh4MbILg1svgEvp~@i;LXYgQqK&84}tU<CJ+w*
D@G>x8
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/webextensions/dummy/manifest.json
@@ -0,0 +1,12 @@
+{
+  "applications": {
+    "gecko": {
+      "id": "talos@mozilla.org"
+    }
+  },
+  "manifest_version": 2,
+  "name": "Talos Blank Web Extension System Add-on",
+  "version": "0.1",
+  "permissions": [],
+  "background": {"scripts": ["background.js"]}
+}
\ No newline at end of file