Bug 1460743 - Add 'speedometer' benchmark to raptor for google chrome; r?jmaher draft
authorRob Wood <rwood@mozilla.com>
Mon, 04 Jun 2018 15:42:44 -0400
changeset 805948 03454b0b5d41336a06d825733e1128bda0812d23
parent 805817 3e1da01d48780656bc9e0dd014b89ee0a75129d6
child 805949 2dfc22b6ca3a0f46e04b82872a72e8f8711aec3f
push id112817
push userrwood@mozilla.com
push dateFri, 08 Jun 2018 19:11:26 +0000
reviewersjmaher
bugs1460743
milestone62.0a1
Bug 1460743 - Add 'speedometer' benchmark to raptor for google chrome; r?jmaher Includes a temporary downloading/installing Chrome 'hack' just so we can get going on try, until 'official' chrome builds are available in CI. Also some small raptor fixes/changes to support speedometer on chrome. MozReview-Commit-ID: 8ZBlOg0BOE4
testing/mozharness/configs/raptor/mac_config.py
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/raptor/mach_commands.py
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/raptor-speedometer.ini
testing/raptor/webext/raptor/manifest.json
--- a/testing/mozharness/configs/raptor/mac_config.py
+++ b/testing/mozharness/configs/raptor/mac_config.py
@@ -21,16 +21,17 @@ config = {
         "http://pypi.pub.build.mozilla.org/pub",
     ],
     "pip_index": False,
     "title": os.uname()[1].lower().split('.')[0],
     "default_actions": [
         "clobber",
         "download-and-extract",
         "populate-webroot",
+        "install-chrome",
         "create-virtualenv",
         "install",
         "run-tests",
     ],
     "run_cmd_checks_enabled": True,
     "preflight_run_cmd_suites": [
         SCREEN_RESOLUTION_CHECK,
     ],
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -5,16 +5,17 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 import json
 import os
 import re
 import sys
 import subprocess
+import time
 
 from shutil import copyfile
 
 import mozharness
 
 from mozharness.base.errors import PythonErrorList
 from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO
 from mozharness.base.python import Python3Virtualenv
@@ -22,16 +23,17 @@ from mozharness.mozilla.testing.testbase
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 
 scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
 external_tools_path = os.path.join(scripts_path, 'external_tools')
+here = os.path.abspath(os.path.dirname(__file__))
 
 RaptorErrorList = PythonErrorList + [
     {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
     {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
     {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
     {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
     {'substr': r'''raptorError''', 'level': CRITICAL},
     {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
@@ -47,73 +49,90 @@ class Raptor(TestingMixin, MercurialScri
     install and run raptor tests
     """
     config_options = [
         [["--test"],
          {"action": "store",
           "dest": "test",
           "help": "Raptor test to run"
           }],
+        [["--app"],
+         {"default": "firefox",
+          "choices": ["firefox", "chrome"],
+          "dest": "app",
+          "help": "name of the application we are testing (default: firefox)"
+          }],
         [["--branch-name"],
          {"action": "store",
           "dest": "branch",
           "help": "branch running against"
           }],
         [["--add-option"],
          {"action": "extend",
-          "dest": "raptor_extra_options",
+          "dest": "raptor_cmd_line_args",
           "default": None,
           "help": "extra options to raptor"
           }],
     ] + testing_config_options + copy.deepcopy(code_coverage_config_options)
 
     def __init__(self, **kwargs):
         kwargs.setdefault('config_options', self.config_options)
         kwargs.setdefault('all_actions', ['clobber',
                                           'download-and-extract',
                                           'populate-webroot',
+                                          'install-chrome',
                                           'create-virtualenv',
                                           'install',
                                           'run-tests',
                                           ])
         kwargs.setdefault('default_actions', ['clobber',
                                               'download-and-extract',
                                               'populate-webroot',
+                                              'install-chrome',
                                               'create-virtualenv',
                                               'install',
                                               'run-tests',
                                               ])
         kwargs.setdefault('config', {})
         super(Raptor, self).__init__(**kwargs)
 
         self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
 
         self.run_local = self.config.get('run_local')
+
+        # app (browser testing on) defaults to firefox
+        self.app = "firefox"
+
+        if self.run_local:
+            # raptor initiated locally, get app from command line args
+            # which are passed in from mach inside 'raptor_cmd_line_args'
+            self.app = "firefox"
+            if 'raptor_cmd_line_args' in self.config:
+                for next_arg in self.config['raptor_cmd_line_args']:
+                    if "chrome" in next_arg:
+                        self.app = "chrome"
+                        break
+        else:
+            # raptor initiated in production via mozharness
+            self.test = self.config['test']
+            self.app = self.config.get("app", "firefox")
+
         self.installer_url = self.config.get("installer_url")
         self.raptor_json_url = self.config.get("raptor_json_url")
         self.raptor_json = self.config.get("raptor_json")
         self.raptor_json_config = self.config.get("raptor_json_config")
         self.repo_path = self.config.get("repo_path")
         self.obj_path = self.config.get("obj_path")
-        self.tests = None
+        self.test = None
         self.gecko_profile = self.config.get('gecko_profile')
         self.gecko_profile_interval = self.config.get('gecko_profile_interval')
-        # some platforms download a mitmproxy release binary
-        self.mitmproxy_rel_bin = None
-        # zip file found on tooltool that contains all of the mitmproxy recordings
-        self.mitmproxy_pageset = None
-        # files inside the recording set
-        self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None)
-        # path to mitmdump tool itself, in py3 venv
-        self.mitmdump = None
 
     # We accept some configuration options from the try commit message in the
-    # format mozharness: <options>
-    # Example try commit message:
-    #   mozharness: --geckoProfile try: <stuff>
+    # format mozharness: <options>. Example try commit message: mozharness:
+    # --geckoProfile try: <stuff>
     def query_gecko_profile_options(self):
         gecko_results = []
         # if gecko_profile is set, we add that to the raptor options
         if self.gecko_profile:
             gecko_results.append('--geckoProfile')
             if self.gecko_profile_interval:
                 gecko_results.extend(
                     ['--geckoProfileInterval', str(self.gecko_profile_interval)]
@@ -125,29 +144,87 @@ class Raptor(TestingMixin, MercurialScri
             return self.abs_dirs
         abs_dirs = super(Raptor, self).query_abs_dirs()
         abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'],
                                                        'blobber_upload_dir')
         abs_dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
+    def install_chrome(self):
+        # temporary hack to install google chrome in production; until chrome is in our CI
+        if self.app != "chrome":
+            self.info("Google Chrome is not required")
+            return
+
+        if self.config.get("run_local"):
+            self.info("expecting Google Chrome to be pre-installed locally")
+            return
+
+        chrome_url = "https://dl.google.com/chrome/mac/stable/GGRO/googlechrome.dmg"
+        # in production we can put the chrome build in mozharness/mozilla/testing/chrome
+        self.chrome_dest = os.path.join(here, 'chrome')
+        chrome_dmg = os.path.join(self.chrome_dest, 'googlechrome.dmg')
+
+        self.info("installing google chrome - temporary install hack")
+        self.info("chrome_dest is: %s" % self.chrome_dest)
+
+        self.chrome_path = os.path.join(self.chrome_dest, 'Google Chrome.app',
+                                        'Contents', 'MacOS', 'Google Chrome')
+
+        if os.path.exists(self.chrome_path):
+            self.info("google chrome binary already exists at: %s" % self.chrome_path)
+            return
+
+        if not os.path.exists(chrome_dmg):
+            # download the chrome dmg
+            self.download_file(chrome_url, parent_dir=self.chrome_dest)
+
+        command = ["open", "googlechrome.dmg"]
+        return_code = self.run_command(command, cwd=self.chrome_dest)
+        if return_code not in [0]:
+            self.info("abort: failed to open %s/googlechrome.dmg" % self.chrome_dest)
+            return
+        # give 30 sec for open cmd to finish
+        time.sleep(30)
+
+        # now that the googlechrome dmg is mounted, extract/copy app from mnt to our folder
+        command = ["cp", "-r", "/Volumes/Google Chrome/Google Chrome.app", "."]
+        return_code = self.run_command(command, cwd=self.chrome_dest)
+        if return_code not in [0]:
+            self.info("abort: failed to open %s/googlechrome.dmg" % self.chrome_dest)
+            return
+
+        # now ensure chrome binary exists
+        if os.path.exists(self.chrome_path):
+            self.info("successfully installed Google Chrome to: %s" % self.chrome_path)
+        else:
+            self.info("abort: failed to install Google Chrome")
+
     def raptor_options(self, args=None, **kw):
         """return options to raptor"""
-        # binary path
-        binary_path = self.binary_path or self.config.get('binary_path')
-        if not binary_path:
-            msg = """Raptor requires a path to the binary. You can specify binary_path or add
-            download-and-extract to your action list."""
-            self.fatal(msg)
-        # raptor options
-        if binary_path.endswith('.exe'):
-            binary_path = binary_path[:-4]
         options = []
-        kw_options = {'binary': binary_path}
+        kw_options = {}
+
+        # binary path; if testing on firefox the binary path already came from mozharness/pro;
+        # otherwise the binary path is forwarded from cmd line arg (raptor_cmd_line_args)
+        kw_options['app'] = self.app
+        if self.app == "firefox":
+            binary_path = self.binary_path or self.config.get('binary_path')
+            if not binary_path:
+                self.fatal("Raptor requires a path to the binary.")
+            if binary_path.endswith('.exe'):
+                binary_path = binary_path[:-4]
+            kw_options['binary'] = binary_path
+        else:
+            if not self.run_local:
+                # in production we aready installed google chrome, so set the binary path for arg
+                # when running locally a --binary arg as passed in, already in raptor_cmd_line_args
+                kw_options['binary'] = self.chrome_path
+
         # options overwritten from **kw
         if 'test' in self.config:
             kw_options['test'] = self.config['test']
         if self.config.get('branch'):
             kw_options['branchName'] = self.config['branch']
         if self.symbols_path:
             kw_options['symbolsPath'] = self.symbols_path
         if self.config.get('obj_path', None) is not None:
@@ -155,44 +232,32 @@ class Raptor(TestingMixin, MercurialScri
         kw_options.update(kw)
         # configure profiling options
         options.extend(self.query_gecko_profile_options())
         # extra arguments
         if args is not None:
             options += args
         if self.config.get('run_local', False):
             options.extend(['--run-local'])
-        if 'raptor_extra_options' in self.config:
-            options += self.config['raptor_extra_options']
+        if 'raptor_cmd_line_args' in self.config:
+            options += self.config['raptor_cmd_line_args']
         if self.config.get('code_coverage', False):
             options.extend(['--code-coverage'])
         for key, value in kw_options.items():
             options.extend(['--%s' % key, value])
+
         return options
 
     def populate_webroot(self):
         """Populate the production test slaves' webroots"""
         self.raptor_path = os.path.join(
             self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
         )
-
         if self.config.get('run_local'):
-            # raptor initiated locally, get and verify test from cmd line
             self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
-            if 'raptor_extra_options' in self.config:
-                if '--test' in self.config['raptor_extra_options']:
-                    # --test specified, get test from cmd line and ensure is valid
-                    test_name_index = self.config['raptor_extra_options'].index('--test') + 1
-                    if test_name_index < len(self.config['raptor_extra_options']):
-                        self.test = self.config['raptor_extra_options'][test_name_index]
-                    else:
-                        self.fatal("Test name not provided")
-        else:
-            # raptor initiated in production via mozharness
-            self.test = self.config['test']
 
     # Action methods. {{{1
     # clobber defined in BaseScript
 
     def download_and_extract(self, extract_dirs=None, suite_categories=None):
         return super(Raptor, self).download_and_extract(
             suite_categories=['common', 'raptor']
         )
--- a/testing/raptor/mach_commands.py
+++ b/testing/raptor/mach_commands.py
@@ -44,34 +44,34 @@ class RaptorRunner(MozbuildObject):
         self.virtualenv_script = os.path.join(self.topsrcdir, 'third_party', 'python',
                                               'virtualenv', 'virtualenv.py')
         self.virtualenv_path = os.path.join(self._topobjdir, 'testing',
                                             'raptor-venv')
         self.python_interp = sys.executable
         self.raptor_args = raptor_args
 
     def make_config(self):
-        default_actions = ['populate-webroot', 'create-virtualenv', 'run-tests']
+        default_actions = ['populate-webroot', 'install-chrome', 'create-virtualenv', 'run-tests']
         self.config = {
             'run_local': True,
             'binary_path': self.binary_path,
             'repo_path': self.topsrcdir,
             'raptor_path': self.raptor_dir,
             'obj_path': self.topobjdir,
             'log_name': 'raptor',
             'virtualenv_path': self.virtualenv_path,
             'pypi_url': 'http://pypi.python.org/simple',
             'base_work_dir': self.mozharness_dir,
             'exes': {
                 'python': self.python_interp,
                 'virtualenv': [self.python_interp, self.virtualenv_script],
             },
             'title': socket.gethostname(),
             'default_actions': default_actions,
-            'raptor_extra_options': self.raptor_args,
+            'raptor_cmd_line_args': self.raptor_args,
             'python3_manifest': {
                 'win32': 'python3.manifest',
                 'win64': 'python3_x64.manifest',
             }
         }
 
     def make_args(self):
         self.args = {
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -10,37 +10,39 @@ from mozlog.commandline import add_loggi
 
 
 def create_parser(mach_interface=False):
     parser = argparse.ArgumentParser()
     add_arg = parser.add_argument
 
     add_arg('-t', '--test', required=True, dest='test',
             help="name of raptor test to run")
+    add_arg('--app', default='firefox', dest='app',
+            help="name of the application we are testing (default: firefox)",
+            choices=['firefox', 'chrome'])
+    add_arg('-b', '--binary', dest='binary',
+            help="path to the browser executable that we are testing")
     if not mach_interface:
-        add_arg('--app', default='firefox', dest='app',
-                help="name of the application we are testing (default: firefox)",
-                choices=['firefox', 'chrome'])
-        add_arg('-b', '--binary', required=True, dest='binary',
-                help="path to the browser executable that we are testing")
-        add_arg('--branchName', dest="branch_name", default=None,
+        add_arg('--branchName', dest="branch_name", default='',
                 help="Name of the branch we are testing on")
         add_arg('--symbolsPath', dest='symbols_path',
                 help="Path to the symbols for the build we are testing")
         add_arg('--run-local', dest="run_local", default=False, action="store_true",
                 help="Flag that indicates if raptor is running locally or in production")
         add_arg('--obj-path', dest="obj_path", default=None,
                 help="Browser build obj_path (received when running in production)")
 
     add_logging_group(parser)
     return parser
 
 
 def verify_options(parser, args):
     ctx = vars(args)
+    if args.binary is None:
+        parser.error("--binary is required!")
 
     if not os.path.isfile(args.binary):
         parser.error("{binary} does not exist!".format(**ctx))
 
 
 def parse_args(argv=None):
     parser = create_parser()
     args = parser.parse_args(argv)
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -115,17 +115,19 @@ class Raptor(object):
                         test['name'],
                         self.control_server.port,
                         benchmark_port)
 
         # must intall raptor addon each time because we dynamically update some content
         raptor_webext = os.path.join(webext_dir, 'raptor')
         self.log.info("installing webext %s" % raptor_webext)
         self.profile.addons.install(raptor_webext)
-        webext_id = self.profile.addons.addon_details(raptor_webext)['id']
+        # on firefox we can get an addon id; chrome addon actually is just cmd line arg
+        if self.config['app'] == "firefox":
+            webext_id = self.profile.addons.addon_details(raptor_webext)['id']
 
         # some tests require tools to playback the test pages
         if test.get('playback', None) is not None:
             self.get_playback_config(test)
             # startup the playback tool
             self.playback = get_playback(self.config)
 
         self.runner.start()
@@ -141,18 +143,20 @@ class Raptor(object):
                 self.runner.check_for_crashes()
             except NotImplementedError:  # not implemented for Chrome
                 pass
 
         if self.playback is not None:
             self.playback.stop()
 
         # remove the raptor webext; as it must be reloaded with each subtest anyway
-        self.log.info("removing webext %s" % raptor_webext)
-        self.profile.addons.remove_addon(webext_id)
+        # applies to firefox only; chrome the addon is actually just cmd line arg
+        if self.config['app'] == "firefox":
+            self.log.info("removing webext %s" % raptor_webext)
+            self.profile.addons.remove_addon(webext_id)
 
         if self.runner.is_running():
             self.log("Application timed out after {} seconds".format(timeout))
             self.runner.stop()
 
     def process_results(self):
         # when running locally output results in build/raptor.json; when running
         # in production output to a local.json to be turned into tc job artifact
@@ -174,16 +178,18 @@ class Raptor(object):
         self.log.info("finished")
 
 
 def main(args=sys.argv[1:]):
     args = parse_args()
     commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
     LOG = get_default_logger(component='raptor-main')
 
+    LOG.info("received command line arguments: %s" % str(args))
+
     # if a test name specified on command line, and it exists, just run that one
     # otherwise run all available raptor tests that are found for this browser
     raptor_test_list = get_raptor_test_list(args)
 
     # ensure we have at least one valid test to run
     if len(raptor_test_list) == 0:
         LOG.critical("abort: no tests found")
         sys.exit(1)
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ b/testing/raptor/raptor/tests/raptor-speedometer.ini
@@ -1,15 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # speedometer benchmark for firefox and chrome
 
 [raptor-speedometer]
-apps = firefox
+apps = firefox, chrome
 type =  benchmark
 test_url = http://localhost:<port>/Speedometer/index.html?raptor
 page_cycles = 5
 page_timeout = 120000
 unit = score
 lower_is_better = false
 alert_threshold = 2.0
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -11,16 +11,17 @@
   "background": {
     "scripts": ["auto_gen_test_config.js", "runner.js"]
   },
   "content_scripts": [
     {
       "matches": ["*://*.amazon.com/*",
                   "*://*.facebook.com/*",
                   "*://*.google.com/*",
+                  "*://*.google.ca/*",
                   "*://*.youtube.com/*"],
       "js": ["measure.js"]
     },
     {
       "matches": ["*://*/Speedometer/index.html*"],
       "js": ["benchmark-relay.js"]
     }
   ],