--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -49,16 +49,17 @@ MACH_MODULES = [
'taskcluster/mach_commands.py',
'testing/awsy/mach_commands.py',
'testing/firefox-ui/mach_commands.py',
'testing/geckodriver/mach_commands.py',
'testing/mach_commands.py',
'testing/marionette/mach_commands.py',
'testing/mochitest/mach_commands.py',
'testing/mozharness/mach_commands.py',
+ 'testing/raptor/mach_commands.py',
'testing/talos/mach_commands.py',
'testing/web-platform/mach_commands.py',
'testing/xpcshell/mach_commands.py',
'tools/compare-locales/mach_commands.py',
'tools/docs/mach_commands.py',
'tools/lint/mach_commands.py',
'tools/mach_commands.py',
'tools/power/mach_commands.py',
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/linux64_config_taskcluster.py
@@ -0,0 +1,45 @@
+import os
+import sys
+
+PYTHON = sys.executable
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+
+exes = {
+ 'python': PYTHON,
+}
+ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
+INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
+
+config = {
+ "log_name": "raptor",
+ "buildbot_json_path": "buildprops.json",
+ "installer_path": INSTALLER_PATH,
+ "virtualenv_path": VENV_PATH,
+ "find_links": [
+ "http://pypi.pvt.build.mozilla.org/pub",
+ "http://pypi.pub.build.mozilla.org/pub",
+ ],
+ "pip_index": False,
+ "exes": exes,
+ "title": os.uname()[1].lower().split('.')[0],
+ "default_actions": [
+ "clobber",
+ "read-buildbot-config",
+ "download-and-extract",
+ "populate-webroot",
+ "create-virtualenv",
+ "install",
+ "run-tests",
+ ],
+ "default_blob_upload_servers": [
+ "https://blobupload.elasticbeanstalk.com",
+ ],
+ "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+ "download_minidump_stackwalk": True,
+ "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+ "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+ "tooltool_cache": "/builds/worker/tooltool-cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/linux_config.py
@@ -0,0 +1,41 @@
+import os
+import platform
+
+VENV_PATH = '%s/build/venv' % os.getcwd()
+if platform.architecture()[0] == '64bit':
+ TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+ MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+else:
+ TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
+ MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
+
+config = {
+ "log_name": "raptor",
+ "buildbot_json_path": "buildprops.json",
+ "installer_path": "installer.exe",
+ "virtualenv_path": VENV_PATH,
+ "find_links": [
+ "http://pypi.pvt.build.mozilla.org/pub",
+ "http://pypi.pub.build.mozilla.org/pub",
+ ],
+ "pip_index": False,
+ "title": os.uname()[1].lower().split('.')[0],
+ "default_actions": [
+ "clobber",
+ "read-buildbot-config",
+ "download-and-extract",
+ "populate-webroot",
+ "create-virtualenv",
+ "install",
+ "setup-mitmproxy",
+ "run-tests",
+ ],
+ "default_blob_upload_servers": [
+ "https://blobupload.elasticbeanstalk.com",
+ ],
+ "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+ "download_minidump_stackwalk": True,
+ "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+ "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+ "tooltool_cache": "/builds/tooltool_cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/mac_config.py
@@ -0,0 +1,50 @@
+ENABLE_SCREEN_RESOLUTION_CHECK = True
+
+SCREEN_RESOLUTION_CHECK = {
+ "name": "check_screen_resolution",
+ "cmd": ["bash", "-c", "screenresolution get && screenresolution list && system_profiler SPDisplaysDataType"],
+ "architectures": ["32bit", "64bit"],
+ "halt_on_failure": False,
+ "enabled": ENABLE_SCREEN_RESOLUTION_CHECK
+}
+
+import os
+
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+config = {
+ "log_name": "raptor",
+ "buildbot_json_path": "buildprops.json",
+ "installer_path": "installer.exe",
+ "virtualenv_path": VENV_PATH,
+ "find_links": [
+ "http://pypi.pvt.build.mozilla.org/pub",
+ "http://pypi.pub.build.mozilla.org/pub",
+ ],
+ "pip_index": False,
+ "title": os.uname()[1].lower().split('.')[0],
+ "default_actions": [
+ "clobber",
+ "read-buildbot-config",
+ "download-and-extract",
+ "populate-webroot",
+ "create-virtualenv",
+ "install",
+ "run-tests",
+ ],
+ "run_cmd_checks_enabled": True,
+ "preflight_run_cmd_suites": [
+ SCREEN_RESOLUTION_CHECK,
+ ],
+ "postflight_run_cmd_suites": [
+ SCREEN_RESOLUTION_CHECK,
+ ],
+ "default_blob_upload_servers": [
+ "https://blobupload.elasticbeanstalk.com",
+ ],
+ "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+ "download_minidump_stackwalk": True,
+ "minidump_stackwalk_path": "macosx64-minidump_stackwalk",
+ "minidump_tooltool_manifest_path": "config/tooltool-manifests/macosx64/releng.manifest",
+ "tooltool_cache": "/builds/tooltool_cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/windows_config.py
@@ -0,0 +1,56 @@
+import os
+import socket
+import sys
+
+PYTHON = sys.executable
+PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
+VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
+
+config = {
+ "log_name": "raptor",
+ "buildbot_json_path": "buildprops.json",
+ "installer_path": "installer.exe",
+ "virtualenv_path": VENV_PATH,
+ "pip_index": False,
+ "find_links": [
+ "http://pypi.pvt.build.mozilla.org/pub",
+ "http://pypi.pub.build.mozilla.org/pub",
+ ],
+ "virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
+ "exes": {
+ 'python': PYTHON,
+ 'easy_install': ['%s/scripts/python' % VENV_PATH,
+ '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
+ 'mozinstall': ['%s/scripts/python' % VENV_PATH,
+ '%s/scripts/mozinstall-script.py' % VENV_PATH],
+ 'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+ 'tooltool.py': [PYTHON, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
+ },
+ "title": socket.gethostname().split('.')[0],
+ "default_actions": [
+ "clobber",
+ "read-buildbot-config",
+ "download-and-extract",
+ "populate-webroot",
+ "create-virtualenv",
+ "install",
+ "run-tests",
+ ],
+ "default_blob_upload_servers": [
+ "https://blobupload.elasticbeanstalk.com",
+ ],
+ "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+ "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
+ "download_minidump_stackwalk": True,
+ "tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
+ "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
+ "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+ "python3_manifest": {
+ "win32": "python3.manifest",
+ "win64": "python3_x64.manifest",
+ },
+ "env": {
+ # python3 requires C runtime, found in firefox installation; see bug 1361732
+ "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
+ }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/windows_vm_config.py
@@ -0,0 +1,55 @@
+import os
+import socket
+import sys
+
+PYTHON = sys.executable
+PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
+VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
+
+config = {
+ "log_name": "raptor",
+ "buildbot_json_path": "buildprops.json",
+ "installer_path": "installer.exe",
+ "virtualenv_path": VENV_PATH,
+ "pip_index": False,
+ "find_links": [
+ "http://pypi.pvt.build.mozilla.org/pub",
+ "http://pypi.pub.build.mozilla.org/pub",
+ ],
+ "virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
+ "exes": {
+ 'python': PYTHON,
+ 'easy_install': ['%s/scripts/python' % VENV_PATH,
+ '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
+ 'mozinstall': ['%s/scripts/python' % VENV_PATH,
+ '%s/scripts/mozinstall-script.py' % VENV_PATH],
+ 'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+ },
+ "title": socket.gethostname().split('.')[0],
+ "default_actions": [
+ "clobber",
+ "read-buildbot-config",
+ "download-and-extract",
+ "populate-webroot",
+ "create-virtualenv",
+ "install",
+ "run-tests",
+ ],
+ "default_blob_upload_servers": [
+ "https://blobupload.elasticbeanstalk.com",
+ ],
+ "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+ "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
+ "download_minidump_stackwalk": True,
+ "tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
+ "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
+ "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+ "python3_manifest": {
+ "win32": "python3.manifest",
+ "win64": "python3_x64.manifest",
+ },
+ "env": {
+ # python3 requires C runtime, found in firefox installation; see bug 1361732
+ "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
+ }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -0,0 +1,401 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import os
+import re
+import sys
+
+import mozharness
+
+from mozharness.base.config import parse_config_file
+from mozharness.base.errors import PythonErrorList
+from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO, WARNING
+from mozharness.base.python import Python3Virtualenv
+from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.mozilla.tooltool import TooltoolMixin
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.testing.codecoverage import (
+ CodeCoverageMixin,
+ code_coverage_config_options
+)
+
+scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
+external_tools_path = os.path.join(scripts_path, 'external_tools')
+
+RaptorErrorList = PythonErrorList + [
+ {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
+ {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
+ {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
+ {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
+ {'substr': r'''raptorError''', 'level': CRITICAL},
+ {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
+ {'substr': r"""No such file or directory: 'browser_output.txt'""",
+ 'level': CRITICAL,
+ 'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
+]
+
+class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin):
+ """
+ install and run raptor tests
+ """
+ config_options = [
+ [["--test"],
+ {"action": "store",
+ "dest": "test",
+ "help": "Raptor test to run"
+ }],
+ [["--branch-name"],
+ {"action": "store",
+ "dest": "branch",
+ "help": "branch running against"
+ }],
+ [["--add-option"],
+ {"action": "extend",
+ "dest": "raptor_extra_options",
+ "default": None,
+ "help": "extra options to raptor"
+ }],
+ ] + testing_config_options + copy.deepcopy(blobupload_config_options) \
+ + copy.deepcopy(code_coverage_config_options)
+
+ def __init__(self, **kwargs):
+ kwargs.setdefault('config_options', self.config_options)
+ kwargs.setdefault('all_actions', ['clobber',
+ 'read-buildbot-config',
+ 'download-and-extract',
+ 'populate-webroot',
+ 'create-virtualenv',
+ 'install',
+ 'run-tests',
+ ])
+ kwargs.setdefault('default_actions', ['clobber',
+ 'download-and-extract',
+ 'populate-webroot',
+ 'create-virtualenv',
+ 'install',
+ 'run-tests',
+ ])
+ kwargs.setdefault('config', {})
+ super(Raptor, self).__init__(**kwargs)
+
+ self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
+
+ self.run_local = self.config.get('run_local')
+ self.installer_url = self.config.get("installer_url")
+ self.raptor_json_url = self.config.get("raptor_json_url")
+ self.raptor_json = self.config.get("raptor_json")
+ self.raptor_json_config = self.config.get("raptor_json_config")
+ self.repo_path = self.config.get("repo_path")
+ self.obj_path = self.config.get("obj_path")
+ self.tests = None
+ self.gecko_profile = self.config.get('gecko_profile')
+ self.gecko_profile_interval = self.config.get('gecko_profile_interval')
+ self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary
+ self.mitmproxy_pageset = None # zip file found on tooltool that contains all of the mitmproxy recordings
+ self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set
+ self.mitmdump = None # path to mitmdump tool itself, in py3 venv
+
+ # We accept some configuration options from the try commit message in the format mozharness: <options>
+ # Example try commit message:
+ # mozharness: --geckoProfile try: <stuff>
+ def query_gecko_profile_options(self):
+ gecko_results = []
+ if self.buildbot_config:
+ # this is inside automation
+ # now let's see if we added GeckoProfile specs in the commit message
+ try:
+ junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:')
+ except IndexError:
+ # when we don't have comments on changes (bug 1255187)
+ opts = None
+
+ if opts:
+ # In the case of a multi-line commit message, only examine
+ # the first line for mozharness options
+ opts = opts.split('\n')[0]
+ opts = re.sub(r'\w+:.*', '', opts).strip().split(' ')
+ if "--geckoProfile" in opts:
+ # overwrite whatever was set here.
+ self.gecko_profile = True
+ try:
+ idx = opts.index('--geckoProfileInterval')
+ if len(opts) > idx + 1:
+ self.gecko_profile_interval = opts[idx + 1]
+ except ValueError:
+ pass
+ else:
+ # no opts, check for '--geckoProfile' in try message text directly
+ if self.try_message_has_flag('geckoProfile'):
+ self.gecko_profile = True
+
+ # finally, if gecko_profile is set, we add that to the raptor options
+ if self.gecko_profile:
+ gecko_results.append('--geckoProfile')
+ if self.gecko_profile_interval:
+ gecko_results.extend(
+ ['--geckoProfileInterval', str(self.gecko_profile_interval)]
+ )
+ return gecko_results
+
+ def query_abs_dirs(self):
+ if self.abs_dirs:
+ return self.abs_dirs
+ abs_dirs = super(Raptor, self).query_abs_dirs()
+ abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+ abs_dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
+ self.abs_dirs = abs_dirs
+ return self.abs_dirs
+
+ def raptor_options(self, args=None, **kw):
+ """return options to raptor"""
+ # binary path
+ binary_path = self.binary_path or self.config.get('binary_path')
+ if not binary_path:
+ self.fatal("Raptor requires a path to the binary. You can specify binary_path or add download-and-extract to your action list.")
+ # raptor options
+ if binary_path.endswith('.exe'):
+ binary_path = binary_path[:-4]
+ options = []
+ kw_options = {'binary': binary_path}
+ # options overwritten from **kw
+ if 'suite' in self.config:
+ kw_options['suite'] = self.config['suite']
+ if self.config.get('branch'):
+ kw_options['branchName'] = self.config['branch']
+ if self.symbols_path:
+ kw_options['symbolsPath'] = self.symbols_path
+ kw_options.update(kw)
+ # configure profiling options
+ options.extend(self.query_gecko_profile_options())
+ # extra arguments
+ if args is not None:
+ options += args
+ if 'raptor_extra_options' in self.config:
+ options += self.config['raptor_extra_options']
+ if self.config.get('code_coverage', False):
+ options.extend(['--code-coverage'])
+ for key, value in kw_options.items():
+ options.extend(['--%s' % key, value])
+ return options
+
+ def populate_webroot(self):
+ """Populate the production test slaves' webroots"""
+ self.raptor_path = os.path.join(
+ self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
+ )
+
+ if self.config.get('run_local'):
+ # raptor initiated locally, get and verify test from cmd line
+ self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
+ if 'raptor_extra_options' in self.config:
+ if '--test' in self.config['raptor_extra_options']:
+ # --test specified, get test from cmd line and ensure is valid
+ test_name_index = self.config['raptor_extra_options'].index('--test') + 1
+ if test_name_index < len(self.config['raptor_extra_options']):
+ self.test = self.config['raptor_extra_options'][test_name_index]
+ else:
+ self.fatal("Test name not provided")
+ else:
+ # raptor initiated in production via mozharness
+ self.test = self.config['test']
+
+ # Action methods. {{{1
+ # clobber defined in BaseScript
+ # read_buildbot_config defined in BuildbotMixin
+
+ def download_and_extract(self, extract_dirs=None, suite_categories=None):
+ return super(Raptor, self).download_and_extract(
+ suite_categories=['common', 'raptor']
+ )
+
+ def create_virtualenv(self, **kwargs):
+ """VirtualenvMixin.create_virtualenv() assuemes we're using
+ self.config['virtualenv_modules']. Since we are installing
+ raptor from its source, we have to wrap that method here."""
+ # if virtualenv already exists, just add to path and don't re-install, need it
+ # in path so can import jsonschema later when validating output for perfherder
+ _virtualenv_path = self.config.get("virtualenv_path")
+
+ if self.run_local and os.path.exists(_virtualenv_path):
+ self.info("Virtualenv already exists, skipping creation")
+ _python_interp = self.config.get('exes')['python']
+
+ if 'win' in self.platform_name():
+ _path = os.path.join(_virtualenv_path,
+ 'Lib',
+ 'site-packages')
+ else:
+ _path = os.path.join(_virtualenv_path,
+ 'lib',
+ os.path.basename(_python_interp),
+ 'site-packages')
+ sys.path.append(_path)
+ return
+
+ # virtualenv doesn't already exist so create it
+ # install mozbase first, so we use in-tree versions
+ if not self.run_local:
+ mozbase_requirements = os.path.join(
+ self.query_abs_dirs()['abs_test_install_dir'],
+ 'config',
+ 'mozbase_requirements.txt'
+ )
+ else:
+ mozbase_requirements = os.path.join(
+ os.path.dirname(self.raptor_path),
+ 'config',
+ 'mozbase_source_requirements.txt'
+ )
+ self.register_virtualenv_module(
+ requirements=[mozbase_requirements],
+ two_pass=True,
+ editable=True,
+ )
+ # require pip >= 1.5 so pip will prefer .whl files to install
+ super(Raptor, self).create_virtualenv(
+ modules=['pip>=1.5']
+ )
+ # raptor in harness requires what else is
+ # listed in raptor requirements.txt file.
+ self.install_module(
+ requirements=[os.path.join(self.raptor_path,
+ 'requirements.txt')]
+ )
+
+ def _validate_treeherder_data(self, parser):
+ # late import is required, because install is done in create_virtualenv
+ import jsonschema
+
+ if len(parser.found_perf_data) != 1:
+ self.critical("PERFHERDER_DATA was seen %d times, expected 1."
+ % len(parser.found_perf_data))
+ return
+
+ schema_path = os.path.join(external_tools_path,
+ 'performance-artifact-schema.json')
+ self.info("Validating PERFHERDER_DATA against %s" % schema_path)
+ try:
+ with open(schema_path) as f:
+ schema = json.load(f)
+ data = json.loads(parser.found_perf_data[0])
+ jsonschema.validate(data, schema)
+ except:
+ self.exception("Error while validating PERFHERDER_DATA")
+
+ def _artifact_perf_data(self, dest):
+ src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'local.json')
+ try:
+ shutil.copyfile(src, dest)
+ except:
+ self.critical("Error copying results %s to upload dir %s" % (src, dest))
+
+ def run_tests(self, args=None, **kw):
+ """run raptor tests"""
+
+ # get raptor options
+ options = self.raptor_options(args=args, **kw)
+
+ # python version check
+ python = self.query_python_path()
+ self.run_command([python, "--version"])
+ parser = RaptorOutputParser(config=self.config, log_obj=self.log_obj,
+ error_list=RaptorErrorList)
+ env = {}
+ env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
+ if not self.run_local:
+ env['MINIDUMP_STACKWALK'] = self.query_minidump_stackwalk()
+ env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
+ env['RUST_BACKTRACE'] = 'full'
+ if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
+ self.mkdir_p(env['MOZ_UPLOAD_DIR'])
+ env = self.query_env(partial_env=env, log_level=INFO)
+ # adjust PYTHONPATH to be able to use raptor as a python package
+ if 'PYTHONPATH' in env:
+ env['PYTHONPATH'] = self.raptor_path + os.pathsep + env['PYTHONPATH']
+ else:
+ env['PYTHONPATH'] = self.raptor_path
+
+ # mitmproxy needs path to mozharness when installing the cert
+ env['SCRIPTSPATH'] = scripts_path
+
+ if self.repo_path is not None:
+ env['MOZ_DEVELOPER_REPO_DIR'] = self.repo_path
+ if self.obj_path is not None:
+ env['MOZ_DEVELOPER_OBJ_DIR'] = self.obj_path
+
+ # sets a timeout for how long raptor should run without output
+ output_timeout = self.config.get('raptor_output_timeout', 3600)
+ # run raptor tests
+ run_tests = os.path.join(self.raptor_path, 'raptor', 'raptor.py')
+
+ mozlog_opts = ['--log-tbpl-level=debug']
+ if not self.run_local and 'suite' in self.config:
+ fname_pattern = '%s_%%s.log' % self.config['test']
+ mozlog_opts.append('--log-errorsummary=%s'
+ % os.path.join(env['MOZ_UPLOAD_DIR'],
+ fname_pattern % 'errorsummary'))
+ mozlog_opts.append('--log-raw=%s'
+ % os.path.join(env['MOZ_UPLOAD_DIR'],
+ fname_pattern % 'raw'))
+
+ def launch_in_debug_mode(cmdline):
+ cmdline = set(cmdline)
+ debug_opts = {'--debug', '--debugger', '--debugger_args'}
+
+ return bool(debug_opts.intersection(cmdline))
+
+ command = [python, run_tests] + options + mozlog_opts
+ if launch_in_debug_mode(command):
+ raptor_process = subprocess.Popen(command, cwd=self.workdir, env=env)
+ raptor_process.wait()
+ else:
+ self.return_code = self.run_command(command, cwd=self.workdir,
+ output_timeout=output_timeout,
+ output_parser=parser,
+ env=env)
+ if parser.minidump_output:
+ self.info("Looking at the minidump files for debugging purposes...")
+ for item in parser.minidump_output:
+ self.run_command(["ls", "-l", item])
+
+ if self.return_code not in [0]:
+ # update the worst log level
+ log_level = ERROR
+ if self.return_code == 1:
+ log_level = WARNING
+ if self.return_code == 4:
+ log_level = WARNING
+
+ elif '--no-upload-results' not in options:
+ if not self.gecko_profile:
+ self._validate_treeherder_data(parser)
+ if not self.run_local:
+ # copy results to upload dir so they are included as an artifact
+ dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
+ self._artifact_perf_data(dest)
+
+
+class RaptorOutputParser(OutputParser):
+ minidump_regex = re.compile(r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''')
+ RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
+
+ def __init__(self, **kwargs):
+ super(RaptorOutputParser, self).__init__(**kwargs)
+ self.minidump_output = None
+ self.found_perf_data = []
+
+ def parse_single_line(self, line):
+ m = self.minidump_regex.search(line)
+ if m:
+ self.minidump_output = (m.group(1), m.group(2), m.group(3))
+
+ m = self.RE_PERF_DATA.match(line)
+ if m:
+ self.found_perf_data.append(m.group(1))
+ super(RaptorOutputParser, self).parse_single_line(line)
+
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/raptor_script.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""raptor
+
+"""
+
+import os
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.mozilla.testing.raptor import Raptor
+
+if __name__ == '__main__':
+ raptor = Raptor()
+ raptor.run_and_exit()
new file mode 100644
--- /dev/null
+++ b/testing/raptor/mach_commands.py
@@ -0,0 +1,118 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Originally taken from /talos/mach_commands.py
+
+# Integrates raptor mozharness with mach
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import sys
+import json
+import socket
+
+from mozbuild.base import MozbuildObject, MachCommandBase
+from mach.decorators import CommandProvider, Command
+
+HERE = os.path.dirname(os.path.realpath(__file__))
+
+
+class RaptorRunner(MozbuildObject):
+ def run_test(self, raptor_args):
+ """
+ We want to do couple of things before running raptor
+ 1. Clone mozharness
+ 2. Make config for raptor mozharness
+ 3. Run mozharness
+ """
+
+ self.init_variables(raptor_args)
+ self.make_config()
+ self.write_config()
+ self.make_args()
+ return self.run_mozharness()
+
+ def init_variables(self, raptor_args):
+ self.raptor_dir = os.path.join(self.topsrcdir, 'testing', 'raptor')
+ self.mozharness_dir = os.path.join(self.topsrcdir, 'testing',
+ 'mozharness')
+ self.config_file_path = os.path.join(self._topobjdir, 'testing',
+ 'raptor-in_tree_conf.json')
+ self.binary_path = self.get_binary_path()
+ self.virtualenv_script = os.path.join(self.topsrcdir, 'third_party', 'python',
+ 'virtualenv', 'virtualenv.py')
+ self.virtualenv_path = os.path.join(self._topobjdir, 'testing',
+ 'raptor-venv')
+ self.python_interp = sys.executable
+ self.raptor_args = raptor_args
+
+ def make_config(self):
+ default_actions = ['populate-webroot', 'create-virtualenv', 'run-tests']
+ self.config = {
+ 'run_local': True,
+ 'binary_path': self.binary_path,
+ 'repo_path': self.topsrcdir,
+ 'raptor_path': self.raptor_dir,
+ 'obj_path': self.topobjdir,
+ 'log_name': 'raptor',
+ 'virtualenv_path': self.virtualenv_path,
+ 'pypi_url': 'http://pypi.python.org/simple',
+ 'base_work_dir': self.mozharness_dir,
+ 'exes': {
+ 'python': self.python_interp,
+ 'virtualenv': [self.python_interp, self.virtualenv_script],
+ },
+ 'title': socket.gethostname(),
+ 'default_actions': default_actions,
+ 'raptor_extra_options': self.raptor_args,
+ 'python3_manifest': {
+ 'win32': 'python3.manifest',
+ 'win64': 'python3_x64.manifest',
+ }
+ }
+
+ def make_args(self):
+ self.args = {
+ 'config': {},
+ 'initial_config_file': self.config_file_path,
+ }
+
+ def write_config(self):
+ try:
+ config_file = open(self.config_file_path, 'wb')
+ config_file.write(json.dumps(self.config))
+ config_file.close()
+ except IOError as e:
+ err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
+ print(err_str.format(self.config_file_path, str(e)))
+ raise e
+
+ def run_mozharness(self):
+ sys.path.insert(0, self.mozharness_dir)
+ from mozharness.mozilla.testing.raptor import Raptor
+ raptor_mh = Raptor(config=self.args['config'],
+ initial_config_file=self.args['initial_config_file'])
+ return raptor_mh.run()
+
+
+def create_parser():
+ sys.path.insert(0, HERE) # allow to import the raptor package
+ from raptor.cmdline import create_parser
+ return create_parser(mach_interface=True)
+
+
+@CommandProvider
+class MachRaptor(MachCommandBase):
+ @Command('raptor-test', category='testing',
+ description='Run raptor performance tests.',
+ parser=create_parser)
+ def run_raptor_test(self, **kwargs):
+ raptor = self._spawn(RaptorRunner)
+
+ try:
+ return raptor.run_test(sys.argv[2:])
+ except Exception as e:
+ print(str(e))
+ return 1
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -8,23 +8,27 @@ import os
from mozlog.commandline import add_logging_group
def create_parser(mach_interface=False):
parser = argparse.ArgumentParser()
add_arg = parser.add_argument
- add_arg('-t', '--test', default=None, dest="test",
+ if not mach_interface:
+ add_arg('--app', default='firefox', dest='app',
+ help="name of the application we are testing (default: firefox)",
+ choices=['firefox', 'chrome'])
+ add_arg('-b', '--binary', required=True, dest='binary',
+ help="path to the browser executable that we are testing")
+
+ # remaining arg is test name
+ add_arg("test",
+ nargs="*",
help="name of raptor test to run")
- add_arg('--app', default='firefox', dest='app',
- help="name of the application we are testing (default: firefox)",
- choices=['firefox', 'chrome'])
- add_arg('-b', '--binary', required=True,
- help="path to the browser executable that we are testing")
add_logging_group(parser)
return parser
def verify_options(parser, args):
ctx = vars(args)
--- a/testing/raptor/raptor/control_server.py
+++ b/testing/raptor/raptor/control_server.py
@@ -1,19 +1,20 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-# simple local server on port 8000, to demonstrate
-# receiving hero element timing results from a web extension
+# control server for raptor performance framework
+# communicates with the raptor browser webextension
from __future__ import absolute_import
import BaseHTTPServer
import json
import os
+import socket
import threading
from mozlog import get_proxy_logger
LOG = get_proxy_logger(component='control_server')
here = os.path.abspath(os.path.dirname(__file__))
@@ -64,29 +65,36 @@ class MyHandler(BaseHTTPServer.BaseHTTPR
class RaptorControlServer():
"""Container class for Raptor Control Server"""
def __init__(self):
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.server = None
self._server_thread = None
+ self.port = None
def start(self):
config_dir = os.path.join(here, 'tests')
os.chdir(config_dir)
- server_address = ('', 8000)
+
+ # pick a free port
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.bind(('', 0))
+ self.port = sock.getsockname()[1]
+ sock.close()
+ server_address = ('', self.port)
server_class = BaseHTTPServer.HTTPServer
handler_class = MyHandler
httpd = server_class(server_address, handler_class)
self._server_thread = threading.Thread(target=httpd.serve_forever)
self._server_thread.setDaemon(True) # don't hang on exit
self._server_thread.start()
- LOG.info("raptor control server running on port 8000...")
+ LOG.info("raptor control server running on port %d..." % self.port)
self.server = httpd
def stop(self):
LOG.info("shutting down control server")
self.server.shutdown()
self._server_thread.join()
--- a/testing/raptor/raptor/gen_test_config.py
+++ b/testing/raptor/raptor/gen_test_config.py
@@ -8,25 +8,25 @@ import os
from mozlog import get_proxy_logger
here = os.path.abspath(os.path.dirname(__file__))
webext_dir = os.path.join(os.path.dirname(here), 'webext', 'raptor')
LOG = get_proxy_logger(component="gen_test_url")
-def gen_test_config(browser, test):
+def gen_test_config(browser, test, cs_port):
LOG.info("writing test settings url background js, so webext can get it")
data = """// this file is auto-generated by raptor, do not edit directly
function getTestConfig() {
- return {"browser": "%s", "test_settings_url": "http://localhost:8000/%s.json"};
+ return {"browser": "%s", "test_settings_url": "http://localhost:%d/%s.json"};
}
-""" % (browser, test)
+""" % (browser, cs_port, test)
webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
file = open(webext_background_script, "w")
file.write(data)
file.close()
LOG.info("finished writing test config into webext")
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -61,17 +61,18 @@ def write_test_settings_json(test_detail
except IOError:
LOG.info("abort: exception writing test settings json!")
def get_raptor_test_list(args):
# get a list of available raptor tests, for the browser we're testing on
available_tests = get_browser_test_list(args.app)
tests_to_run = []
-
+ # currently only support one test name on cmd line
+ args.test = args.test[0]
# if test name not provided on command line, run all available raptor tests for this browser;
# if test name provided on command line, make sure it exists, and then only include that one
if args.test is not None:
for next_test in available_tests:
if next_test['name'] == args.test:
tests_to_run = [next_test]
break
if len(tests_to_run) == 0:
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -11,25 +11,27 @@ import sys
import time
import mozinfo
from mozlog import commandline, get_default_logger
from mozprofile import create_profile
from mozrunner import runners
-from raptor.cmdline import parse_args
-from raptor.control_server import RaptorControlServer
-from raptor.gen_test_config import gen_test_config
-from raptor.outputhandler import OutputHandler
-from raptor.playback import get_playback
-from raptor.manifest import get_raptor_test_list
-
+# need this so raptor imports work both from /raptor and via mach
here = os.path.abspath(os.path.dirname(__file__))
webext_dir = os.path.join(os.path.dirname(here), 'webext')
+sys.path.insert(0, here)
+
+from cmdline import parse_args
+from control_server import RaptorControlServer
+from gen_test_config import gen_test_config
+from outputhandler import OutputHandler
+from playback import get_playback
+from manifest import get_raptor_test_list
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary):
self.config = {}
self.config['app'] = app
@@ -74,17 +76,17 @@ class Raptor(object):
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings', None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
- gen_test_config(self.config['app'], test['name'])
+ gen_test_config(self.config['app'], test['name'], self.control_server.port)
self.profile.addons.install(os.path.join(webext_dir, 'raptor'))
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config)
deleted file mode 100644
--- a/testing/raptor/raptor/tests/raptor-chrome-tp7.ini
+++ /dev/null
@@ -1,22 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor tp7 chrome
-
-[DEFAULT]
-apps = chrome
-type = pageload
-playback = mitmproxy
-release_bin_mac = mitmproxy-2.0.2-osx.tar.gz
-page_cycles = 25
-
-[raptor-chrome-tp7]
-test_url = http://localhost:8081/heroes
-measure =
- fcp
- hero
-hero =
- mugshot
- title
- anime
--- a/testing/raptor/raptor/tests/raptor-firefox-tp6.ini
+++ b/testing/raptor/raptor/tests/raptor-firefox-tp6.ini
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-# raptor tp6 firefox
+# raptor tp6 on firefox
[DEFAULT]
apps = firefox
type = pageload
playback = mitmproxy
playback_binary_manifest = mitmproxy-rel-bin-osx.manifest
playback_binary_zip_mac = mitmproxy-2.0.2-osx.tar.gz
playback_pageset_manifest = mitmproxy-playback-set.manifest
deleted file mode 100644
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ /dev/null
@@ -1,14 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor speedometer
-
-[raptor-speedometer]
-apps =
- firefox
- chrome
-type = benchmark
-test_url = http://localhost:8081/Speedometer/index.html?raptor
-page_cycles = 1
-page_timeout = 120000
--- a/testing/raptor/test/test_raptor.py
+++ b/testing/raptor/test/test_raptor.py
@@ -31,22 +31,26 @@ def test_create_profile(options, app, ge
prefs_file = os.path.join(raptor.profile.profile, 'user.js')
with open(prefs_file, 'r') as fh:
prefs = fh.read()
assert firefox_pref in prefs
assert raptor_pref in prefs
def test_start_and_stop_server(raptor):
+ print("*RW* control server is now:")
+ print(str(raptor.control_server))
assert raptor.control_server is None
raptor.start_control_server()
- assert isinstance(raptor.control_server, RaptorControlServer)
assert raptor.control_server._server_thread.is_alive()
+ assert raptor.control_server.port is not None
+ assert raptor.control_server.server is not None
+
raptor.clean_up()
assert not raptor.control_server._server_thread.is_alive()
@pytest.mark.parametrize('app', [
'firefox',
pytest.mark.xfail('chrome'),
])