Bug 1455107 - Integrate raptor into mach; r=gbrown
authorRob Wood <rwood@mozilla.com>
Mon, 23 Apr 2018 16:43:30 -0400
changeset 416107 37a65d39653beaab749b5110c099b840ada555fe
parent 416106 0f3dbbd73ed424a47974388b0cf97db84a5c756c
child 416108 9d124f189e612ef45f466456934ccc1b1a4ff824
push id33916
push userapavel@mozilla.com
push dateSat, 28 Apr 2018 17:29:16 +0000
treeherdermozilla-central@807e94988d9a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1455107
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1455107 - Integrate raptor into mach; r=gbrown MozReview-Commit-ID: 84vIqU2NWkE
build/mach_bootstrap.py
testing/mozharness/configs/raptor/linux64_config_taskcluster.py
testing/mozharness/configs/raptor/linux_config.py
testing/mozharness/configs/raptor/mac_config.py
testing/mozharness/configs/raptor/windows_config.py
testing/mozharness/configs/raptor/windows_vm_config.py
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/mozharness/scripts/raptor_script.py
testing/raptor/mach_commands.py
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/control_server.py
testing/raptor/raptor/gen_test_config.py
testing/raptor/raptor/manifest.py
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/raptor-chrome-tp7.ini
testing/raptor/raptor/tests/raptor-firefox-tp6.ini
testing/raptor/raptor/tests/raptor-speedometer.ini
testing/raptor/test/test_raptor.py
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -49,16 +49,17 @@ MACH_MODULES = [
     'taskcluster/mach_commands.py',
     'testing/awsy/mach_commands.py',
     'testing/firefox-ui/mach_commands.py',
     'testing/geckodriver/mach_commands.py',
     'testing/mach_commands.py',
     'testing/marionette/mach_commands.py',
     'testing/mochitest/mach_commands.py',
     'testing/mozharness/mach_commands.py',
+    'testing/raptor/mach_commands.py',
     'testing/talos/mach_commands.py',
     'testing/web-platform/mach_commands.py',
     'testing/xpcshell/mach_commands.py',
     'tools/compare-locales/mach_commands.py',
     'tools/docs/mach_commands.py',
     'tools/lint/mach_commands.py',
     'tools/mach_commands.py',
     'tools/power/mach_commands.py',
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/linux64_config_taskcluster.py
@@ -0,0 +1,45 @@
+import os
+import sys
+
+PYTHON = sys.executable
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+
+exes = {
+    'python': PYTHON,
+}
+ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
+INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": INSTALLER_PATH,
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "exes": exes,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+    "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+    "tooltool_cache": "/builds/worker/tooltool-cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/linux_config.py
@@ -0,0 +1,41 @@
+import os
+import platform
+
+VENV_PATH = '%s/build/venv' % os.getcwd()
+if platform.architecture()[0] == '64bit':
+    TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+    MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+else:
+    TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
+    MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "setup-mitmproxy",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+    "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+    "tooltool_cache": "/builds/tooltool_cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/mac_config.py
@@ -0,0 +1,50 @@
+ENABLE_SCREEN_RESOLUTION_CHECK = True
+
+SCREEN_RESOLUTION_CHECK = {
+    "name": "check_screen_resolution",
+    "cmd": ["bash", "-c", "screenresolution get && screenresolution list && system_profiler SPDisplaysDataType"],
+    "architectures": ["32bit", "64bit"],
+    "halt_on_failure": False,
+    "enabled": ENABLE_SCREEN_RESOLUTION_CHECK
+}
+
+import os
+
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "run_cmd_checks_enabled": True,
+    "preflight_run_cmd_suites": [
+        SCREEN_RESOLUTION_CHECK,
+    ],
+    "postflight_run_cmd_suites": [
+        SCREEN_RESOLUTION_CHECK,
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": "macosx64-minidump_stackwalk",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/macosx64/releng.manifest",
+    "tooltool_cache": "/builds/tooltool_cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/windows_config.py
@@ -0,0 +1,56 @@
+import os
+import socket
+import sys
+
+PYTHON = sys.executable
+PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
+VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "pip_index": False,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
+    "exes": {
+        'python': PYTHON,
+        'easy_install': ['%s/scripts/python' % VENV_PATH,
+                         '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
+        'mozinstall': ['%s/scripts/python' % VENV_PATH,
+                       '%s/scripts/mozinstall-script.py' % VENV_PATH],
+        'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+        'tooltool.py': [PYTHON, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
+    },
+    "title": socket.gethostname().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
+    "download_minidump_stackwalk": True,
+    "tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
+    "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+    "python3_manifest": {
+        "win32": "python3.manifest",
+        "win64": "python3_x64.manifest",
+    },
+    "env": {
+        # python3 requires C runtime, found in firefox installation; see bug 1361732
+        "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/windows_vm_config.py
@@ -0,0 +1,55 @@
+import os
+import socket
+import sys
+
+PYTHON = sys.executable
+PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
+VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "pip_index": False,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
+    "exes": {
+        'python': PYTHON,
+        'easy_install': ['%s/scripts/python' % VENV_PATH,
+                         '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
+        'mozinstall': ['%s/scripts/python' % VENV_PATH,
+                       '%s/scripts/mozinstall-script.py' % VENV_PATH],
+        'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+    },
+    "title": socket.gethostname().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
+    "download_minidump_stackwalk": True,
+    "tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
+    "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+    "python3_manifest": {
+        "win32": "python3.manifest",
+        "win64": "python3_x64.manifest",
+    },
+    "env": {
+        # python3 requires C runtime, found in firefox installation; see bug 1361732
+        "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -0,0 +1,401 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import os
+import re
+import sys
+
+import mozharness
+
+from mozharness.base.config import parse_config_file
+from mozharness.base.errors import PythonErrorList
+from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO, WARNING
+from mozharness.base.python import Python3Virtualenv
+from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.mozilla.tooltool import TooltoolMixin
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.testing.codecoverage import (
+    CodeCoverageMixin,
+    code_coverage_config_options
+)
+
+scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
+external_tools_path = os.path.join(scripts_path, 'external_tools')
+
+RaptorErrorList = PythonErrorList + [
+    {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
+    {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
+    {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
+    {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
+    {'substr': r'''raptorError''', 'level': CRITICAL},
+    {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
+    {'substr': r"""No such file or directory: 'browser_output.txt'""",
+     'level': CRITICAL,
+     'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
+]
+
+class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin):
+    """
+    install and run raptor tests
+    """
+    config_options = [
+        [["--test"],
+         {"action": "store",
+          "dest": "test",
+          "help": "Raptor test to run"
+          }],
+        [["--branch-name"],
+         {"action": "store",
+          "dest": "branch",
+          "help": "branch running against"
+          }],
+        [["--add-option"],
+         {"action": "extend",
+          "dest": "raptor_extra_options",
+          "default": None,
+          "help": "extra options to raptor"
+          }],
+    ] + testing_config_options + copy.deepcopy(blobupload_config_options) \
+                               + copy.deepcopy(code_coverage_config_options)
+
+    def __init__(self, **kwargs):
+        kwargs.setdefault('config_options', self.config_options)
+        kwargs.setdefault('all_actions', ['clobber',
+                                          'read-buildbot-config',
+                                          'download-and-extract',
+                                          'populate-webroot',
+                                          'create-virtualenv',
+                                          'install',
+                                          'run-tests',
+                                          ])
+        kwargs.setdefault('default_actions', ['clobber',
+                                              'download-and-extract',
+                                              'populate-webroot',
+                                              'create-virtualenv',
+                                              'install',
+                                              'run-tests',
+                                              ])
+        kwargs.setdefault('config', {})
+        super(Raptor, self).__init__(**kwargs)
+
+        self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
+
+        self.run_local = self.config.get('run_local')
+        self.installer_url = self.config.get("installer_url")
+        self.raptor_json_url = self.config.get("raptor_json_url")
+        self.raptor_json = self.config.get("raptor_json")
+        self.raptor_json_config = self.config.get("raptor_json_config")
+        self.repo_path = self.config.get("repo_path")
+        self.obj_path = self.config.get("obj_path")
+        self.tests = None
+        self.gecko_profile = self.config.get('gecko_profile')
+        self.gecko_profile_interval = self.config.get('gecko_profile_interval')
+        self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary
+        self.mitmproxy_pageset = None # zip file found on tooltool that contains all of the mitmproxy recordings
+        self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set
+        self.mitmdump = None # path to mitmdump tool itself, in py3 venv
+
+    # We accept some configuration options from the try commit message in the format mozharness: <options>
+    # Example try commit message:
+    #   mozharness: --geckoProfile try: <stuff>
+    def query_gecko_profile_options(self):
+        gecko_results = []
+        if self.buildbot_config:
+            # this is inside automation
+            # now let's see if we added GeckoProfile specs in the commit message
+            try:
+                junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:')
+            except IndexError:
+                # when we don't have comments on changes (bug 1255187)
+                opts = None
+
+            if opts:
+                # In the case of a multi-line commit message, only examine
+                # the first line for mozharness options
+                opts = opts.split('\n')[0]
+                opts = re.sub(r'\w+:.*', '', opts).strip().split(' ')
+                if "--geckoProfile" in opts:
+                    # overwrite whatever was set here.
+                    self.gecko_profile = True
+                try:
+                    idx = opts.index('--geckoProfileInterval')
+                    if len(opts) > idx + 1:
+                        self.gecko_profile_interval = opts[idx + 1]
+                except ValueError:
+                    pass
+            else:
+                # no opts, check for '--geckoProfile' in try message text directly
+                if self.try_message_has_flag('geckoProfile'):
+                    self.gecko_profile = True
+
+        # finally, if gecko_profile is set, we add that to the raptor options
+        if self.gecko_profile:
+            gecko_results.append('--geckoProfile')
+            if self.gecko_profile_interval:
+                gecko_results.extend(
+                    ['--geckoProfileInterval', str(self.gecko_profile_interval)]
+                )
+        return gecko_results
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(Raptor, self).query_abs_dirs()
+        abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+        abs_dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def raptor_options(self, args=None, **kw):
+        """return options to raptor"""
+        # binary path
+        binary_path = self.binary_path or self.config.get('binary_path')
+        if not binary_path:
+            self.fatal("Raptor requires a path to the binary.  You can specify binary_path or add download-and-extract to your action list.")
+        # raptor options
+        if binary_path.endswith('.exe'):
+            binary_path = binary_path[:-4]
+        options = []
+        kw_options = {'binary': binary_path}
+        # options overwritten from **kw
+        if 'suite' in self.config:
+            kw_options['suite'] = self.config['suite']
+        if self.config.get('branch'):
+            kw_options['branchName'] = self.config['branch']
+        if self.symbols_path:
+            kw_options['symbolsPath'] = self.symbols_path
+        kw_options.update(kw)
+        # configure profiling options
+        options.extend(self.query_gecko_profile_options())
+        # extra arguments
+        if args is not None:
+            options += args
+        if 'raptor_extra_options' in self.config:
+            options += self.config['raptor_extra_options']
+        if self.config.get('code_coverage', False):
+            options.extend(['--code-coverage'])
+        for key, value in kw_options.items():
+            options.extend(['--%s' % key, value])
+        return options
+
+    def populate_webroot(self):
+        """Populate the production test slaves' webroots"""
+        self.raptor_path = os.path.join(
+            self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
+        )
+
+        if self.config.get('run_local'):
+            # raptor initiated locally, get and verify test from cmd line
+            self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
+            if 'raptor_extra_options' in self.config:
+                if '--test' in self.config['raptor_extra_options']:
+                    # --test specified, get test from cmd line and ensure is valid
+                    test_name_index = self.config['raptor_extra_options'].index('--test') + 1
+                    if test_name_index < len(self.config['raptor_extra_options']):
+                        self.test = self.config['raptor_extra_options'][test_name_index]
+                    else:
+                        self.fatal("Test name not provided")
+        else:
+            # raptor initiated in production via mozharness
+            self.test = self.config['test']
+
+    # Action methods. {{{1
+    # clobber defined in BaseScript
+    # read_buildbot_config defined in BuildbotMixin
+
+    def download_and_extract(self, extract_dirs=None, suite_categories=None):
+        return super(Raptor, self).download_and_extract(
+            suite_categories=['common', 'raptor']
+        )
+
+    def create_virtualenv(self, **kwargs):
+        """VirtualenvMixin.create_virtualenv() assuemes we're using
+        self.config['virtualenv_modules']. Since we are installing
+        raptor from its source, we have to wrap that method here."""
+        # if virtualenv already exists, just add to path and don't re-install, need it
+        # in path so can import jsonschema later when validating output for perfherder
+        _virtualenv_path = self.config.get("virtualenv_path")
+
+        if self.run_local and os.path.exists(_virtualenv_path):
+            self.info("Virtualenv already exists, skipping creation")
+            _python_interp = self.config.get('exes')['python']
+
+            if 'win' in self.platform_name():
+                _path = os.path.join(_virtualenv_path,
+                                     'Lib',
+                                     'site-packages')
+            else:
+                _path = os.path.join(_virtualenv_path,
+                                     'lib',
+                                     os.path.basename(_python_interp),
+                                     'site-packages')
+            sys.path.append(_path)
+            return
+
+        # virtualenv doesn't already exist so create it
+        # install mozbase first, so we use in-tree versions
+        if not self.run_local:
+            mozbase_requirements = os.path.join(
+                self.query_abs_dirs()['abs_test_install_dir'],
+                'config',
+                'mozbase_requirements.txt'
+            )
+        else:
+            mozbase_requirements = os.path.join(
+                os.path.dirname(self.raptor_path),
+                'config',
+                'mozbase_source_requirements.txt'
+            )
+        self.register_virtualenv_module(
+            requirements=[mozbase_requirements],
+            two_pass=True,
+            editable=True,
+        )
+        # require pip >= 1.5 so pip will prefer .whl files to install
+        super(Raptor, self).create_virtualenv(
+            modules=['pip>=1.5']
+        )
+        # raptor in harness requires what else is
+        # listed in raptor requirements.txt file.
+        self.install_module(
+            requirements=[os.path.join(self.raptor_path,
+                                       'requirements.txt')]
+        )
+
+    def _validate_treeherder_data(self, parser):
+        # late import is required, because install is done in create_virtualenv
+        import jsonschema
+
+        if len(parser.found_perf_data) != 1:
+            self.critical("PERFHERDER_DATA was seen %d times, expected 1."
+                          % len(parser.found_perf_data))
+            return
+
+        schema_path = os.path.join(external_tools_path,
+                                   'performance-artifact-schema.json')
+        self.info("Validating PERFHERDER_DATA against %s" % schema_path)
+        try:
+            with open(schema_path) as f:
+                schema = json.load(f)
+            data = json.loads(parser.found_perf_data[0])
+            jsonschema.validate(data, schema)
+        except:
+            self.exception("Error while validating PERFHERDER_DATA")
+
+    def _artifact_perf_data(self, dest):
+        src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'local.json')
+        try:
+            shutil.copyfile(src, dest)
+        except:
+            self.critical("Error copying results %s to upload dir %s" % (src, dest))
+
+    def run_tests(self, args=None, **kw):
+        """run raptor tests"""
+
+        # get raptor options
+        options = self.raptor_options(args=args, **kw)
+
+        # python version check
+        python = self.query_python_path()
+        self.run_command([python, "--version"])
+        parser = RaptorOutputParser(config=self.config, log_obj=self.log_obj,
+                                   error_list=RaptorErrorList)
+        env = {}
+        env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
+        if not self.run_local:
+            env['MINIDUMP_STACKWALK'] = self.query_minidump_stackwalk()
+        env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
+        env['RUST_BACKTRACE'] = 'full'
+        if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
+            self.mkdir_p(env['MOZ_UPLOAD_DIR'])
+        env = self.query_env(partial_env=env, log_level=INFO)
+        # adjust PYTHONPATH to be able to use raptor as a python package
+        if 'PYTHONPATH' in env:
+            env['PYTHONPATH'] = self.raptor_path + os.pathsep + env['PYTHONPATH']
+        else:
+            env['PYTHONPATH'] = self.raptor_path
+
+        # mitmproxy needs path to mozharness when installing the cert
+        env['SCRIPTSPATH'] = scripts_path
+
+        if self.repo_path is not None:
+            env['MOZ_DEVELOPER_REPO_DIR'] = self.repo_path
+        if self.obj_path is not None:
+            env['MOZ_DEVELOPER_OBJ_DIR'] = self.obj_path
+
+        # sets a timeout for how long raptor should run without output
+        output_timeout = self.config.get('raptor_output_timeout', 3600)
+        # run raptor tests
+        run_tests = os.path.join(self.raptor_path, 'raptor', 'raptor.py')
+
+        mozlog_opts = ['--log-tbpl-level=debug']
+        if not self.run_local and 'suite' in self.config:
+            fname_pattern = '%s_%%s.log' % self.config['test']
+            mozlog_opts.append('--log-errorsummary=%s'
+                               % os.path.join(env['MOZ_UPLOAD_DIR'],
+                                              fname_pattern % 'errorsummary'))
+            mozlog_opts.append('--log-raw=%s'
+                               % os.path.join(env['MOZ_UPLOAD_DIR'],
+                                              fname_pattern % 'raw'))
+
+        def launch_in_debug_mode(cmdline):
+            cmdline = set(cmdline)
+            debug_opts = {'--debug', '--debugger', '--debugger_args'}
+
+            return bool(debug_opts.intersection(cmdline))
+
+        command = [python, run_tests] + options + mozlog_opts
+        if launch_in_debug_mode(command):
+            raptor_process = subprocess.Popen(command, cwd=self.workdir, env=env)
+            raptor_process.wait()
+        else:
+            self.return_code = self.run_command(command, cwd=self.workdir,
+                                            output_timeout=output_timeout,
+                                            output_parser=parser,
+                                            env=env)
+        if parser.minidump_output:
+            self.info("Looking at the minidump files for debugging purposes...")
+            for item in parser.minidump_output:
+                self.run_command(["ls", "-l", item])
+
+        if self.return_code not in [0]:
+            # update the worst log level
+            log_level = ERROR
+            if self.return_code == 1:
+                log_level = WARNING
+            if self.return_code == 4:
+                log_level = WARNING
+
+        elif '--no-upload-results' not in options:
+            if not self.gecko_profile:
+                self._validate_treeherder_data(parser)
+                if not self.run_local:
+                    # copy results to upload dir so they are included as an artifact
+                    dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
+                    self._artifact_perf_data(dest)
+
+
+class RaptorOutputParser(OutputParser):
+    minidump_regex = re.compile(r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''')
+    RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
+
+    def __init__(self, **kwargs):
+        super(RaptorOutputParser, self).__init__(**kwargs)
+        self.minidump_output = None
+        self.found_perf_data = []
+
+    def parse_single_line(self, line):
+        m = self.minidump_regex.search(line)
+        if m:
+            self.minidump_output = (m.group(1), m.group(2), m.group(3))
+
+        m = self.RE_PERF_DATA.match(line)
+        if m:
+            self.found_perf_data.append(m.group(1))
+        super(RaptorOutputParser, self).parse_single_line(line)
+
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/raptor_script.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""raptor
+
+"""
+
+import os
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.mozilla.testing.raptor import Raptor
+
+if __name__ == '__main__':
+    raptor = Raptor()
+    raptor.run_and_exit()
new file mode 100644
--- /dev/null
+++ b/testing/raptor/mach_commands.py
@@ -0,0 +1,118 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Originally taken from /talos/mach_commands.py
+
+# Integrates raptor mozharness with mach
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import sys
+import json
+import socket
+
+from mozbuild.base import MozbuildObject, MachCommandBase
+from mach.decorators import CommandProvider, Command
+
+HERE = os.path.dirname(os.path.realpath(__file__))
+
+
+class RaptorRunner(MozbuildObject):
+    def run_test(self, raptor_args):
+        """
+        We want to do couple of things before running raptor
+        1. Clone mozharness
+        2. Make config for raptor mozharness
+        3. Run mozharness
+        """
+
+        self.init_variables(raptor_args)
+        self.make_config()
+        self.write_config()
+        self.make_args()
+        return self.run_mozharness()
+
+    def init_variables(self, raptor_args):
+        self.raptor_dir = os.path.join(self.topsrcdir, 'testing', 'raptor')
+        self.mozharness_dir = os.path.join(self.topsrcdir, 'testing',
+                                           'mozharness')
+        self.config_file_path = os.path.join(self._topobjdir, 'testing',
+                                             'raptor-in_tree_conf.json')
+        self.binary_path = self.get_binary_path()
+        self.virtualenv_script = os.path.join(self.topsrcdir, 'third_party', 'python',
+                                              'virtualenv', 'virtualenv.py')
+        self.virtualenv_path = os.path.join(self._topobjdir, 'testing',
+                                            'raptor-venv')
+        self.python_interp = sys.executable
+        self.raptor_args = raptor_args
+
+    def make_config(self):
+        default_actions = ['populate-webroot', 'create-virtualenv', 'run-tests']
+        self.config = {
+            'run_local': True,
+            'binary_path': self.binary_path,
+            'repo_path': self.topsrcdir,
+            'raptor_path': self.raptor_dir,
+            'obj_path': self.topobjdir,
+            'log_name': 'raptor',
+            'virtualenv_path': self.virtualenv_path,
+            'pypi_url': 'http://pypi.python.org/simple',
+            'base_work_dir': self.mozharness_dir,
+            'exes': {
+                'python': self.python_interp,
+                'virtualenv': [self.python_interp, self.virtualenv_script],
+            },
+            'title': socket.gethostname(),
+            'default_actions': default_actions,
+            'raptor_extra_options': self.raptor_args,
+            'python3_manifest': {
+                'win32': 'python3.manifest',
+                'win64': 'python3_x64.manifest',
+            }
+        }
+
+    def make_args(self):
+        self.args = {
+            'config': {},
+            'initial_config_file': self.config_file_path,
+        }
+
+    def write_config(self):
+        try:
+            config_file = open(self.config_file_path, 'wb')
+            config_file.write(json.dumps(self.config))
+            config_file.close()
+        except IOError as e:
+            err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
+            print(err_str.format(self.config_file_path, str(e)))
+            raise e
+
+    def run_mozharness(self):
+        sys.path.insert(0, self.mozharness_dir)
+        from mozharness.mozilla.testing.raptor import Raptor
+        raptor_mh = Raptor(config=self.args['config'],
+                           initial_config_file=self.args['initial_config_file'])
+        return raptor_mh.run()
+
+
+def create_parser():
+    sys.path.insert(0, HERE)  # allow to import the raptor package
+    from raptor.cmdline import create_parser
+    return create_parser(mach_interface=True)
+
+
+@CommandProvider
+class MachRaptor(MachCommandBase):
+    @Command('raptor-test', category='testing',
+             description='Run raptor performance tests.',
+             parser=create_parser)
+    def run_raptor_test(self, **kwargs):
+        raptor = self._spawn(RaptorRunner)
+
+        try:
+            return raptor.run_test(sys.argv[2:])
+        except Exception as e:
+            print(str(e))
+            return 1
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -8,23 +8,27 @@ import os
 
 from mozlog.commandline import add_logging_group
 
 
 def create_parser(mach_interface=False):
     parser = argparse.ArgumentParser()
     add_arg = parser.add_argument
 
-    add_arg('-t', '--test', default=None, dest="test",
+    if not mach_interface:
+        add_arg('--app', default='firefox', dest='app',
+                help="name of the application we are testing (default: firefox)",
+                choices=['firefox', 'chrome'])
+        add_arg('-b', '--binary', required=True, dest='binary',
+                help="path to the browser executable that we are testing")
+
+    # remaining arg is test name
+    add_arg("test",
+            nargs="*",
             help="name of raptor test to run")
-    add_arg('--app', default='firefox', dest='app',
-            help="name of the application we are testing (default: firefox)",
-            choices=['firefox', 'chrome'])
-    add_arg('-b', '--binary', required=True,
-            help="path to the browser executable that we are testing")
 
     add_logging_group(parser)
     return parser
 
 
 def verify_options(parser, args):
     ctx = vars(args)
 
--- a/testing/raptor/raptor/control_server.py
+++ b/testing/raptor/raptor/control_server.py
@@ -1,19 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# simple local server on port 8000, to demonstrate
-# receiving hero element timing results from a web extension
+# control server for raptor performance framework
+# communicates with the raptor browser webextension
 from __future__ import absolute_import
 
 import BaseHTTPServer
 import json
 import os
+import socket
 import threading
 
 from mozlog import get_proxy_logger
 
 LOG = get_proxy_logger(component='control_server')
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -64,29 +65,36 @@ class MyHandler(BaseHTTPServer.BaseHTTPR
 
 class RaptorControlServer():
     """Container class for Raptor Control Server"""
 
     def __init__(self):
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.server = None
         self._server_thread = None
+        self.port = None
 
     def start(self):
         config_dir = os.path.join(here, 'tests')
         os.chdir(config_dir)
-        server_address = ('', 8000)
+
+        # pick a free port
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.bind(('', 0))
+        self.port = sock.getsockname()[1]
+        sock.close()
+        server_address = ('', self.port)
 
         server_class = BaseHTTPServer.HTTPServer
         handler_class = MyHandler
 
         httpd = server_class(server_address, handler_class)
 
         self._server_thread = threading.Thread(target=httpd.serve_forever)
         self._server_thread.setDaemon(True)  # don't hang on exit
         self._server_thread.start()
-        LOG.info("raptor control server running on port 8000...")
+        LOG.info("raptor control server running on port %d..." % self.port)
         self.server = httpd
 
     def stop(self):
         LOG.info("shutting down control server")
         self.server.shutdown()
         self._server_thread.join()
--- a/testing/raptor/raptor/gen_test_config.py
+++ b/testing/raptor/raptor/gen_test_config.py
@@ -8,25 +8,25 @@ import os
 from mozlog import get_proxy_logger
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext', 'raptor')
 LOG = get_proxy_logger(component="gen_test_url")
 
 
-def gen_test_config(browser, test):
+def gen_test_config(browser, test, cs_port):
     LOG.info("writing test settings url background js, so webext can get it")
 
     data = """// this file is auto-generated by raptor, do not edit directly
 function getTestConfig() {
-    return {"browser": "%s", "test_settings_url": "http://localhost:8000/%s.json"};
+    return {"browser": "%s", "test_settings_url": "http://localhost:%d/%s.json"};
 }
 
-""" % (browser, test)
+""" % (browser, cs_port, test)
 
     webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
 
     file = open(webext_background_script, "w")
     file.write(data)
     file.close()
 
     LOG.info("finished writing test config into webext")
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -61,17 +61,18 @@ def write_test_settings_json(test_detail
     except IOError:
         LOG.info("abort: exception writing test settings json!")
 
 
 def get_raptor_test_list(args):
     # get a list of available raptor tests, for the browser we're testing on
     available_tests = get_browser_test_list(args.app)
     tests_to_run = []
-
+    # currently only support one test name on cmd line
+    args.test = args.test[0]
     # if test name not provided on command line, run all available raptor tests for this browser;
     # if test name provided on command line, make sure it exists, and then only include that one
     if args.test is not None:
         for next_test in available_tests:
             if next_test['name'] == args.test:
                 tests_to_run = [next_test]
                 break
         if len(tests_to_run) == 0:
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -11,25 +11,27 @@ import sys
 import time
 
 import mozinfo
 
 from mozlog import commandline, get_default_logger
 from mozprofile import create_profile
 from mozrunner import runners
 
-from raptor.cmdline import parse_args
-from raptor.control_server import RaptorControlServer
-from raptor.gen_test_config import gen_test_config
-from raptor.outputhandler import OutputHandler
-from raptor.playback import get_playback
-from raptor.manifest import get_raptor_test_list
-
+# need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext')
+sys.path.insert(0, here)
+
+from cmdline import parse_args
+from control_server import RaptorControlServer
+from gen_test_config import gen_test_config
+from outputhandler import OutputHandler
+from playback import get_playback
+from manifest import get_raptor_test_list
 
 
 class Raptor(object):
     """Container class for Raptor"""
 
     def __init__(self, app, binary):
         self.config = {}
         self.config['app'] = app
@@ -74,17 +76,17 @@ class Raptor(object):
         self.config['playback_binary_zip'] = test.get(_key, None)
         self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
         _key = 'playback_pageset_zip_%s' % self.config['platform']
         self.config['playback_pageset_zip'] = test.get(_key, None)
         self.config['playback_recordings'] = test.get('playback_recordings', None)
 
     def run_test(self, test, timeout=None):
         self.log.info("starting raptor test: %s" % test['name'])
-        gen_test_config(self.config['app'], test['name'])
+        gen_test_config(self.config['app'], test['name'], self.control_server.port)
 
         self.profile.addons.install(os.path.join(webext_dir, 'raptor'))
 
         # some tests require tools to playback the test pages
         if test.get('playback', None) is not None:
             self.get_playback_config(test)
             # startup the playback tool
             self.playback = get_playback(self.config)
deleted file mode 100644
--- a/testing/raptor/raptor/tests/raptor-chrome-tp7.ini
+++ /dev/null
@@ -1,22 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor tp7 chrome
-
-[DEFAULT]
-apps = chrome
-type =  pageload
-playback = mitmproxy
-release_bin_mac = mitmproxy-2.0.2-osx.tar.gz
-page_cycles = 25
-
-[raptor-chrome-tp7]
-test_url = http://localhost:8081/heroes
-measure =
-  fcp
-  hero
-hero =
-  mugshot
-  title
-  anime
--- a/testing/raptor/raptor/tests/raptor-firefox-tp6.ini
+++ b/testing/raptor/raptor/tests/raptor-firefox-tp6.ini
@@ -1,13 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# raptor tp6 firefox
+# raptor tp6 on firefox
 
 [DEFAULT]
 apps = firefox
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-osx.manifest
 playback_binary_zip_mac = mitmproxy-2.0.2-osx.tar.gz
 playback_pageset_manifest = mitmproxy-playback-set.manifest
deleted file mode 100644
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ /dev/null
@@ -1,14 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor speedometer
-
-[raptor-speedometer]
-apps =
-  firefox
-  chrome
-type = benchmark
-test_url = http://localhost:8081/Speedometer/index.html?raptor
-page_cycles = 1
-page_timeout = 120000
--- a/testing/raptor/test/test_raptor.py
+++ b/testing/raptor/test/test_raptor.py
@@ -5,17 +5,16 @@ import threading
 import time
 
 import mozunit
 import pytest
 
 from mozprofile import BaseProfile
 from mozrunner.errors import RunnerNotStartedError
 
-from raptor.control_server import RaptorControlServer
 from raptor.raptor import Raptor
 
 
 @pytest.mark.parametrize('app', ['firefox', 'chrome'])
 def test_create_profile(options, app, get_prefs):
     options['app'] = app
     raptor = Raptor(**options)
 
@@ -31,22 +30,26 @@ def test_create_profile(options, app, ge
     prefs_file = os.path.join(raptor.profile.profile, 'user.js')
     with open(prefs_file, 'r') as fh:
         prefs = fh.read()
         assert firefox_pref in prefs
         assert raptor_pref in prefs
 
 
 def test_start_and_stop_server(raptor):
+    print("*RW* control server is now:")
+    print(str(raptor.control_server))
     assert raptor.control_server is None
 
     raptor.start_control_server()
-    assert isinstance(raptor.control_server, RaptorControlServer)
 
     assert raptor.control_server._server_thread.is_alive()
+    assert raptor.control_server.port is not None
+    assert raptor.control_server.server is not None
+
     raptor.clean_up()
     assert not raptor.control_server._server_thread.is_alive()
 
 
 @pytest.mark.parametrize('app', [
     'firefox',
     pytest.mark.xfail('chrome'),
 ])