Bug 1455107 - Integrate raptor into mach; r?gbrown draft
authorRob Wood <rwood@mozilla.com>
Mon, 23 Apr 2018 16:43:30 -0400
changeset 789158 141f892ea39fbd3b0ddb41054c73f6a60cc626e7
parent 789073 3e459e1186feff716668d9a8351376aebfc052b1
push id108201
push userrwood@mozilla.com
push dateFri, 27 Apr 2018 17:55:32 +0000
reviewersgbrown
bugs1455107
milestone61.0a1
Bug 1455107 - Integrate raptor into mach; r?gbrown MozReview-Commit-ID: 84vIqU2NWkE
build/mach_bootstrap.py
testing/mozharness/configs/raptor/linux64_config_taskcluster.py
testing/mozharness/configs/raptor/linux_config.py
testing/mozharness/configs/raptor/mac_config.py
testing/mozharness/configs/raptor/windows_config.py
testing/mozharness/configs/raptor/windows_vm_config.py
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/mozharness/scripts/raptor_script.py
testing/raptor/mach_commands.py
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/control_server.py
testing/raptor/raptor/gen_test_config.py
testing/raptor/raptor/manifest.py
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/raptor-chrome-tp7.ini
testing/raptor/raptor/tests/raptor-firefox-tp6.ini
testing/raptor/raptor/tests/raptor-speedometer.ini
testing/raptor/test/test_raptor.py
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -49,16 +49,17 @@ MACH_MODULES = [
     'taskcluster/mach_commands.py',
     'testing/awsy/mach_commands.py',
     'testing/firefox-ui/mach_commands.py',
     'testing/geckodriver/mach_commands.py',
     'testing/mach_commands.py',
     'testing/marionette/mach_commands.py',
     'testing/mochitest/mach_commands.py',
     'testing/mozharness/mach_commands.py',
+    'testing/raptor/mach_commands.py',
     'testing/talos/mach_commands.py',
     'testing/web-platform/mach_commands.py',
     'testing/xpcshell/mach_commands.py',
     'tools/compare-locales/mach_commands.py',
     'tools/docs/mach_commands.py',
     'tools/lint/mach_commands.py',
     'tools/mach_commands.py',
     'tools/power/mach_commands.py',
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/linux64_config_taskcluster.py
@@ -0,0 +1,45 @@
+import os
+import sys
+
+PYTHON = sys.executable
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+
+exes = {
+    'python': PYTHON,
+}
+ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
+INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": INSTALLER_PATH,
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "exes": exes,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+    "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+    "tooltool_cache": "/builds/worker/tooltool-cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/linux_config.py
@@ -0,0 +1,41 @@
+import os
+import platform
+
+VENV_PATH = '%s/build/venv' % os.getcwd()
+if platform.architecture()[0] == '64bit':
+    TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+    MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+else:
+    TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
+    MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "setup-mitmproxy",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+    "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+    "tooltool_cache": "/builds/tooltool_cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/mac_config.py
@@ -0,0 +1,50 @@
+ENABLE_SCREEN_RESOLUTION_CHECK = True
+
+SCREEN_RESOLUTION_CHECK = {
+    "name": "check_screen_resolution",
+    "cmd": ["bash", "-c", "screenresolution get && screenresolution list && system_profiler SPDisplaysDataType"],
+    "architectures": ["32bit", "64bit"],
+    "halt_on_failure": False,
+    "enabled": ENABLE_SCREEN_RESOLUTION_CHECK
+}
+
+import os
+
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "run_cmd_checks_enabled": True,
+    "preflight_run_cmd_suites": [
+        SCREEN_RESOLUTION_CHECK,
+    ],
+    "postflight_run_cmd_suites": [
+        SCREEN_RESOLUTION_CHECK,
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": "macosx64-minidump_stackwalk",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/macosx64/releng.manifest",
+    "tooltool_cache": "/builds/tooltool_cache",
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/windows_config.py
@@ -0,0 +1,56 @@
+import os
+import socket
+import sys
+
+PYTHON = sys.executable
+PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
+VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "pip_index": False,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
+    "exes": {
+        'python': PYTHON,
+        'easy_install': ['%s/scripts/python' % VENV_PATH,
+                         '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
+        'mozinstall': ['%s/scripts/python' % VENV_PATH,
+                       '%s/scripts/mozinstall-script.py' % VENV_PATH],
+        'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+        'tooltool.py': [PYTHON, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
+    },
+    "title": socket.gethostname().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
+    "download_minidump_stackwalk": True,
+    "tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
+    "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+    "python3_manifest": {
+        "win32": "python3.manifest",
+        "win64": "python3_x64.manifest",
+    },
+    "env": {
+        # python3 requires C runtime, found in firefox installation; see bug 1361732
+        "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/raptor/windows_vm_config.py
@@ -0,0 +1,55 @@
+import os
+import socket
+import sys
+
+PYTHON = sys.executable
+PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
+VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
+
+config = {
+    "log_name": "raptor",
+    "buildbot_json_path": "buildprops.json",
+    "installer_path": "installer.exe",
+    "virtualenv_path": VENV_PATH,
+    "pip_index": False,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
+    "exes": {
+        'python': PYTHON,
+        'easy_install': ['%s/scripts/python' % VENV_PATH,
+                         '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
+        'mozinstall': ['%s/scripts/python' % VENV_PATH,
+                       '%s/scripts/mozinstall-script.py' % VENV_PATH],
+        'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+    },
+    "title": socket.gethostname().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "read-buildbot-config",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
+    "download_minidump_stackwalk": True,
+    "tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
+    "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+    "python3_manifest": {
+        "win32": "python3.manifest",
+        "win64": "python3_x64.manifest",
+    },
+    "env": {
+        # python3 requires C runtime, found in firefox installation; see bug 1361732
+        "PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
+    }
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -0,0 +1,401 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import os
+import re
+import sys
+
+import mozharness
+
+from mozharness.base.config import parse_config_file
+from mozharness.base.errors import PythonErrorList
+from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO, WARNING
+from mozharness.base.python import Python3Virtualenv
+from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.mozilla.tooltool import TooltoolMixin
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.testing.codecoverage import (
+    CodeCoverageMixin,
+    code_coverage_config_options
+)
+
+scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
+external_tools_path = os.path.join(scripts_path, 'external_tools')
+
+RaptorErrorList = PythonErrorList + [
+    {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
+    {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
+    {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
+    {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
+    {'substr': r'''raptorError''', 'level': CRITICAL},
+    {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
+    {'substr': r"""No such file or directory: 'browser_output.txt'""",
+     'level': CRITICAL,
+     'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
+]
+
+class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin):
+    """
+    install and run raptor tests
+    """
+    config_options = [
+        [["--test"],
+         {"action": "store",
+          "dest": "test",
+          "help": "Raptor test to run"
+          }],
+        [["--branch-name"],
+         {"action": "store",
+          "dest": "branch",
+          "help": "branch running against"
+          }],
+        [["--add-option"],
+         {"action": "extend",
+          "dest": "raptor_extra_options",
+          "default": None,
+          "help": "extra options to raptor"
+          }],
+    ] + testing_config_options + copy.deepcopy(blobupload_config_options) \
+                               + copy.deepcopy(code_coverage_config_options)
+
+    def __init__(self, **kwargs):
+        kwargs.setdefault('config_options', self.config_options)
+        kwargs.setdefault('all_actions', ['clobber',
+                                          'read-buildbot-config',
+                                          'download-and-extract',
+                                          'populate-webroot',
+                                          'create-virtualenv',
+                                          'install',
+                                          'run-tests',
+                                          ])
+        kwargs.setdefault('default_actions', ['clobber',
+                                              'download-and-extract',
+                                              'populate-webroot',
+                                              'create-virtualenv',
+                                              'install',
+                                              'run-tests',
+                                              ])
+        kwargs.setdefault('config', {})
+        super(Raptor, self).__init__(**kwargs)
+
+        self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
+
+        self.run_local = self.config.get('run_local')
+        self.installer_url = self.config.get("installer_url")
+        self.raptor_json_url = self.config.get("raptor_json_url")
+        self.raptor_json = self.config.get("raptor_json")
+        self.raptor_json_config = self.config.get("raptor_json_config")
+        self.repo_path = self.config.get("repo_path")
+        self.obj_path = self.config.get("obj_path")
+        self.tests = None
+        self.gecko_profile = self.config.get('gecko_profile')
+        self.gecko_profile_interval = self.config.get('gecko_profile_interval')
+        self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary
+        self.mitmproxy_pageset = None # zip file found on tooltool that contains all of the mitmproxy recordings
+        self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set
+        self.mitmdump = None # path to mitmdump tool itself, in py3 venv
+
+    # We accept some configuration options from the try commit message in the format mozharness: <options>
+    # Example try commit message:
+    #   mozharness: --geckoProfile try: <stuff>
+    def query_gecko_profile_options(self):
+        gecko_results = []
+        if self.buildbot_config:
+            # this is inside automation
+            # now let's see if we added GeckoProfile specs in the commit message
+            try:
+                junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:')
+            except IndexError:
+                # when we don't have comments on changes (bug 1255187)
+                opts = None
+
+            if opts:
+                # In the case of a multi-line commit message, only examine
+                # the first line for mozharness options
+                opts = opts.split('\n')[0]
+                opts = re.sub(r'\w+:.*', '', opts).strip().split(' ')
+                if "--geckoProfile" in opts:
+                    # overwrite whatever was set here.
+                    self.gecko_profile = True
+                try:
+                    idx = opts.index('--geckoProfileInterval')
+                    if len(opts) > idx + 1:
+                        self.gecko_profile_interval = opts[idx + 1]
+                except ValueError:
+                    pass
+            else:
+                # no opts, check for '--geckoProfile' in try message text directly
+                if self.try_message_has_flag('geckoProfile'):
+                    self.gecko_profile = True
+
+        # finally, if gecko_profile is set, we add that to the raptor options
+        if self.gecko_profile:
+            gecko_results.append('--geckoProfile')
+            if self.gecko_profile_interval:
+                gecko_results.extend(
+                    ['--geckoProfileInterval', str(self.gecko_profile_interval)]
+                )
+        return gecko_results
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(Raptor, self).query_abs_dirs()
+        abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+        abs_dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def raptor_options(self, args=None, **kw):
+        """return options to raptor"""
+        # binary path
+        binary_path = self.binary_path or self.config.get('binary_path')
+        if not binary_path:
+            self.fatal("Raptor requires a path to the binary.  You can specify binary_path or add download-and-extract to your action list.")
+        # raptor options
+        if binary_path.endswith('.exe'):
+            binary_path = binary_path[:-4]
+        options = []
+        kw_options = {'binary': binary_path}
+        # options overwritten from **kw
+        if 'suite' in self.config:
+            kw_options['suite'] = self.config['suite']
+        if self.config.get('branch'):
+            kw_options['branchName'] = self.config['branch']
+        if self.symbols_path:
+            kw_options['symbolsPath'] = self.symbols_path
+        kw_options.update(kw)
+        # configure profiling options
+        options.extend(self.query_gecko_profile_options())
+        # extra arguments
+        if args is not None:
+            options += args
+        if 'raptor_extra_options' in self.config:
+            options += self.config['raptor_extra_options']
+        if self.config.get('code_coverage', False):
+            options.extend(['--code-coverage'])
+        for key, value in kw_options.items():
+            options.extend(['--%s' % key, value])
+        return options
+
+    def populate_webroot(self):
+        """Populate the production test slaves' webroots"""
+        self.raptor_path = os.path.join(
+            self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
+        )
+
+        if self.config.get('run_local'):
+            # raptor initiated locally, get and verify test from cmd line
+            self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
+            if 'raptor_extra_options' in self.config:
+                if '--test' in self.config['raptor_extra_options']:
+                    # --test specified, get test from cmd line and ensure is valid
+                    test_name_index = self.config['raptor_extra_options'].index('--test') + 1
+                    if test_name_index < len(self.config['raptor_extra_options']):
+                        self.test = self.config['raptor_extra_options'][test_name_index]
+                    else:
+                        self.fatal("Test name not provided")
+        else:
+            # raptor initiated in production via mozharness
+            self.test = self.config['test']
+
+    # Action methods. {{{1
+    # clobber defined in BaseScript
+    # read_buildbot_config defined in BuildbotMixin
+
+    def download_and_extract(self, extract_dirs=None, suite_categories=None):
+        return super(Raptor, self).download_and_extract(
+            suite_categories=['common', 'raptor']
+        )
+
+    def create_virtualenv(self, **kwargs):
+        """VirtualenvMixin.create_virtualenv() assuemes we're using
+        self.config['virtualenv_modules']. Since we are installing
+        raptor from its source, we have to wrap that method here."""
+        # if virtualenv already exists, just add to path and don't re-install, need it
+        # in path so can import jsonschema later when validating output for perfherder
+        _virtualenv_path = self.config.get("virtualenv_path")
+
+        if self.run_local and os.path.exists(_virtualenv_path):
+            self.info("Virtualenv already exists, skipping creation")
+            _python_interp = self.config.get('exes')['python']
+
+            if 'win' in self.platform_name():
+                _path = os.path.join(_virtualenv_path,
+                                     'Lib',
+                                     'site-packages')
+            else:
+                _path = os.path.join(_virtualenv_path,
+                                     'lib',
+                                     os.path.basename(_python_interp),
+                                     'site-packages')
+            sys.path.append(_path)
+            return
+
+        # virtualenv doesn't already exist so create it
+        # install mozbase first, so we use in-tree versions
+        if not self.run_local:
+            mozbase_requirements = os.path.join(
+                self.query_abs_dirs()['abs_test_install_dir'],
+                'config',
+                'mozbase_requirements.txt'
+            )
+        else:
+            mozbase_requirements = os.path.join(
+                os.path.dirname(self.raptor_path),
+                'config',
+                'mozbase_source_requirements.txt'
+            )
+        self.register_virtualenv_module(
+            requirements=[mozbase_requirements],
+            two_pass=True,
+            editable=True,
+        )
+        # require pip >= 1.5 so pip will prefer .whl files to install
+        super(Raptor, self).create_virtualenv(
+            modules=['pip>=1.5']
+        )
+        # raptor in harness requires what else is
+        # listed in raptor requirements.txt file.
+        self.install_module(
+            requirements=[os.path.join(self.raptor_path,
+                                       'requirements.txt')]
+        )
+
+    def _validate_treeherder_data(self, parser):
+        # late import is required, because install is done in create_virtualenv
+        import jsonschema
+
+        if len(parser.found_perf_data) != 1:
+            self.critical("PERFHERDER_DATA was seen %d times, expected 1."
+                          % len(parser.found_perf_data))
+            return
+
+        schema_path = os.path.join(external_tools_path,
+                                   'performance-artifact-schema.json')
+        self.info("Validating PERFHERDER_DATA against %s" % schema_path)
+        try:
+            with open(schema_path) as f:
+                schema = json.load(f)
+            data = json.loads(parser.found_perf_data[0])
+            jsonschema.validate(data, schema)
+        except:
+            self.exception("Error while validating PERFHERDER_DATA")
+
+    def _artifact_perf_data(self, dest):
+        src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'local.json')
+        try:
+            shutil.copyfile(src, dest)
+        except:
+            self.critical("Error copying results %s to upload dir %s" % (src, dest))
+
+    def run_tests(self, args=None, **kw):
+        """run raptor tests"""
+
+        # get raptor options
+        options = self.raptor_options(args=args, **kw)
+
+        # python version check
+        python = self.query_python_path()
+        self.run_command([python, "--version"])
+        parser = RaptorOutputParser(config=self.config, log_obj=self.log_obj,
+                                   error_list=RaptorErrorList)
+        env = {}
+        env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
+        if not self.run_local:
+            env['MINIDUMP_STACKWALK'] = self.query_minidump_stackwalk()
+        env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
+        env['RUST_BACKTRACE'] = 'full'
+        if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
+            self.mkdir_p(env['MOZ_UPLOAD_DIR'])
+        env = self.query_env(partial_env=env, log_level=INFO)
+        # adjust PYTHONPATH to be able to use raptor as a python package
+        if 'PYTHONPATH' in env:
+            env['PYTHONPATH'] = self.raptor_path + os.pathsep + env['PYTHONPATH']
+        else:
+            env['PYTHONPATH'] = self.raptor_path
+
+        # mitmproxy needs path to mozharness when installing the cert
+        env['SCRIPTSPATH'] = scripts_path
+
+        if self.repo_path is not None:
+            env['MOZ_DEVELOPER_REPO_DIR'] = self.repo_path
+        if self.obj_path is not None:
+            env['MOZ_DEVELOPER_OBJ_DIR'] = self.obj_path
+
+        # sets a timeout for how long raptor should run without output
+        output_timeout = self.config.get('raptor_output_timeout', 3600)
+        # run raptor tests
+        run_tests = os.path.join(self.raptor_path, 'raptor', 'raptor.py')
+
+        mozlog_opts = ['--log-tbpl-level=debug']
+        if not self.run_local and 'suite' in self.config:
+            fname_pattern = '%s_%%s.log' % self.config['test']
+            mozlog_opts.append('--log-errorsummary=%s'
+                               % os.path.join(env['MOZ_UPLOAD_DIR'],
+                                              fname_pattern % 'errorsummary'))
+            mozlog_opts.append('--log-raw=%s'
+                               % os.path.join(env['MOZ_UPLOAD_DIR'],
+                                              fname_pattern % 'raw'))
+
+        def launch_in_debug_mode(cmdline):
+            cmdline = set(cmdline)
+            debug_opts = {'--debug', '--debugger', '--debugger_args'}
+
+            return bool(debug_opts.intersection(cmdline))
+
+        command = [python, run_tests] + options + mozlog_opts
+        if launch_in_debug_mode(command):
+            raptor_process = subprocess.Popen(command, cwd=self.workdir, env=env)
+            raptor_process.wait()
+        else:
+            self.return_code = self.run_command(command, cwd=self.workdir,
+                                            output_timeout=output_timeout,
+                                            output_parser=parser,
+                                            env=env)
+        if parser.minidump_output:
+            self.info("Looking at the minidump files for debugging purposes...")
+            for item in parser.minidump_output:
+                self.run_command(["ls", "-l", item])
+
+        if self.return_code not in [0]:
+            # update the worst log level
+            log_level = ERROR
+            if self.return_code == 1:
+                log_level = WARNING
+            if self.return_code == 4:
+                log_level = WARNING
+
+        elif '--no-upload-results' not in options:
+            if not self.gecko_profile:
+                self._validate_treeherder_data(parser)
+                if not self.run_local:
+                    # copy results to upload dir so they are included as an artifact
+                    dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
+                    self._artifact_perf_data(dest)
+
+
+class RaptorOutputParser(OutputParser):
+    minidump_regex = re.compile(r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''')
+    RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
+
+    def __init__(self, **kwargs):
+        super(RaptorOutputParser, self).__init__(**kwargs)
+        self.minidump_output = None
+        self.found_perf_data = []
+
+    def parse_single_line(self, line):
+        m = self.minidump_regex.search(line)
+        if m:
+            self.minidump_output = (m.group(1), m.group(2), m.group(3))
+
+        m = self.RE_PERF_DATA.match(line)
+        if m:
+            self.found_perf_data.append(m.group(1))
+        super(RaptorOutputParser, self).parse_single_line(line)
+
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/raptor_script.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""raptor
+
+"""
+
+import os
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.mozilla.testing.raptor import Raptor
+
+if __name__ == '__main__':
+    raptor = Raptor()
+    raptor.run_and_exit()
new file mode 100644
--- /dev/null
+++ b/testing/raptor/mach_commands.py
@@ -0,0 +1,118 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Originally taken from /talos/mach_commands.py
+
+# Integrates raptor mozharness with mach
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import sys
+import json
+import socket
+
+from mozbuild.base import MozbuildObject, MachCommandBase
+from mach.decorators import CommandProvider, Command
+
+HERE = os.path.dirname(os.path.realpath(__file__))
+
+
+class RaptorRunner(MozbuildObject):
+    def run_test(self, raptor_args):
+        """
+        We want to do couple of things before running raptor
+        1. Clone mozharness
+        2. Make config for raptor mozharness
+        3. Run mozharness
+        """
+
+        self.init_variables(raptor_args)
+        self.make_config()
+        self.write_config()
+        self.make_args()
+        return self.run_mozharness()
+
+    def init_variables(self, raptor_args):
+        self.raptor_dir = os.path.join(self.topsrcdir, 'testing', 'raptor')
+        self.mozharness_dir = os.path.join(self.topsrcdir, 'testing',
+                                           'mozharness')
+        self.config_file_path = os.path.join(self._topobjdir, 'testing',
+                                             'raptor-in_tree_conf.json')
+        self.binary_path = self.get_binary_path()
+        self.virtualenv_script = os.path.join(self.topsrcdir, 'third_party', 'python',
+                                              'virtualenv', 'virtualenv.py')
+        self.virtualenv_path = os.path.join(self._topobjdir, 'testing',
+                                            'raptor-venv')
+        self.python_interp = sys.executable
+        self.raptor_args = raptor_args
+
+    def make_config(self):
+        default_actions = ['populate-webroot', 'create-virtualenv', 'run-tests']
+        self.config = {
+            'run_local': True,
+            'binary_path': self.binary_path,
+            'repo_path': self.topsrcdir,
+            'raptor_path': self.raptor_dir,
+            'obj_path': self.topobjdir,
+            'log_name': 'raptor',
+            'virtualenv_path': self.virtualenv_path,
+            'pypi_url': 'http://pypi.python.org/simple',
+            'base_work_dir': self.mozharness_dir,
+            'exes': {
+                'python': self.python_interp,
+                'virtualenv': [self.python_interp, self.virtualenv_script],
+            },
+            'title': socket.gethostname(),
+            'default_actions': default_actions,
+            'raptor_extra_options': self.raptor_args,
+            'python3_manifest': {
+                'win32': 'python3.manifest',
+                'win64': 'python3_x64.manifest',
+            }
+        }
+
+    def make_args(self):
+        self.args = {
+            'config': {},
+            'initial_config_file': self.config_file_path,
+        }
+
+    def write_config(self):
+        try:
+            config_file = open(self.config_file_path, 'wb')
+            config_file.write(json.dumps(self.config))
+            config_file.close()
+        except IOError as e:
+            err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
+            print(err_str.format(self.config_file_path, str(e)))
+            raise e
+
+    def run_mozharness(self):
+        sys.path.insert(0, self.mozharness_dir)
+        from mozharness.mozilla.testing.raptor import Raptor
+        raptor_mh = Raptor(config=self.args['config'],
+                           initial_config_file=self.args['initial_config_file'])
+        return raptor_mh.run()
+
+
+def create_parser():
+    sys.path.insert(0, HERE)  # allow to import the raptor package
+    from raptor.cmdline import create_parser
+    return create_parser(mach_interface=True)
+
+
+@CommandProvider
+class MachRaptor(MachCommandBase):
+    @Command('raptor-test', category='testing',
+             description='Run raptor performance tests.',
+             parser=create_parser)
+    def run_raptor_test(self, **kwargs):
+        raptor = self._spawn(RaptorRunner)
+
+        try:
+            return raptor.run_test(sys.argv[2:])
+        except Exception as e:
+            print(str(e))
+            return 1
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -8,23 +8,27 @@ import os
 
 from mozlog.commandline import add_logging_group
 
 
 def create_parser(mach_interface=False):
     parser = argparse.ArgumentParser()
     add_arg = parser.add_argument
 
-    add_arg('-t', '--test', default=None, dest="test",
+    if not mach_interface:
+        add_arg('--app', default='firefox', dest='app',
+                help="name of the application we are testing (default: firefox)",
+                choices=['firefox', 'chrome'])
+        add_arg('-b', '--binary', required=True, dest='binary',
+                help="path to the browser executable that we are testing")
+
+    # remaining arg is test name
+    add_arg("test",
+            nargs="*",
             help="name of raptor test to run")
-    add_arg('--app', default='firefox', dest='app',
-            help="name of the application we are testing (default: firefox)",
-            choices=['firefox', 'chrome'])
-    add_arg('-b', '--binary', required=True,
-            help="path to the browser executable that we are testing")
 
     add_logging_group(parser)
     return parser
 
 
 def verify_options(parser, args):
     ctx = vars(args)
 
--- a/testing/raptor/raptor/control_server.py
+++ b/testing/raptor/raptor/control_server.py
@@ -1,19 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# simple local server on port 8000, to demonstrate
-# receiving hero element timing results from a web extension
+# control server for raptor performance framework
+# communicates with the raptor browser webextension
 from __future__ import absolute_import
 
 import BaseHTTPServer
 import json
 import os
+import socket
 import threading
 
 from mozlog import get_proxy_logger
 
 LOG = get_proxy_logger(component='control_server')
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -64,29 +65,36 @@ class MyHandler(BaseHTTPServer.BaseHTTPR
 
 class RaptorControlServer():
     """Container class for Raptor Control Server"""
 
     def __init__(self):
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.server = None
         self._server_thread = None
+        self.port = None
 
     def start(self):
         config_dir = os.path.join(here, 'tests')
         os.chdir(config_dir)
-        server_address = ('', 8000)
+
+        # pick a free port
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.bind(('', 0))
+        self.port = sock.getsockname()[1]
+        sock.close()
+        server_address = ('', self.port)
 
         server_class = BaseHTTPServer.HTTPServer
         handler_class = MyHandler
 
         httpd = server_class(server_address, handler_class)
 
         self._server_thread = threading.Thread(target=httpd.serve_forever)
         self._server_thread.setDaemon(True)  # don't hang on exit
         self._server_thread.start()
-        LOG.info("raptor control server running on port 8000...")
+        LOG.info("raptor control server running on port %d..." % self.port)
         self.server = httpd
 
     def stop(self):
         LOG.info("shutting down control server")
         self.server.shutdown()
         self._server_thread.join()
--- a/testing/raptor/raptor/gen_test_config.py
+++ b/testing/raptor/raptor/gen_test_config.py
@@ -8,25 +8,25 @@ import os
 from mozlog import get_proxy_logger
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext', 'raptor')
 LOG = get_proxy_logger(component="gen_test_url")
 
 
-def gen_test_config(browser, test):
+def gen_test_config(browser, test, cs_port):
     LOG.info("writing test settings url background js, so webext can get it")
 
     data = """// this file is auto-generated by raptor, do not edit directly
 function getTestConfig() {
-    return {"browser": "%s", "test_settings_url": "http://localhost:8000/%s.json"};
+    return {"browser": "%s", "test_settings_url": "http://localhost:%d/%s.json"};
 }
 
-""" % (browser, test)
+""" % (browser, cs_port, test)
 
     webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
 
     file = open(webext_background_script, "w")
     file.write(data)
     file.close()
 
     LOG.info("finished writing test config into webext")
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -61,17 +61,18 @@ def write_test_settings_json(test_detail
     except IOError:
         LOG.info("abort: exception writing test settings json!")
 
 
 def get_raptor_test_list(args):
     # get a list of available raptor tests, for the browser we're testing on
     available_tests = get_browser_test_list(args.app)
     tests_to_run = []
-
+    # currently only support one test name on cmd line
+    args.test = args.test[0]
     # if test name not provided on command line, run all available raptor tests for this browser;
     # if test name provided on command line, make sure it exists, and then only include that one
     if args.test is not None:
         for next_test in available_tests:
             if next_test['name'] == args.test:
                 tests_to_run = [next_test]
                 break
         if len(tests_to_run) == 0:
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -11,25 +11,27 @@ import sys
 import time
 
 import mozinfo
 
 from mozlog import commandline, get_default_logger
 from mozprofile import create_profile
 from mozrunner import runners
 
-from raptor.cmdline import parse_args
-from raptor.control_server import RaptorControlServer
-from raptor.gen_test_config import gen_test_config
-from raptor.outputhandler import OutputHandler
-from raptor.playback import get_playback
-from raptor.manifest import get_raptor_test_list
-
+# need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext')
+sys.path.insert(0, here)
+
+from cmdline import parse_args
+from control_server import RaptorControlServer
+from gen_test_config import gen_test_config
+from outputhandler import OutputHandler
+from playback import get_playback
+from manifest import get_raptor_test_list
 
 
 class Raptor(object):
     """Container class for Raptor"""
 
     def __init__(self, app, binary):
         self.config = {}
         self.config['app'] = app
@@ -74,17 +76,17 @@ class Raptor(object):
         self.config['playback_binary_zip'] = test.get(_key, None)
         self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
         _key = 'playback_pageset_zip_%s' % self.config['platform']
         self.config['playback_pageset_zip'] = test.get(_key, None)
         self.config['playback_recordings'] = test.get('playback_recordings', None)
 
     def run_test(self, test, timeout=None):
         self.log.info("starting raptor test: %s" % test['name'])
-        gen_test_config(self.config['app'], test['name'])
+        gen_test_config(self.config['app'], test['name'], self.control_server.port)
 
         self.profile.addons.install(os.path.join(webext_dir, 'raptor'))
 
         # some tests require tools to playback the test pages
         if test.get('playback', None) is not None:
             self.get_playback_config(test)
             # startup the playback tool
             self.playback = get_playback(self.config)
deleted file mode 100644
--- a/testing/raptor/raptor/tests/raptor-chrome-tp7.ini
+++ /dev/null
@@ -1,22 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor tp7 chrome
-
-[DEFAULT]
-apps = chrome
-type =  pageload
-playback = mitmproxy
-release_bin_mac = mitmproxy-2.0.2-osx.tar.gz
-page_cycles = 25
-
-[raptor-chrome-tp7]
-test_url = http://localhost:8081/heroes
-measure =
-  fcp
-  hero
-hero =
-  mugshot
-  title
-  anime
--- a/testing/raptor/raptor/tests/raptor-firefox-tp6.ini
+++ b/testing/raptor/raptor/tests/raptor-firefox-tp6.ini
@@ -1,13 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# raptor tp6 firefox
+# raptor tp6 on firefox
 
 [DEFAULT]
 apps = firefox
 type =  pageload
 playback = mitmproxy
 playback_binary_manifest = mitmproxy-rel-bin-osx.manifest
 playback_binary_zip_mac = mitmproxy-2.0.2-osx.tar.gz
 playback_pageset_manifest = mitmproxy-playback-set.manifest
deleted file mode 100644
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ /dev/null
@@ -1,14 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor speedometer
-
-[raptor-speedometer]
-apps =
-  firefox
-  chrome
-type = benchmark
-test_url = http://localhost:8081/Speedometer/index.html?raptor
-page_cycles = 1
-page_timeout = 120000
--- a/testing/raptor/test/test_raptor.py
+++ b/testing/raptor/test/test_raptor.py
@@ -31,22 +31,26 @@ def test_create_profile(options, app, ge
     prefs_file = os.path.join(raptor.profile.profile, 'user.js')
     with open(prefs_file, 'r') as fh:
         prefs = fh.read()
         assert firefox_pref in prefs
         assert raptor_pref in prefs
 
 
 def test_start_and_stop_server(raptor):
+    print("*RW* control server is now:")
+    print(str(raptor.control_server))
     assert raptor.control_server is None
 
     raptor.start_control_server()
-    assert isinstance(raptor.control_server, RaptorControlServer)
 
     assert raptor.control_server._server_thread.is_alive()
+    assert raptor.control_server.port is not None
+    assert raptor.control_server.server is not None
+
     raptor.clean_up()
     assert not raptor.control_server._server_thread.is_alive()
 
 
 @pytest.mark.parametrize('app', [
     'firefox',
     pytest.mark.xfail('chrome'),
 ])