Bug 1191324 - Extend Marionette to allow automation of telemetry tests - changes; r=automatedtester
authorAndré Reinald <areinald@mozilla.com>
Tue, 31 May 2016 19:15:37 +0200
changeset 339074 e876e86ee17d8a1059614a312ebd884e55045e9d
parent 339073 9aca0e22adb5bde8dd4d4ef80d34e03d2f4431c6
child 339075 beb2dbb917ea5cd6e7fba72e832fa5e3eb7597c9
push id6249
push userjlund@mozilla.com
push dateMon, 01 Aug 2016 13:59:36 +0000
treeherdermozilla-beta@bad9d4f5bf7e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersautomatedtester
bugs1191324
milestone49.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1191324 - Extend Marionette to allow automation of telemetry tests - changes; r=automatedtester MozReview-Commit-ID: oDGtQ2Vnq4 Marionette harness has undergone lots of changes, so I decided to take current sources from M-C and do the copy + changes patches again with my own changes to get a Session harness.
testing/marionette/harness/marionette/runtests.py
testing/marionette/harness/session/__init__.py
testing/marionette/harness/session/runner/__init__.py
testing/marionette/harness/session/runner/base.py
testing/marionette/harness/session/runtests.py
testing/marionette/harness/session/session_test.py
testing/marionette/harness/session/tests/test_session.py
testing/marionette/harness/session/tests/unit-tests.ini
testing/marionette/mach_commands.py
--- a/testing/marionette/harness/marionette/runtests.py
+++ b/testing/marionette/harness/marionette/runtests.py
@@ -26,19 +26,21 @@ class MarionetteArguments(BaseMarionette
         BaseMarionetteArguments.__init__(self, **kwargs)
         self.register_argument_container(BrowserMobProxyArguments())
 
 
 class MarionetteHarness(object):
     def __init__(self,
                  runner_class=MarionetteTestRunner,
                  parser_class=MarionetteArguments,
+                 testcase_class=MarionetteTestCase,
                  args=None):
         self._runner_class = runner_class
         self._parser_class = parser_class
+        self._testcase_class = testcase_class
         self.args = args or self.parse_args()
 
     def parse_args(self, logger_defaults=None):
         parser = self._parser_class(usage='%(prog)s [options] test_file_or_dir <test_file_or_dir> ...')
         parser.add_argument('--version', action='version',
             help="Show version information.",
             version="%(prog)s {version}"
                     " (using marionette-driver: {driver_version}, ".format(
@@ -52,17 +54,17 @@ class MarionetteHarness(object):
         logger = mozlog.commandline.setup_logging(
             args.logger_name, args, logger_defaults or {"tbpl": sys.stdout})
 
         args.logger = logger
         return vars(args)
 
     def process_args(self):
         if self.args.get('pydebugger'):
-            MarionetteTestCase.pydebugger = __import__(self.args['pydebugger'])
+            self._testcase_class.pydebugger = __import__(self.args['pydebugger'])
 
     def run(self):
         try:
             self.process_args()
             tests = self.args.pop('tests')
             runner = self._runner_class(**self.args)
             runner.run_tests(tests)
             return runner.failed + runner.crashed
@@ -70,28 +72,28 @@ class MarionetteHarness(object):
             logger = self.args.get('logger')
             if logger:
                 logger.error('Failure during test execution.',
                                        exc_info=True)
             raise
 
 
 def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
-        harness_class=MarionetteHarness, args=None):
+        harness_class=MarionetteHarness, testcase_class=MarionetteTestCase, args=None):
     """
     Call the harness to parse args and run tests.
 
     The following exit codes are expected:
     - Test failures: 10
     - Harness/other failures: 1
     - Success: 0
     """
     logger = mozlog.commandline.setup_logging('Marionette test runner', {})
     try:
-        failed = harness_class(runner_class, parser_class, args=args).run()
+        failed = harness_class(runner_class, parser_class, testcase_class, args=args).run()
         if failed > 0:
             sys.exit(10)
     except Exception:
         logger.error('Failure during harness setup', exc_info=True)
         sys.exit(1)
     sys.exit(0)
 
 if __name__ == "__main__":
--- a/testing/marionette/harness/session/__init__.py
+++ b/testing/marionette/harness/session/__init__.py
@@ -1,34 +1,30 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = '3.0.0'
-
-from .marionette_test import (
-    CommonTestCase,
+from marionette.marionette_test import (
     expectedFailure,
-    MarionetteJSTestCase,
-    MarionetteTestCase,
     skip,
     skip_if_desktop,
     SkipTest,
     skip_unless_protocol,
 )
-from .runner import (
-    BaseMarionetteArguments,
-    BaseMarionetteTestRunner,
-    BrowserMobProxyTestCaseMixin,
-    EnduranceArguments,
-    EnduranceTestCaseMixin,
-    HTMLReportingArguments,
-    HTMLReportingTestResultMixin,
-    HTMLReportingTestRunnerMixin,
-    Marionette,
-    MarionetteTest,
-    MarionetteTestResult,
-    MarionetteTextTestRunner,
-    MemoryEnduranceTestCaseMixin,
+
+from marionette.runner import (
     TestManifest,
     TestResult,
     TestResultCollection,
 )
+
+from .session_test import (
+    SessionJSTestCase,
+    SessionTestCase,
+)
+
+from .runner import (
+    BaseSessionArguments,
+    BaseSessionTestRunner,
+    SessionTest,
+    SessionTestResult,
+    SessionTextTestRunner,
+)
--- a/testing/marionette/harness/session/runner/__init__.py
+++ b/testing/marionette/harness/session/runner/__init__.py
@@ -1,27 +1,11 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from .base import (
-    BaseMarionetteArguments,
-    BaseMarionetteTestRunner,
-    Marionette,
-    MarionetteTest,
-    MarionetteTestResult,
-    MarionetteTextTestRunner,
-    TestManifest,
-    TestResult,
-    TestResultCollection,
+    BaseSessionArguments,
+    BaseSessionTestRunner,
+    SessionTest,
+    SessionTestResult,
+    SessionTextTestRunner,
 )
-
-from .mixins import (
-    EnduranceArguments,
-    EnduranceTestCaseMixin,
-    HTMLReportingArguments,
-    HTMLReportingTestResultMixin,
-    HTMLReportingTestRunnerMixin,
-    MemoryEnduranceTestCaseMixin,
-    BrowserMobProxyTestCaseMixin,
-    BrowserMobProxyArguments,
-    BrowserMobTestCase,
-)
--- a/testing/marionette/harness/session/runner/base.py
+++ b/testing/marionette/harness/session/runner/base.py
@@ -21,53 +21,39 @@ import mozprofile
 from manifestparser import TestManifest
 from manifestparser.filters import tags
 from marionette_driver.marionette import Marionette
 from mozlog import get_default_logger
 from moztest.adapters.unit import StructuredTestRunner, StructuredTestResult
 from moztest.results import TestResultCollection, TestResult, relevant_line
 import mozversion
 
-import httpd
+from marionette.runner import httpd
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-def update_mozinfo(path=None):
-    """walk up directories to find mozinfo.json and update the info"""
 
-    path = path or here
-    dirs = set()
-    while path != os.path.expanduser('~'):
-        if path in dirs:
-            break
-        dirs.add(path)
-        path = os.path.split(path)[0]
-
-    return mozinfo.find_and_update_from_json(*dirs)
-
-
-class MarionetteTest(TestResult):
+class SessionTest(TestResult):
 
     @property
     def test_name(self):
         if self.test_class is not None:
             return '%s.py %s.%s' % (self.test_class.split('.')[0],
                                     self.test_class,
                                     self.name)
         else:
             return self.name
 
-class MarionetteTestResult(StructuredTestResult, TestResultCollection):
+class SessionTestResult(StructuredTestResult, TestResultCollection):
 
-    resultClass = MarionetteTest
+    resultClass = SessionTest
 
     def __init__(self, *args, **kwargs):
-        self.marionette = kwargs.pop('marionette')
-        TestResultCollection.__init__(self, 'MarionetteTest')
+        TestResultCollection.__init__(self, 'SessionTest')
         self.passed = 0
         self.testsRun = 0
         self.result_modifiers = [] # used by mixins to modify the result
         StructuredTestResult.__init__(self, *args, **kwargs)
 
     @property
     def skipped(self):
         return [t for t in self if t.result == 'SKIPPED']
@@ -139,41 +125,41 @@ class MarionetteTestResult(StructuredTes
         t.finish(result_actual,
                  time_end=time.time() if test.start_time else 0,
                  reason=relevant_line(output),
                  output=output)
         self.append(t)
 
     def addError(self, test, err):
         self.add_test_result(test, output=self._exc_info_to_string(err, test), result_actual='ERROR')
-        super(MarionetteTestResult, self).addError(test, err)
+        super(SessionTestResult, self).addError(test, err)
 
     def addFailure(self, test, err):
         self.add_test_result(test, output=self._exc_info_to_string(err, test), result_actual='UNEXPECTED-FAIL')
-        super(MarionetteTestResult, self).addFailure(test, err)
+        super(SessionTestResult, self).addFailure(test, err)
 
     def addSuccess(self, test):
         self.passed += 1
         self.add_test_result(test, result_actual='PASS')
-        super(MarionetteTestResult, self).addSuccess(test)
+        super(SessionTestResult, self).addSuccess(test)
 
     def addExpectedFailure(self, test, err):
         """Called when an expected failure/error occured."""
         self.add_test_result(test, output=self._exc_info_to_string(err, test),
                              result_actual='KNOWN-FAIL')
-        super(MarionetteTestResult, self).addExpectedFailure(test, err)
+        super(SessionTestResult, self).addExpectedFailure(test, err)
 
     def addUnexpectedSuccess(self, test):
         """Called when a test was expected to fail, but succeed."""
         self.add_test_result(test, result_actual='UNEXPECTED-PASS')
-        super(MarionetteTestResult, self).addUnexpectedSuccess(test)
+        super(SessionTestResult, self).addUnexpectedSuccess(test)
 
     def addSkip(self, test, reason):
         self.add_test_result(test, output=reason, result_actual='SKIPPED')
-        super(MarionetteTestResult, self).addSkip(test, reason)
+        super(SessionTestResult, self).addSkip(test, reason)
 
     def getInfo(self, test):
         return test.test_name
 
     def getDescription(self, test):
         doc_first_line = test.shortDescription()
         if self.descriptions and doc_first_line:
             return '\n'.join((str(test), doc_first_line))
@@ -198,50 +184,40 @@ class MarionetteTestResult(StructuredTes
                     return
                 self.logger.info('START LOG:')
                 for line in testcase.loglines:
                     self.logger.info(' '.join(line).encode('ascii', 'replace'))
                 self.logger.info('END LOG:')
 
     def stopTest(self, *args, **kwargs):
         unittest._TextTestResult.stopTest(self, *args, **kwargs)
-        if self.marionette.check_for_crash():
-            # this tells unittest.TestSuite not to continue running tests
-            self.shouldStop = True
-            test = next((a for a in args if isinstance(a, unittest.TestCase)),
-                        None)
-            if test:
-                self.addError(test, sys.exc_info())
 
 
-class MarionetteTextTestRunner(StructuredTestRunner):
+class SessionTextTestRunner(StructuredTestRunner):
 
-    resultclass = MarionetteTestResult
+    resultclass = SessionTestResult
 
     def __init__(self, **kwargs):
-        self.marionette = kwargs.pop('marionette')
-        self.capabilities = kwargs.pop('capabilities')
-
+        self.binary = kwargs.pop('binary')
         StructuredTestRunner.__init__(self, **kwargs)
 
     def _makeResult(self):
         return self.resultclass(self.stream,
                                 self.descriptions,
                                 self.verbosity,
-                                marionette=self.marionette,
                                 logger=self.logger,
                                 result_callbacks=self.result_callbacks)
 
     def run(self, test):
-        result = super(MarionetteTextTestRunner, self).run(test)
+        result = super(SessionTextTestRunner, self).run(test)
         result.printLogs(test)
         return result
 
 
-class BaseMarionetteArguments(ArgumentParser):
+class BaseSessionArguments(ArgumentParser):
     socket_timeout_default = 360.0
 
     def __init__(self, **kwargs):
         ArgumentParser.__init__(self, **kwargs)
 
         def dir_path(path):
             path = os.path.abspath(os.path.expanduser(path))
             if not os.access(path, os.F_OK):
@@ -252,21 +228,16 @@ class BaseMarionetteArguments(ArgumentPa
         self.add_argument('tests',
                           nargs='*',
                           default=[],
                           help='Tests to run.')
         self.add_argument('-v', '--verbose',
                         action='count',
                         help='Increase verbosity to include debug messages with -v, '
                             'and trace messages with -vv.')
-        self.add_argument('--address',
-                        help='host:port of running Gecko instance to connect to')
-        self.add_argument('--device',
-                        dest='device_serial',
-                        help='serial ID of a device to use for adb / fastboot')
         self.add_argument('--app',
                         help='application to use')
         self.add_argument('--app-arg',
                         dest='app_args',
                         action='append',
                         default=[],
                         help='specify a command line argument to be passed onto the application')
         self.add_argument('--binary',
@@ -314,18 +285,16 @@ class BaseMarionetteArguments(ArgumentPa
                         default=random.randint(0, sys.maxint),
                         help='Use given seed to shuffle tests')
         self.add_argument('--total-chunks',
                         type=int,
                         help='how many chunks to split the tests up into')
         self.add_argument('--this-chunk',
                         type=int,
                         help='which chunk to run')
-        self.add_argument('--sources',
-                        help='path to sources.xml (Firefox OS only)')
         self.add_argument('--server-root',
                         help='url to a webserver or path to a document root from which content '
                         'resources are served (default: {}).'.format(os.path.join(
                             os.path.dirname(here), 'www')))
         self.add_argument('--gecko-log',
                         help="Define the path to store log file. If the path is"
                              " a directory, the real log file will be created"
                              " given the format gecko-(timestamp).log. If it is"
@@ -407,18 +376,18 @@ class BaseMarionetteArguments(ArgumentPa
             print 'must specify one or more test files, manifests, or directories'
             sys.exit(1)
 
         for path in args.tests:
             if not os.path.exists(path):
                 print '{0} does not exist'.format(path)
                 sys.exit(1)
 
-        if not args.address and not args.binary:
-            print 'must specify --binary, or --address'
+        if not args.binary:
+            print 'must specify --binary'
             sys.exit(1)
 
         if args.total_chunks is not None and args.this_chunk is None:
             self.error('You must specify which chunk to run.')
 
         if args.this_chunk is not None and args.total_chunks is None:
             self.error('You must specify how many chunks to split the tests into.')
 
@@ -442,50 +411,47 @@ class BaseMarionetteArguments(ArgumentPa
 
         for container in self.argument_containers:
             if hasattr(container, 'verify_usage_handler'):
                 container.verify_usage_handler(args)
 
         return args
 
 
-class BaseMarionetteTestRunner(object):
+class BaseSessionTestRunner(object):
 
-    textrunnerclass = MarionetteTextTestRunner
+    textrunnerclass = SessionTextTestRunner
     driverclass = Marionette
 
     def __init__(self, address=None,
                  app=None, app_args=None, binary=None, profile=None,
                  logger=None, logdir=None,
                  repeat=0, testvars=None,
                  symbols_path=None, timeout=None,
                  shuffle=False, shuffle_seed=random.randint(0, sys.maxint),
                  sdcard=None, this_chunk=1, total_chunks=1, sources=None,
                  server_root=None, gecko_log=None, result_callbacks=None,
                  prefs=None, test_tags=None,
-                 socket_timeout=BaseMarionetteArguments.socket_timeout_default,
+                 socket_timeout=BaseSessionArguments.socket_timeout_default,
                  startup_timeout=None, addons=None, workspace=None,
                  verbose=0, e10s=True, **kwargs):
         self.address = address
         self.app = app
         self.app_args = app_args or []
         self.bin = binary
         self.profile = profile
         self.addons = addons
         self.logger = logger
         self.httpd = None
-        self.marionette = None
         self.logdir = logdir
         self.repeat = repeat
         self.test_kwargs = kwargs
         self.symbols_path = symbols_path
         self.timeout = timeout
         self.socket_timeout = socket_timeout
-        self._device = None
-        self._capabilities = None
         self._appinfo = None
         self._appName = None
         self.shuffle = shuffle
         self.shuffle_seed = shuffle_seed
         self.sdcard = sdcard
         self.sources = sources
         self.server_root = server_root
         self.this_chunk = this_chunk
@@ -501,17 +467,17 @@ class BaseMarionetteTestRunner(object):
         # If no workspace is set, default location for gecko.log is .
         # and default location for profile is TMP
         self.workspace_path = workspace or os.getcwd()
         self.verbose = verbose
         self.e10s = e10s
 
         def gather_debug(test, status):
             rv = {}
-            marionette = test._marionette_weakref()
+            marionette = test.marionette
 
             # In the event we're gathering debug without starting a session, skip marionette commands
             if marionette.session is not None:
                 try:
                     with marionette.using_context(marionette.CONTEXT_CHROME):
                         rv['screenshot'] = marionette.screenshot()
                     with marionette.using_context(marionette.CONTEXT_CONTENT):
                         rv['source'] = marionette.page_source
@@ -574,236 +540,72 @@ class BaseMarionetteTestRunner(object):
                         data.append(json.loads(f.read()))
                 except ValueError as e:
                     raise Exception("JSON file (%s) is not properly "
                                     "formatted: %s" % (os.path.abspath(path),
                                                        e.message))
         return data
 
     @property
-    def capabilities(self):
-        if self._capabilities:
-            return self._capabilities
-
-        self.marionette.start_session()
-        self._capabilities = self.marionette.session_capabilities
-        self.marionette.delete_session()
-        return self._capabilities
-
-    @property
-    def appinfo(self):
-        if self._appinfo:
-            return self._appinfo
-
-        self.marionette.start_session()
-        with self.marionette.using_context('chrome'):
-            self._appinfo = self.marionette.execute_script("""
-            try {
-              return Services.appinfo;
-            } catch (e) {
-              return null;
-            }""")
-        self.marionette.delete_session()
-        self._appinfo = self._appinfo or {}
-        return self._appinfo
-
-    @property
-    def device(self):
-        if self._device:
-            return self._device
-
-        self._device = self.capabilities.get('device')
-        return self._device
-
-    @property
-    def appName(self):
-        if self._appName:
-            return self._appName
-
-        self._appName = self.capabilities.get('browserName')
-        return self._appName
-
-    @property
     def bin(self):
         return self._bin
 
     @bin.setter
     def bin(self, path):
         """
         Set binary and reset parts of runner accordingly
 
         Intended use: to change binary between calls to run_tests
         """
         self._bin = path
         self.tests = []
-        if hasattr(self, 'marionette') and self.marionette:
-            self.marionette.cleanup()
-            if self.marionette.instance:
-                self.marionette.instance = None
-        self.marionette = None
 
     def reset_test_stats(self):
         self.passed = 0
         self.failed = 0
         self.crashed = 0
         self.unexpected_successes = 0
         self.todo = 0
         self.skipped = 0
         self.failures = []
 
-    def _build_kwargs(self):
-        if self.logdir and not os.access(self.logdir, os.F_OK):
-            os.mkdir(self.logdir)
-
-        kwargs = {
-            'timeout': self.timeout,
-            'socket_timeout': self.socket_timeout,
-            'prefs': self.prefs,
-            'startup_timeout': self.startup_timeout,
-            'verbose': self.verbose,
-        }
-        if self.bin:
-            kwargs.update({
-                'host': 'localhost',
-                'port': 2828,
-                'app': self.app,
-                'app_args': self.app_args,
-                'bin': self.bin,
-                'profile': self.profile,
-                'addons': self.addons,
-                'gecko_log': self.gecko_log,
-            })
-
-        if self.address:
-            host, port = self.address.split(':')
-            kwargs.update({
-                'host': host,
-                'port': int(port),
-            })
-
-            if not self.bin:
-                try:
-                    #establish a socket connection so we can vertify the data come back
-                    connection = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
-                    connection.connect((host,int(port)))
-                    connection.close()
-                except Exception, e:
-                    raise Exception("Connection attempt to %s:%s failed with error: %s" %(host,port,e))
-        if self.workspace:
-            kwargs['workspace'] = self.workspace_path
-        return kwargs
-
-    def start_marionette(self):
-        self.marionette = self.driverclass(**self._build_kwargs())
-
-    def launch_test_container(self):
-        if self.marionette.session is None:
-            self.marionette.start_session()
-        self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
-
-        result = self.marionette.execute_async_script("""
-if((navigator.mozSettings == undefined) || (navigator.mozSettings == null) || (navigator.mozApps == undefined) || (navigator.mozApps == null)) {
-    marionetteScriptFinished(false);
-    return;
-}
-let setReq = navigator.mozSettings.createLock().set({'lockscreen.enabled': false});
-setReq.onsuccess = function() {
-    let appName = 'Test Container';
-    let activeApp = window.wrappedJSObject.Service.currentApp;
-
-    // if the Test Container is already open then do nothing
-    if(activeApp.name === appName){
-        marionetteScriptFinished(true);
-    }
-
-    let appsReq = navigator.mozApps.mgmt.getAll();
-    appsReq.onsuccess = function() {
-        let apps = appsReq.result;
-        for (let i = 0; i < apps.length; i++) {
-            let app = apps[i];
-            if (app.manifest.name === appName) {
-                app.launch();
-                window.addEventListener('appopen', function apploadtime(){
-                    window.removeEventListener('appopen', apploadtime);
-                    marionetteScriptFinished(true);
-                });
-                return;
-            }
-        }
-        marionetteScriptFinished(false);
-    }
-    appsReq.onerror = function() {
-        marionetteScriptFinished(false);
-    }
-}
-setReq.onerror = function() {
-    marionetteScriptFinished(false);
-}""", script_timeout=60000)
-
-        if not result:
-            raise Exception("Could not launch test container app")
-
-    def record_crash(self):
-        crash = True
-        try:
-            crash = self.marionette.check_for_crash()
-            self.crashed += int(crash)
-        except Exception:
-            traceback.print_exc()
-        return crash
-
     def run_tests(self, tests):
         assert len(tests) > 0
         assert len(self.test_handlers) > 0
         self.reset_test_stats()
         self.start_time = time.time()
 
-        need_external_ip = True
-        if not self.marionette:
-            self.start_marionette()
-            # if we're working against a desktop version, we usually don't need
-            # an external ip
-            if self.capabilities['device'] == "desktop":
-                need_external_ip = False
-        self.logger.info('Initial Profile Destination is '
-                         '"{}"'.format(self.marionette.profile_path))
-
         # Gaia sets server_root and that means we shouldn't spin up our own httpd
         if not self.httpd:
             if self.server_root is None or os.path.isdir(self.server_root):
                 self.logger.info("starting httpd")
-                self.start_httpd(need_external_ip)
-                self.marionette.baseurl = self.httpd.get_url()
-                self.logger.info("running httpd on %s" % self.marionette.baseurl)
+                self.httpd = self.create_httpd(False)
+                self.base_url = self.httpd.get_url()
+                self.logger.info("running httpd on %s" % self.base_url)
             else:
-                self.marionette.baseurl = self.server_root
-                self.logger.info("using remote content from %s" % self.marionette.baseurl)
+                self.base_url = self.server_root
+                self.logger.info("using remote content from %s" % self.base_url)
 
         device_info = None
 
         for test in tests:
             self.add_test(test)
 
         # ensure we have only tests files with names starting with 'test_'
         invalid_tests = \
             [t['filepath'] for t in self.tests
              if not os.path.basename(t['filepath']).startswith('test_')]
         if invalid_tests:
             raise Exception("Tests file names must starts with 'test_'."
                             " Invalid test names:\n  %s"
                             % '\n  '.join(invalid_tests))
 
         self.logger.info("running with e10s: {}".format(self.e10s))
-        version_info = mozversion.get_version(binary=self.bin,
-                                              sources=self.sources,
-                                              dm_type=os.environ.get('DM_TRANS', 'adb') )
 
-        self.logger.suite_start(self.tests,
-                                version_info=version_info,
-                                device_info=device_info)
+        self.logger.suite_start(self.tests)
 
         for test in self.manifest_skipped_tests:
             name = os.path.basename(test['path'])
             self.logger.test_start(name)
             self.logger.test_end(name,
                                  'SKIP',
                                  message=test['disabled'])
             self.todo += 1
@@ -845,39 +647,27 @@ setReq.onerror = function() {
         else:
             self.logger.info('todo: %d (skipped: %d)' % (self.todo, self.skipped))
 
         if self.failed > 0:
             self.logger.info('\nFAILED TESTS\n-------')
             for failed_test in self.failures:
                 self.logger.info('%s' % failed_test[0])
 
-        self.record_crash()
         self.end_time = time.time()
         self.elapsedtime = self.end_time - self.start_time
 
-        if self.marionette.instance:
-            self.marionette.instance.close()
-            self.marionette.instance = None
-
-        self.marionette.cleanup()
-
         for run_tests in self.mixin_run_tests:
             run_tests(tests)
         if self.shuffle:
             self.logger.info("Using seed where seed is:%d" % self.shuffle_seed)
 
         self.logger.info('mode: {}'.format('e10s' if self.e10s else 'non-e10s'))
         self.logger.suite_end()
 
-    def start_httpd(self, need_external_ip):
-        warnings.warn("start_httpd has been deprecated in favour of create_httpd",
-            DeprecationWarning)
-        self.httpd = self.create_httpd(need_external_ip)
-
     def create_httpd(self, need_external_ip):
         host = "127.0.0.1"
         if need_external_ip:
             host = moznetwork.get_ip()
         root = self.server_root or os.path.join(os.path.dirname(here), "www")
         rv = httpd.FixtureServer(root, host=host)
         rv.start()
         return rv
@@ -889,33 +679,28 @@ setReq.onerror = function() {
             for root, dirs, files in os.walk(filepath):
                 for filename in files:
                     if (filename.startswith('test_') and
                         (filename.endswith('.py') or filename.endswith('.js'))):
                         filepath = os.path.join(root, filename)
                         self.add_test(filepath)
             return
 
-
         file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
 
         if file_ext == '.ini':
             manifest = TestManifest()
             manifest.read(filepath)
 
             filters = []
             if self.test_tags:
                 filters.append(tags(self.test_tags))
-            json_path = update_mozinfo(filepath)
-            self.logger.info("mozinfo updated with the following: {}".format(None))
             manifest_tests = manifest.active_tests(exists=False,
                                                    disabled=True,
                                                    filters=filters,
-                                                   device=self.device,
-                                                   app=self.appName,
                                                    e10s=self.e10s,
                                                    **mozinfo.info)
             if len(manifest_tests) == 0:
                 self.logger.error("no tests to run using specified "
                                   "combination of filters: {}".format(
                                        manifest.fmt_filters()))
 
             target_tests = []
@@ -936,35 +721,35 @@ setReq.onerror = function() {
             return
 
         self.tests.append({'filepath': filepath, 'expected': expected, 'test_container': test_container})
 
     def run_test(self, filepath, expected, test_container):
 
         testloader = unittest.TestLoader()
         suite = unittest.TestSuite()
+        self.test_kwargs['binary'] = self.bin
         self.test_kwargs['expected'] = expected
+        self.test_kwargs['base_url'] = self.base_url
         self.test_kwargs['test_container'] = test_container
         mod_name = os.path.splitext(os.path.split(filepath)[-1])[0]
         for handler in self.test_handlers:
             if handler.match(os.path.basename(filepath)):
                 handler.add_tests_to_suite(mod_name,
                                            filepath,
                                            suite,
                                            testloader,
-                                           self.marionette,
                                            self.testvars,
                                            **self.test_kwargs)
                 break
 
         if suite.countTestCases():
             runner = self.textrunnerclass(logger=self.logger,
-                                          marionette=self.marionette,
-                                          capabilities=self.capabilities,
-                                          result_callbacks=self.result_callbacks)
+                                          result_callbacks=self.result_callbacks,
+                                          binary=self.bin)
 
             if test_container:
                 self.launch_test_container()
 
             results = runner.run(suite)
             self.results.append(results)
 
             self.failed += len(results.failures) + len(results.errors)
@@ -988,18 +773,16 @@ setReq.onerror = function() {
 
     def run_test_set(self, tests):
         if self.shuffle:
             random.seed(self.shuffle_seed)
             random.shuffle(tests)
 
         for test in tests:
             self.run_test(test['filepath'], test['expected'], test['test_container'])
-            if self.record_crash():
-                break
 
     def run_test_sets(self):
         if len(self.tests) < 1:
             raise Exception('There are no tests to run.')
         elif self.total_chunks > len(self.tests):
             raise ValueError('Total number of chunks must be between 1 and %d.' % len(self.tests))
         if self.total_chunks > 1:
             chunks = [[] for i in range(self.total_chunks)]
@@ -1014,12 +797,9 @@ setReq.onerror = function() {
             self.tests = chunks[self.this_chunk - 1]
 
         self.run_test_set(self.tests)
 
     def cleanup(self):
         if self.httpd:
             self.httpd.stop()
 
-        if self.marionette:
-            self.marionette.cleanup()
-
     __del__ = cleanup
--- a/testing/marionette/harness/session/runtests.py
+++ b/testing/marionette/harness/session/runtests.py
@@ -1,98 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 
-from marionette import __version__
-from marionette_driver import __version__ as driver_version
-from marionette.marionette_test import MarionetteTestCase, MarionetteJSTestCase
-from marionette.runner import (
-    BaseMarionetteTestRunner,
-    BaseMarionetteArguments,
-    BrowserMobProxyArguments,
-)
-import mozlog
-
-
-class MarionetteTestRunner(BaseMarionetteTestRunner):
-    def __init__(self, **kwargs):
-        BaseMarionetteTestRunner.__init__(self, **kwargs)
-        self.test_handlers = [MarionetteTestCase, MarionetteJSTestCase]
-
-
-class MarionetteArguments(BaseMarionetteArguments):
-    def __init__(self, **kwargs):
-        BaseMarionetteArguments.__init__(self, **kwargs)
-        self.register_argument_container(BrowserMobProxyArguments())
+from session.session_test import SessionTestCase, SessionJSTestCase
+from session.runner import BaseSessionTestRunner, BaseSessionArguments
+from marionette.runtests import MarionetteHarness, cli
 
 
-class MarionetteHarness(object):
-    def __init__(self,
-                 runner_class=MarionetteTestRunner,
-                 parser_class=MarionetteArguments,
-                 args=None):
-        self._runner_class = runner_class
-        self._parser_class = parser_class
-        self.args = args or self.parse_args()
-
-    def parse_args(self, logger_defaults=None):
-        parser = self._parser_class(usage='%(prog)s [options] test_file_or_dir <test_file_or_dir> ...')
-        parser.add_argument('--version', action='version',
-            help="Show version information.",
-            version="%(prog)s {version}"
-                    " (using marionette-driver: {driver_version}, ".format(
-                        version=__version__,
-                        driver_version=driver_version
-                    ))
-        mozlog.commandline.add_logging_group(parser)
-        args = parser.parse_args()
-        parser.verify_usage(args)
-
-        logger = mozlog.commandline.setup_logging(
-            args.logger_name, args, logger_defaults or {"tbpl": sys.stdout})
-
-        args.logger = logger
-        return vars(args)
-
-    def process_args(self):
-        if self.args.get('pydebugger'):
-            MarionetteTestCase.pydebugger = __import__(self.args['pydebugger'])
-
-    def run(self):
-        try:
-            self.process_args()
-            tests = self.args.pop('tests')
-            runner = self._runner_class(**self.args)
-            runner.run_tests(tests)
-            return runner.failed + runner.crashed
-        except Exception:
-            logger = self.args.get('logger')
-            if logger:
-                logger.error('Failure during test execution.',
-                                       exc_info=True)
-            raise
+class SessionTestRunner(BaseSessionTestRunner):
+    def __init__(self, **kwargs):
+        BaseSessionTestRunner.__init__(self, **kwargs)
+        self.test_handlers = [SessionTestCase, SessionJSTestCase]
 
 
-def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
-        harness_class=MarionetteHarness, args=None):
-    """
-    Call the harness to parse args and run tests.
+class SessionArguments(BaseSessionArguments):
+    def __init__(self, **kwargs):
+        BaseSessionArguments.__init__(self, **kwargs)
 
-    The following exit codes are expected:
-    - Test failures: 10
-    - Harness/other failures: 1
-    - Success: 0
-    """
-    logger = mozlog.commandline.setup_logging('Marionette test runner', {})
-    try:
-        failed = harness_class(runner_class, parser_class, args=args).run()
-        if failed > 0:
-            sys.exit(10)
-    except Exception:
-        logger.error('Failure during harness setup', exc_info=True)
-        sys.exit(1)
-    sys.exit(0)
 
 if __name__ == "__main__":
-    cli()
+    cli(runner_class=SessionTestRunner, parser_class=SessionArguments,
+        harness_class=MarionetteHarness, testcase_class=SessionTestCase, args=None)
--- a/testing/marionette/harness/session/session_test.py
+++ b/testing/marionette/harness/session/session_test.py
@@ -10,244 +10,44 @@ import sys
 import socket
 import time
 import types
 import unittest
 import weakref
 import warnings
 
 
+from mozprofile import FirefoxProfile
+from mozrunner import FirefoxRunner
 from marionette_driver.errors import (
         MarionetteException, TimeoutException,
         JavascriptException, NoSuchElementException, NoSuchWindowException,
         StaleElementException, ScriptTimeoutException, ElementNotVisibleException,
         NoSuchFrameException, InvalidElementStateException, NoAlertPresentException,
         InvalidCookieDomainException, UnableToSetCookieException, InvalidSelectorException,
         MoveTargetOutOfBoundsException
         )
 from marionette_driver.marionette import Marionette
 from marionette_driver.wait import Wait
 from marionette_driver.expected import element_present, element_not_present
 from mozlog import get_default_logger
 
-
-class SkipTest(Exception):
-    """
-    Raise this exception in a test to skip it.
-
-    Usually you can use TestResult.skip() or one of the skipping decorators
-    instead of raising this directly.
-    """
-    pass
-
-class _ExpectedFailure(Exception):
-    """
-    Raise this when a test is expected to fail.
-
-    This is an implementation detail.
-    """
-
-    def __init__(self, exc_info):
-        super(_ExpectedFailure, self).__init__()
-        self.exc_info = exc_info
-
-class _UnexpectedSuccess(Exception):
-    """
-    The test was supposed to fail, but it didn't!
-    """
-    pass
-
-def skip(reason):
-    """Unconditionally skip a test."""
-    def decorator(test_item):
-        if not isinstance(test_item, (type, types.ClassType)):
-            @functools.wraps(test_item)
-            def skip_wrapper(*args, **kwargs):
-                raise SkipTest(reason)
-            test_item = skip_wrapper
-
-        test_item.__unittest_skip__ = True
-        test_item.__unittest_skip_why__ = reason
-        return test_item
-    return decorator
-
-def expectedFailure(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            func(*args, **kwargs)
-        except Exception:
-            raise _ExpectedFailure(sys.exc_info())
-        raise _UnexpectedSuccess
-    return wrapper
-
-def skip_if_chrome(target):
-    def wrapper(self, *args, **kwargs):
-        if self.marionette._send_message("getContext", key="value") == "chrome":
-            raise SkipTest("skipping test in chrome context")
-        return target(self, *args, **kwargs)
-    return wrapper
-
-def skip_if_desktop(target):
-    def wrapper(self, *args, **kwargs):
-        if self.marionette.session_capabilities.get('b2g') is None:
-            raise SkipTest('skipping due to desktop')
-        return target(self, *args, **kwargs)
-    return wrapper
-
-def skip_if_e10s(target):
-    def wrapper(self, *args, **kwargs):
-        with self.marionette.using_context('chrome'):
-            multi_process_browser = self.marionette.execute_script("""
-            try {
-              return Services.appinfo.browserTabsRemoteAutostart;
-            } catch (e) {
-              return false;
-            }""")
-
-        if multi_process_browser:
-            raise SkipTest('skipping due to e10s')
-        return target(self, *args, **kwargs)
-    return wrapper
-
-def skip_unless_protocol(predicate):
-    """Given a predicate passed the current protocol level, skip the
-    test if the predicate does not match."""
-    def decorator(test_item):
-        @functools.wraps(test_item)
-        def skip_wrapper(self):
-            level = self.marionette.client.protocol
-            if not predicate(level):
-                raise SkipTest('skipping because protocol level is %s' % level)
-            return self
-        return skip_wrapper
-    return decorator
-
-def skip_unless_browser_pref(pref, predicate=bool):
-    """
-    Skip a test based on the value of a browser preference.
-
-    :param pref: the preference name
-    :param predicate: a function that should return false to skip the test.
-                      The function takes one parameter, the preference value.
-                      Defaults to the python built-in bool function.
-
-    Note that the preference must exist, else a failure is raised.
-
-    Example: ::
+from marionette.marionette_test import (
+        SkipTest,
+        _ExpectedFailure,
+        _UnexpectedSuccess,
+        skip,
+        expectedFailure,
+        parameterized,
+        with_parameters,
+        wraps_parameterized,
+        MetaParameterized,
+        JSTest
+        )
 
-      class TestSomething(MarionetteTestCase):
-          @skip_unless_browser_pref("accessibility.tabfocus",
-                                    lambda value: value >= 7)
-          def test_foo(self):
-              pass  # test implementation here
-    """
-    def wrapper(target):
-        @functools.wraps(target)
-        def wrapped(self, *args, **kwargs):
-            value = self.marionette.get_pref(pref)
-            if value is None:
-                self.fail("No such browser preference: %r" % pref)
-            if not predicate(value):
-                raise SkipTest("browser preference %r: %r" % (pref, value))
-            return target(self, *args, **kwargs)
-        return wrapped
-    return wrapper
-
-def parameterized(func_suffix, *args, **kwargs):
-    """
-    A decorator that can generate methods given a base method and some data.
-
-    **func_suffix** is used as a suffix for the new created method and must be
-    unique given a base method. if **func_suffix** countains characters that
-    are not allowed in normal python function name, these characters will be
-    replaced with "_".
-
-    This decorator can be used more than once on a single base method. The class
-    must have a metaclass of :class:`MetaParameterized`.
-
-    Example::
-
-      # This example will generate two methods:
-      #
-      # - MyTestCase.test_it_1
-      # - MyTestCase.test_it_2
-      #
-      class MyTestCase(MarionetteTestCase):
-          @parameterized("1", 5, named='name')
-          @parameterized("2", 6, named='name2')
-          def test_it(self, value, named=None):
-              print value, named
-
-    :param func_suffix: will be used as a suffix for the new method
-    :param \*args: arguments to pass to the new method
-    :param \*\*kwargs: named arguments to pass to the new method
-    """
-    def wrapped(func):
-        if not hasattr(func, 'metaparameters'):
-            func.metaparameters = []
-        func.metaparameters.append((func_suffix, args, kwargs))
-        return func
-    return wrapped
-
-def with_parameters(parameters):
-    """
-    A decorator that can generate methods given a base method and some data.
-    Acts like :func:`parameterized`, but define all methods in one call.
-
-    Example::
-
-      # This example will generate two methods:
-      #
-      # - MyTestCase.test_it_1
-      # - MyTestCase.test_it_2
-      #
-
-      DATA = [("1", [5], {'named':'name'}), ("2", [6], {'named':'name2'})]
-
-      class MyTestCase(MarionetteTestCase):
-          @with_parameters(DATA)
-          def test_it(self, value, named=None):
-              print value, named
-
-    :param parameters: list of tuples (**func_suffix**, **args**, **kwargs**)
-                       defining parameters like in :func:`todo`.
-    """
-    def wrapped(func):
-        func.metaparameters = parameters
-        return func
-    return wrapped
-
-def wraps_parameterized(func, func_suffix, args, kwargs):
-    """Internal: for MetaParameterized"""
-    def wrapper(self):
-        return func(self, *args, **kwargs)
-    wrapper.__name__ = func.__name__ + '_' + str(func_suffix)
-    wrapper.__doc__ = '[%s] %s' % (func_suffix, func.__doc__)
-    return wrapper
-
-class MetaParameterized(type):
-    """
-    A metaclass that allow a class to use decorators like :func:`parameterized`
-    or :func:`with_parameters` to generate new methods.
-    """
-    RE_ESCAPE_BAD_CHARS = re.compile(r'[\.\(\) -/]')
-    def __new__(cls, name, bases, attrs):
-        for k, v in attrs.items():
-            if callable(v) and hasattr(v, 'metaparameters'):
-                for func_suffix, args, kwargs in v.metaparameters:
-                    func_suffix = cls.RE_ESCAPE_BAD_CHARS.sub('_', func_suffix)
-                    wrapper = wraps_parameterized(v, func_suffix, args, kwargs)
-                    if wrapper.__name__ in attrs:
-                        raise KeyError("%s is already a defined method on %s" %
-                                        (wrapper.__name__, name))
-                    attrs[wrapper.__name__] = wrapper
-                del attrs[k]
-
-        return type.__new__(cls, name, bases, attrs)
 
 class JSTest:
     head_js_re = re.compile(r"MARIONETTE_HEAD_JS(\s*)=(\s*)['|\"](.*?)['|\"];")
     context_re = re.compile(r"MARIONETTE_CONTEXT(\s*)=(\s*)['|\"](.*?)['|\"];")
     timeout_re = re.compile(r"MARIONETTE_TIMEOUT(\s*)=(\s*)(\d+);")
     inactivity_timeout_re = re.compile(r"MARIONETTE_INACTIVITY_TIMEOUT(\s*)=(\s*)(\d+);")
 
 class CommonTestCase(unittest.TestCase):
@@ -256,19 +56,20 @@ class CommonTestCase(unittest.TestCase):
     match_re = None
     failureException = AssertionError
     pydebugger = None
 
     def __init__(self, methodName, **kwargs):
         unittest.TestCase.__init__(self, methodName)
         self.loglines = []
         self.duration = 0
-        self.start_time = 0
         self.expected = kwargs.pop('expected', 'pass')
         self.logger = get_default_logger()
+        self.profile = FirefoxProfile()
+        self.binary = kwargs.pop('binary', None)
 
     def _enter_pm(self):
         if self.pydebugger:
             self.pydebugger.post_mortem(sys.exc_info()[2])
 
     def _addSkip(self, result, reason):
         addSkip = getattr(result, 'addSkip', None)
         if addSkip is not None:
@@ -401,17 +202,17 @@ class CommonTestCase(unittest.TestCase):
         using cls.match_re.
         """
         if not cls.match_re:
             return False
         m = cls.match_re.match(filename)
         return m is not None
 
     @classmethod
-    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars):
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars):
         """
         Adds all the tests in the specified file to the specified suite.
         """
         raise NotImplementedError
 
     @property
     def test_name(self):
         if hasattr(self, 'jsFile'):
@@ -429,28 +230,28 @@ class CommonTestCase(unittest.TestCase):
         return self.test_name
 
     def setUp(self):
         # Convert the marionette weakref to an object, just for the
         # duration of the test; this is deleted in tearDown() to prevent
         # a persistent circular reference which in turn would prevent
         # proper garbage collection.
         self.start_time = time.time()
-        self.marionette = self._marionette_weakref()
+        self.marionette = Marionette(bin=self.binary, profile=self.profile)
         if self.marionette.session is None:
             self.marionette.start_session()
         if self.marionette.timeout is not None:
             self.marionette.timeouts(self.marionette.TIMEOUT_SEARCH, self.marionette.timeout)
             self.marionette.timeouts(self.marionette.TIMEOUT_SCRIPT, self.marionette.timeout)
             self.marionette.timeouts(self.marionette.TIMEOUT_PAGE, self.marionette.timeout)
         else:
             self.marionette.timeouts(self.marionette.TIMEOUT_PAGE, 30000)
 
     def tearDown(self):
-        pass
+        self.marionette.cleanup()
 
     def cleanTest(self):
         self._deleteSession()
 
     def _deleteSession(self):
         if hasattr(self, 'start_time'):
             self.duration = time.time() - self.start_time
         if hasattr(self.marionette, 'session'):
@@ -602,32 +403,31 @@ class CommonTestCase(unittest.TestCase):
                 pass
             else:
                 self.loglines = marionette.get_logs()
                 raise
         self.marionette.test_name = original_test_name
 
 
 
-class MarionetteTestCase(CommonTestCase):
+class SessionTestCase(CommonTestCase):
 
     match_re = re.compile(r"test_(.*)\.py$")
 
-    def __init__(self, marionette_weakref, methodName='runTest',
+    def __init__(self, methodName='runTest',
                  filepath='', **kwargs):
-        self._marionette_weakref = marionette_weakref
         self.marionette = None
         self.methodName = methodName
         self.filepath = filepath
         self.testvars = kwargs.pop('testvars', None)
         self.test_container = kwargs.pop('test_container', None)
         CommonTestCase.__init__(self, methodName, **kwargs)
 
     @classmethod
-    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars, **kwargs):
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars, **kwargs):
         # since we use imp.load_source to load test modules, if a module
         # is loaded with the same name as another one the module would just be
         # reloaded.
         #
         # We may end up by finding too many test in a module then since
         # reload() only update the module dict (so old keys are still there!)
         # see https://docs.python.org/2/library/functions.html#reload
         #
@@ -640,85 +440,55 @@ class MarionetteTestCase(CommonTestCase)
         test_mod = imp.load_source(mod_name, filepath)
 
         for name in dir(test_mod):
             obj = getattr(test_mod, name)
             if (isinstance(obj, (type, types.ClassType)) and
                 issubclass(obj, unittest.TestCase)):
                 testnames = testloader.getTestCaseNames(obj)
                 for testname in testnames:
-                    suite.addTest(obj(weakref.ref(marionette),
-                                  methodName=testname,
+                    suite.addTest(obj(methodName=testname,
                                   filepath=filepath,
                                   testvars=testvars,
                                   **kwargs))
 
     def setUp(self):
         CommonTestCase.setUp(self)
-        self.marionette.test_name = self.test_name
-        self.marionette.execute_script("log('TEST-START: %s:%s')" %
-                                       (self.filepath.replace('\\', '\\\\'), self.methodName),
-                                       sandbox="simpletest")
 
     def tearDown(self):
-        if not self.marionette.check_for_crash():
-            try:
-                self.marionette.clear_imported_scripts()
-                self.marionette.execute_script("log('TEST-END: %s:%s')" %
-                                               (self.filepath.replace('\\', '\\\\'),
-                                                self.methodName),
-                                               sandbox="simpletest")
-                self.marionette.test_name = None
-            except (MarionetteException, IOError):
-                # We have tried to log the test end when there is no listener
-                # object that we can access
-                pass
-
         CommonTestCase.tearDown(self)
 
     def wait_for_condition(self, method, timeout=30):
         timeout = float(timeout) + time.time()
         while time.time() < timeout:
             value = method(self.marionette)
             if value:
                 return value
             time.sleep(0.5)
         else:
             raise TimeoutException("wait_for_condition timed out")
 
-class MarionetteJSTestCase(CommonTestCase):
+class SessionJSTestCase(CommonTestCase):
 
     match_re = re.compile(r"test_(.*)\.js$")
 
-    def __init__(self, marionette_weakref, methodName='runTest', jsFile=None, **kwargs):
+    def __init__(self, methodName='runTest', jsFile=None, **kwargs):
         assert(jsFile)
         self.jsFile = jsFile
-        self._marionette_weakref = marionette_weakref
         self.marionette = None
         self.test_container = kwargs.pop('test_container', None)
         CommonTestCase.__init__(self, methodName)
 
     @classmethod
-    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars, **kwargs):
-        suite.addTest(cls(weakref.ref(marionette), jsFile=filepath, **kwargs))
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars, **kwargs):
+        suite.addTest(cls(jsFile=filepath, **kwargs))
 
     def runTest(self):
-        if self.marionette.session is None:
-            self.marionette.start_session()
-        self.marionette.execute_script(
-            "log('TEST-START: %s');" % self.jsFile.replace('\\', '\\\\'),
-            sandbox="simpletest")
-
         self.run_js_test(self.jsFile)
 
-        self.marionette.execute_script(
-            "log('TEST-END: %s');" % self.jsFile.replace('\\', '\\\\'),
-            sandbox="simpletest")
-        self.marionette.test_name = None
-
     def get_test_class_name(self):
         # returns a dot separated folders as class name
         dirname = os.path.dirname(self.jsFile).replace('\\', '/')
         if dirname.startswith('/'):
             dirname = dirname[1:]
         return '.'.join(dirname.split('/'))
 
     def get_test_method_name(self):
--- a/testing/marionette/harness/session/tests/test_session.py
+++ b/testing/marionette/harness/session/tests/test_session.py
@@ -1,60 +1,23 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import itertools
 
 from marionette_driver import errors
-from marionette.marionette_test import MarionetteTestCase as TC
+from session.session_test import SessionTestCase as TC
 
 
-class TestProtocol1Errors(TC):
+class TestHelloWorld(TC):
     def setUp(self):
         TC.setUp(self)
-        self.op = self.marionette.protocol
-        self.marionette.protocol = 1
 
     def tearDown(self):
-        self.marionette.protocol = self.op
-        TC.tearDown(self)
-
-    def test_malformed_packet(self):
-        for t in [{}, {"error": None}]:
-            with self.assertRaisesRegexp(errors.MarionetteException, "Malformed packet"):
-                self.marionette._handle_error(t)
-
-    def test_known_error_code(self):
-        with self.assertRaises(errors.NoSuchElementException):
-            self.marionette._handle_error(
-                {"error": {"status": errors.NoSuchElementException.code[0]}})
-
-    def test_known_error_status(self):
-        with self.assertRaises(errors.NoSuchElementException):
-            self.marionette._handle_error(
-                {"error": {"status": errors.NoSuchElementException.status}})
-
-    def test_unknown_error_code(self):
-        with self.assertRaises(errors.MarionetteException):
-            self.marionette._handle_error({"error": {"status": 123456}})
-
-    def test_unknown_error_status(self):
-        with self.assertRaises(errors.MarionetteException):
-            self.marionette._handle_error({"error": {"status": "barbera"}})
-
-
-class TestProtocol2Errors(TC):
-    def setUp(self):
-        TC.setUp(self)
-        self.op = self.marionette.protocol
-        self.marionette.protocol = 2
-
-    def tearDown(self):
-        self.marionette.protocol = self.op
         TC.tearDown(self)
 
     def test_malformed_packet(self):
         req = ["error", "message", "stacktrace"]
         ps = []
         for p in [p for i in range(0, len(req) + 1) for p in itertools.permutations(req, i)]:
             ps.append(dict((x, None) for x in p))
 
--- a/testing/marionette/harness/session/tests/unit-tests.ini
+++ b/testing/marionette/harness/session/tests/unit-tests.ini
@@ -1,15 +1,2 @@
-; marionette unit tests
-[include:unit/unit-tests.ini]
-test_container = true
-
-; layout tests
-[include:../../../../../layout/base/tests/marionette/manifest.ini]
-
-; loop tests
-[include:../../../../../browser/extensions/loop/manifest.ini]
-
-; microformats tests
-[include:../../../../../toolkit/components/microformats/manifest.ini]
-
-; migration tests
-[include:../../../../../browser/components/migration/tests/marionette/manifest.ini]
+; session unit tests
+[test_session.py]
--- a/testing/marionette/mach_commands.py
+++ b/testing/marionette/mach_commands.py
@@ -15,17 +15,17 @@ from mozbuild.base import (
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 
-def setup_argument_parser():
+def setup_marionette_argument_parser():
     from marionette.runner.base import BaseMarionetteArguments
     return BaseMarionetteArguments()
 
 def run_marionette(tests, testtype=None, address=None, binary=None, topsrcdir=None, **kwargs):
     from mozlog.structured import commandline
 
     from marionette.runtests import (
         MarionetteTestRunner,
@@ -38,32 +38,75 @@ def run_marionette(tests, testtype=None,
 
     if not tests:
         tests = [os.path.join(topsrcdir,
                  'testing/marionette/harness/marionette/tests/unit-tests.ini')]
 
     args = parser.parse_args(args=tests)
 
     args.binary = binary
-    path, exe = os.path.split(args.binary)
 
     for k, v in kwargs.iteritems():
         setattr(args, k, v)
 
     parser.verify_usage(args)
 
     args.logger = commandline.setup_logging("Marionette Unit Tests",
                                             args,
                                             {"mach": sys.stdout})
     failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
     if failed > 0:
         return 1
     else:
         return 0
 
+def setup_session_argument_parser():
+    from session.runner.base import BaseSessionArguments
+    return BaseSessionArguments()
+
+def run_session(tests, testtype=None, address=None, binary=None, topsrcdir=None, **kwargs):
+    from mozlog.structured import commandline
+
+    from marionette.runtests import (
+        MarionetteHarness
+    )
+
+    from session.runtests import (
+        SessionTestRunner,
+        BaseSessionArguments,
+        SessionArguments,
+        SessionTestCase,
+    )
+
+    parser = BaseSessionArguments()
+    commandline.add_logging_group(parser)
+
+    if not tests:
+        tests = [os.path.join(topsrcdir,
+                 'testing/marionette/harness/session/tests/unit-tests.ini')]
+
+    args = parser.parse_args(args=tests)
+
+    args.binary = binary
+
+    for k, v in kwargs.iteritems():
+        setattr(args, k, v)
+
+    parser.verify_usage(args)
+
+    args.logger = commandline.setup_logging("Session Unit Tests",
+                                            args,
+                                            {"mach": sys.stdout})
+    failed = MarionetteHarness(runner_class=SessionTestRunner, parser_class=SessionArguments,
+                               testcase_class=SessionTestCase, args=vars(args)).run()
+    if failed > 0:
+        return 1
+    else:
+        return 0
+
 @CommandProvider
 class B2GCommands(MachCommandBase):
     def __init__(self, context):
         MachCommandBase.__init__(self, context)
 
         for attr in ('b2g_home', 'device_name'):
             setattr(self, attr, getattr(context, attr, None))
     @Command('marionette-webapi', category='testing',
@@ -90,19 +133,34 @@ class B2GCommands(MachCommandBase):
         return run_marionette(tests, b2g_path=self.b2g_home, emulator=emulator,
             topsrcdir=self.topsrcdir, **kwargs)
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('marionette-test', category='testing',
         description='Run a Marionette test (Check UI or the internal JavaScript using marionette).',
         conditions=[conditions.is_firefox],
-        parser=setup_argument_parser,
+        parser=setup_marionette_argument_parser,
     )
     def run_marionette_test(self, tests, **kwargs):
         if 'test_objects' in kwargs:
             tests = []
             for obj in kwargs['test_objects']:
                 tests.append(obj['file_relpath'])
             del kwargs['test_objects']
 
         kwargs['binary'] = self.get_binary_path('app')
         return run_marionette(tests, topsrcdir=self.topsrcdir, **kwargs)
+
+    @Command('session-test', category='testing',
+        description='Run a Session test (Check Telemetry using marionette).',
+        conditions=[conditions.is_firefox],
+        parser=setup_session_argument_parser,
+    )
+    def run_session_test(self, tests, **kwargs):
+        if 'test_objects' in kwargs:
+            tests = []
+            for obj in kwargs['test_objects']:
+                tests.append(obj['file_relpath'])
+            del kwargs['test_objects']
+
+        kwargs['binary'] = self.get_binary_path('app')
+        return run_session(tests, topsrcdir=self.topsrcdir, **kwargs)