Bug 1039833 - Add structured logging command line arguments for mochitest/mach. r=chmanchester, r=ahal, a=test-only
authorAhmed Kachkach <ahmed.kachkach@gmail.com>
Wed, 13 Aug 2014 12:03:00 -0400
changeset 217533 7d0337de2e1567beb3e0c20dd5c4ae223701f94d
parent 217532 2817d9e1a008e069ac57d26d5b4323dcbe5f7d4f
child 217534 40bd25be82639d5a2330bb65c06b0b015e6f0bfa
push id515
push userraliiev@mozilla.com
push dateMon, 06 Oct 2014 12:51:51 +0000
treeherdermozilla-release@267c7a481bef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester, ahal, test-only
bugs1039833
milestone33.0a2
Bug 1039833 - Add structured logging command line arguments for mochitest/mach. r=chmanchester, r=ahal, a=test-only
testing/config/mozharness/android_arm_config.py
testing/config/mozharness/android_panda_config.py
testing/config/mozharness/android_x86_config.py
testing/config/mozharness/b2g_desktop_config.py
testing/config/mozharness/b2g_emulator_config.py
testing/config/mozharness/linux_config.py
testing/config/mozharness/mac_config.py
testing/config/mozharness/windows_config.py
testing/mochitest/mach_commands.py
testing/mochitest/runtests.py
testing/mochitest/runtestsb2g.py
testing/mochitest/runtestsremote.py
testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
--- a/testing/config/mozharness/android_arm_config.py
+++ b/testing/config/mozharness/android_arm_config.py
@@ -7,17 +7,17 @@ config = {
         "mochitest": {
             "run_filename": "runtestsremote.py",
             "options": ["--autorun", "--close-when-done", "--dm_trans=sut",
                 "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
                 "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s",
                 "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
                 "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s",
-                "--quiet"
+                "--quiet", "--log-raw=%(raw_log_file)s"
             ],
         },
         "reftest": {
             "run_filename": "remotereftest.py",
             "options": [ "--app=%(app)s", "--ignore-window-size",
                 "--bootstrap",
                 "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s",
                 "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s",
--- a/testing/config/mozharness/android_panda_config.py
+++ b/testing/config/mozharness/android_panda_config.py
@@ -5,17 +5,17 @@
 config = {
     "mochitest_options": [
         "--deviceIP=%(device_ip)s",
         "--xre-path=../hostutils/xre",
         "--utility-path=../hostutils/bin", "--certificate-path=certs",
         "--app=%(app_name)s", "--console-level=INFO",
         "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
         "--run-only-tests=android.json", "--symbols-path=%(symbols_path)s",
-        "--quiet"
+        "--quiet", "--log-raw=%(raw_log_file)s"
     ],
     "reftest_options": [
         "--deviceIP=%(device_ip)s",
         "--xre-path=../hostutils/xre",
         "--utility-path=../hostutils/bin",
         "--app=%(app_name)s",
         "--ignore-window-size", "--bootstrap",
         "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
--- a/testing/config/mozharness/android_x86_config.py
+++ b/testing/config/mozharness/android_x86_config.py
@@ -7,17 +7,17 @@ config = {
         "mochitest": {
             "run_filename": "runtestsremote.py",
             "options": ["--autorun", "--close-when-done", "--dm_trans=sut",
                 "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
                 "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
                 "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s",
                 "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
                 "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s",
-                "--quiet"
+                "--quiet", "--log-raw=%(raw_log_file)s"
             ],
         },
         "reftest": {
             "run_filename": "remotereftest.py",
             "options": [ "--app=%(app)s", "--ignore-window-size",
                 "--bootstrap",
                 "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s",
                 "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s",
--- a/testing/config/mozharness/b2g_desktop_config.py
+++ b/testing/config/mozharness/b2g_desktop_config.py
@@ -4,17 +4,17 @@
 
 config = {
     "mochitest_options": [
         "--console-level=INFO", "%(test_manifest)s",
         "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
         "--profile=%(gaia_profile)s", "--app=%(application)s", "--desktop",
         "--utility-path=%(utility_path)s", "--certificate-path=%(cert_path)s",
         "--symbols-path=%(symbols_path)s", "--browser-arg=%(browser_arg)s",
-        "--quiet"
+        "--quiet", "--log-raw=%(raw_log_file)s"
     ],
 
     "reftest_options": [
         "--desktop", "--profile=%(gaia_profile)s", "--appname=%(application)s",
         "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
         "--browser-arg=%(browser_arg)s", "--symbols-path=%(symbols_path)s",
         "%(test_manifest)s"
     ]
--- a/testing/config/mozharness/b2g_emulator_config.py
+++ b/testing/config/mozharness/b2g_emulator_config.py
@@ -14,17 +14,18 @@ config = {
     ],
 
     "mochitest_options": [
         "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--console-level=INFO",
         "--emulator=%(emulator)s", "--logdir=%(logcat_dir)s",
         "--remote-webserver=%(remote_webserver)s", "%(test_manifest)s",
         "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
         "--total-chunks=%(total_chunks)s", "--this-chunk=%(this_chunk)s",
-        "--quiet", "--certificate-path=%(certificate_path)s",
+        "--quiet", "--log-raw=%(raw_log_file)s",
+        "--certificate-path=%(certificate_path)s",
         "--test-path=%(test_path)s",
     ],
 
     "reftest_options": [
         "--adbpath=%(adbpath)s", "--b2gpath=%(b2gpath)s", "--emulator=%(emulator)s",
         "--emulator-res=800x1000", "--logdir=%(logcat_dir)s",
         "--remote-webserver=%(remote_webserver)s", "--ignore-window-size",
         "--xre-path=%(xre_path)s", "--symbols-path=%(symbols_path)s", "--busybox=%(busybox)s",
--- a/testing/config/mozharness/linux_config.py
+++ b/testing/config/mozharness/linux_config.py
@@ -7,17 +7,17 @@ config = {
         "--appname=%(binary_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
     ],
     "mochitest_options": [
         "--appname=%(binary_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
         "--certificate-path=tests/certs", "--autorun", "--close-when-done",
         "--console-level=INFO", "--setpref=webgl.force-enabled=true",
-        "--quiet",
+        "--quiet", "--log-raw=%(raw_log_file)s",
         "--use-test-media-devices"
     ],
     "webapprt_options": [
         "--app=%(app_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
         "--certificate-path=tests/certs", "--autorun", "--close-when-done",
         "--console-level=INFO", "--testing-modules-dir=tests/modules",
         "--quiet"
--- a/testing/config/mozharness/mac_config.py
+++ b/testing/config/mozharness/mac_config.py
@@ -7,17 +7,17 @@ config = {
         "--appname=%(binary_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
     ],
     "mochitest_options": [
         "--appname=%(binary_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
         "--certificate-path=tests/certs", "--autorun", "--close-when-done",
         "--console-level=INFO",
-        "--quiet"
+        "--quiet", "--log-raw=%(raw_log_file)s"
     ],
     "webapprt_options": [
         "--app=%(app_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
         "--certificate-path=tests/certs", "--autorun", "--close-when-done",
         "--console-level=INFO", "--testing-modules-dir=tests/modules",
         "--quiet"
     ],
--- a/testing/config/mozharness/windows_config.py
+++ b/testing/config/mozharness/windows_config.py
@@ -7,17 +7,17 @@ config = {
         "--appname=%(binary_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s"
     ],
     "mochitest_options": [
         "--appname=%(binary_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
         "--certificate-path=tests/certs", "--autorun", "--close-when-done",
         "--console-level=INFO",
-        "--quiet"
+        "--quiet", "--log-raw=%(raw_log_file)s"
     ],
     "webapprt_options": [
         "--app=%(app_path)s", "--utility-path=tests/bin",
         "--extra-profile-file=tests/bin/plugins", "--symbols-path=%(symbols_path)s",
         "--certificate-path=tests/certs", "--autorun", "--close-when-done",
         "--console-level=INFO", "--testing-modules-dir=tests/modules",
         "--quiet"
     ],
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
+import argparse
 import logging
 import mozpack.path
 import os
 import platform
 import sys
 import warnings
 import which
 
@@ -20,16 +21,18 @@ from mozbuild.base import (
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 
+from mozlog import structured
+
 ADB_NOT_FOUND = '''
 The %s command requires the adb binary to be on your path.
 
 If you have a B2G build, this can be found in
 '%s/out/host/<platform>/bin'.
 '''.lstrip()
 
 GAIA_PROFILE_NOT_FOUND = '''
@@ -245,18 +248,16 @@ class MochitestRunner(MozbuildObject):
 
         # Automation installs its own stream handler to stdout. Since we want
         # all logging to go through us, we just remove their handler.
         remove_handlers = [l for l in logging.getLogger().handlers
             if isinstance(l, logging.StreamHandler)]
         for handler in remove_handlers:
             logging.getLogger().removeHandler(handler)
 
-        runner = mochitest.Mochitest()
-
         opts = mochitest.MochitestOptions()
         options, args = opts.parse_args([])
 
         options.subsuite = ''
         flavor = suite
 
         # Need to set the suite options before verifyOptions below.
         if suite == 'plain':
@@ -356,16 +357,18 @@ class MochitestRunner(MozbuildObject):
                 return 1
             options.debuggerArgs = debugger_args
 
         if app_override == "dist":
             options.app = self.get_binary_path(where='staged-package')
         elif app_override:
             options.app = app_override
 
+        logger_options = {key: value for key, value in vars(options).iteritems() if key.startswith('log')}
+        runner = mochitest.Mochitest(logger_options)
         options = opts.verifyOptions(options, runner)
 
         if options is None:
             raise Exception('mochitest option validator failed.')
 
         # We need this to enable colorization of output.
         self.log_manager.enable_unstructured()
 
@@ -589,22 +592,24 @@ def B2GCommand(func):
         help='Test to run. Can be specified as a single file, a ' \
             'directory, or omitted. If omitted, the entire test suite is ' \
             'executed.')
     func = path(func)
 
     return func
 
 
+_st_parser = argparse.ArgumentParser()
+structured.commandline.add_logging_group(_st_parser)
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('mochitest-plain', category='testing',
         conditions=[conditions.is_firefox],
-        description='Run a plain mochitest.')
+        description='Run a plain mochitest.', parser=_st_parser)
     @MochitestCommand
     def run_mochitest_plain(self, test_paths, **kwargs):
         return self.run_mochitest(test_paths, 'plain', **kwargs)
 
     @Command('mochitest-chrome', category='testing',
         conditions=[conditions.is_firefox],
         description='Run a chrome mochitest.')
     @MochitestCommand
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -35,18 +35,18 @@ import bisection
 from automationutils import environment, getDebuggerInfo, isURL, KeyValueParseError, parseKeyValue, processLeakLog, dumpScreen, ShutdownLeaks, printstatus, LSANLeaks
 from datetime import datetime
 from manifestparser import TestManifest
 from mochitest_options import MochitestOptions
 from mozprofile import Profile, Preferences
 from mozprofile.permissions import ServerLocations
 from urllib import quote_plus as encodeURIComponent
 from mozlog.structured.formatters import TbplFormatter
+from mozlog.structured.commandline import add_logging_group, setup_logging
 from mozlog.structured.handlers import StreamHandler
-from mozlog.structured.structuredlog import StructuredLogger
 
 # This should use the `which` module already in tree, but it is
 # not yet present in the mozharness environment
 from mozrunner.utils import findInPath as which
 
 
 # Necessary to set up the global logger in automationutils.py
 import logging
@@ -93,17 +93,17 @@ class MessageLogger(object):
     BUFFERING_THRESHOLD = 100
     # This is a delimiter used by the JS side to avoid logs interleaving
     DELIMITER = u'\ue175\uee31\u2c32\uacbf'
     BUFFERED_ACTIONS = set(['test_status', 'log'])
     VALID_ACTIONS = set(['suite_start', 'suite_end', 'test_start', 'test_end',
                          'test_status', 'log',
                          'buffering_on', 'buffering_off'])
 
-    def __init__(self, logger, buffering=True, name='mochitest'):
+    def __init__(self, logger, buffering=True):
         self.logger = logger
         self.buffering = buffering
         self.tests_started = False
 
         # Message buffering
         self.buffered_messages = []
 
         # Failures reporting, after the end of the tests execution
@@ -215,17 +215,17 @@ class MessageLogger(object):
 def call(*args, **kwargs):
   """front-end function to mozprocess.ProcessHandler"""
   # TODO: upstream -> mozprocess
   # https://bugzilla.mozilla.org/show_bug.cgi?id=791383
   process = mozprocess.ProcessHandler(*args, **kwargs)
   process.run()
   return process.wait()
 
-def killPid(pid):
+def killPid(pid, log):
   # see also https://bugzilla.mozilla.org/show_bug.cgi?id=911249#c58
   try:
     os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
   except Exception, e:
     log.info("Failed to kill process %d: %s" % (pid, str(e)))
 
 if mozinfo.isWin:
   import ctypes, ctypes.wintypes, time, msvcrt
@@ -264,19 +264,20 @@ else:
 
 #######################
 # HTTP SERVER SUPPORT #
 #######################
 
 class MochitestServer(object):
   "Web server used to serve Mochitests, for closer fidelity to the real web."
 
-  def __init__(self, options):
+  def __init__(self, options, logger):
     if isinstance(options, optparse.Values):
       options = vars(options)
+    self._log = logger
     self._closeWhenDone = options['closeWhenDone']
     self._utilityPath = options['utilityPath']
     self._xrePath = options['xrePath']
     self._profileDir = options['profilePath']
     self.webServer = options['webServer']
     self.httpPort = options['httpPort']
     self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { "server" : self.webServer, "port" : self.httpPort }
     self.testPrefix = "'webapprt_'" if options.get('webapprtContent') else "undefined"
@@ -312,32 +313,32 @@ class MochitestServer(object):
                     "testPrefix" : self.testPrefix, "displayResults" : str(not self._closeWhenDone).lower() },
             "-f", os.path.join(SCRIPT_DIR, "server.js")]
 
     xpcshell = os.path.join(self._utilityPath,
                             "xpcshell" + mozinfo.info['bin_suffix'])
     command = [xpcshell] + args
     self._process = mozprocess.ProcessHandler(command, cwd=SCRIPT_DIR, env=env)
     self._process.run()
-    log.info("%s : launching %s" % (self.__class__.__name__, command))
+    self._log.info("%s : launching %s" % (self.__class__.__name__, command))
     pid = self._process.pid
-    log.info("runtests.py | Server pid: %d" % pid)
+    self._log.info("runtests.py | Server pid: %d" % pid)
 
   def ensureReady(self, timeout):
     assert timeout >= 0
 
     aliveFile = os.path.join(self._profileDir, "server_alive.txt")
     i = 0
     while i < timeout:
       if os.path.exists(aliveFile):
         break
       time.sleep(1)
       i += 1
     else:
-      log.error("TEST-UNEXPECTED-FAIL | runtests.py | Timed out while waiting for server startup.")
+      self._log.error("TEST-UNEXPECTED-FAIL | runtests.py | Timed out while waiting for server startup.")
       self.stop()
       sys.exit(1)
 
   def stop(self):
     try:
       with urllib2.urlopen(self.shutdownURL) as c:
         c.read()
 
@@ -351,20 +352,22 @@ class MochitestServer(object):
         # self._process.terminate()
         self._process.proc.terminate()
     except:
       self._process.kill()
 
 class WebSocketServer(object):
   "Class which encapsulates the mod_pywebsocket server"
 
-  def __init__(self, options, scriptdir, debuggerInfo=None):
+  def __init__(self, options, scriptdir, logger, debuggerInfo=None):
     self.port = options.webSocketPort
+    self.debuggerInfo = debuggerInfo
+    self._log = logger
     self._scriptdir = scriptdir
-    self.debuggerInfo = debuggerInfo
+
 
   def start(self):
     # Invoke pywebsocket through a wrapper which adds special SIGINT handling.
     #
     # If we're in an interactive debugger, the wrapper causes the server to
     # ignore SIGINT so the server doesn't capture a ctrl+c meant for the
     # debugger.
     #
@@ -378,17 +381,17 @@ class WebSocketServer(object):
         cmd += ['--interactive']
     cmd += ['-p', str(self.port), '-w', self._scriptdir, '-l',      \
            os.path.join(self._scriptdir, "websock.log"),            \
            '--log-level=debug', '--allow-handlers-outside-root-dir']
     # start the process
     self._process = mozprocess.ProcessHandler(cmd, cwd=SCRIPT_DIR)
     self._process.run()
     pid = self._process.pid
-    log.info("runtests.py | Websocket server pid: %d" % pid)
+    self._log.info("runtests.py | Websocket server pid: %d" % pid)
 
   def stop(self):
     self._process.kill()
 
 class MochitestUtilsMixin(object):
   """
   Class containing some utility functions common to both local and remote
   mochitest runners
@@ -401,23 +404,38 @@ class MochitestUtilsMixin(object):
 
   oldcwd = os.getcwd()
   jarDir = 'mochijar'
 
   # Path to the test script on the server
   TEST_PATH = "tests"
   CHROME_PATH = "redirect.html"
   urlOpts = []
+  structured_logger = None
 
-  def __init__(self):
+  def __init__(self, logger_options):
     self.update_mozinfo()
     self.server = None
     self.wsserver = None
     self.sslTunnel = None
     self._locations = None
+    # Structured logger
+    if self.structured_logger is None:
+        self.structured_logger = setup_logging('mochitest', logger_options, {})
+        # Add the tbpl logger if no handler is logging to stdout, to display formatted logs by default
+        has_stdout_logger = any(h.stream == sys.stdout for h in self.structured_logger.handlers)
+        if not has_stdout_logger:
+            handler = StreamHandler(sys.stdout, MochitestFormatter())
+            self.structured_logger.add_handler(handler)
+        MochitestUtilsMixin.structured_logger = self.structured_logger
+
+    self.message_logger = MessageLogger(logger=self.structured_logger)
+
+    # self.log should also be structured_logger, but to avoid regressions like bug 1044206 we're now logging with the stdlib's logger
+    self.log = log
 
   def update_mozinfo(self):
     """walk up directories to find mozinfo.json update the info"""
     # TODO: This should go in a more generic place, e.g. mozinfo
 
     path = SCRIPT_DIR
     dirs = set()
     while path != os.path.expanduser('~'):
@@ -623,23 +641,23 @@ class MochitestUtilsMixin(object):
     with open(os.path.join(SCRIPT_DIR, 'tests.json'), 'w') as manifestFile:
       manifestFile.write(json.dumps({'tests': paths}))
     options.manifestFile = 'tests.json'
 
     return self.buildTestURL(options)
 
   def startWebSocketServer(self, options, debuggerInfo):
     """ Launch the websocket server """
-    self.wsserver = WebSocketServer(options, SCRIPT_DIR, debuggerInfo)
+    self.wsserver = WebSocketServer(options, SCRIPT_DIR, self.log, debuggerInfo)
     self.wsserver.start()
 
   def startWebServer(self, options):
     """Create the webserver and start it up"""
 
-    self.server = MochitestServer(options)
+    self.server = MochitestServer(options, self.log)
     self.server.start()
 
     if options.pidFile != "":
       with open(options.pidFile + ".xpcshell.pid", 'w') as f:
         f.write("%s" % self.server._process.pid)
 
   def startServers(self, options, debuggerInfo):
     # start servers and set ports
@@ -654,62 +672,62 @@ class MochitestUtilsMixin(object):
     # specified, try to select the one from hostutils.zip, as required in bug 882932.
     if not options.httpdPath:
       options.httpdPath = os.path.join(options.utilityPath, "components")
 
     self.startWebServer(options)
     self.startWebSocketServer(options, debuggerInfo)
 
     # start SSL pipe
-    self.sslTunnel = SSLTunnel(options)
+    self.sslTunnel = SSLTunnel(options, logger=self.log)
     self.sslTunnel.buildConfig(self.locations)
     self.sslTunnel.start()
 
     # If we're lucky, the server has fully started by now, and all paths are
     # ready, etc.  However, xpcshell cold start times suck, at least for debug
     # builds.  We'll try to connect to the server for awhile, and if we fail,
     # we'll try to kill the server and exit with an error.
     if self.server is not None:
       self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
 
   def stopServers(self):
     """Servers are no longer needed, and perhaps more importantly, anything they
         might spew to console might confuse things."""
     if self.server is not None:
       try:
-        log.info('Stopping web server')
+        self.log.info('Stopping web server')
         self.server.stop()
       except Exception:
-        log.critical('Exception when stopping web server')
+        self.log.critical('Exception when stopping web server')
 
     if self.wsserver is not None:
       try:
-        log.info('Stopping web socket server')
+        self.log.info('Stopping web socket server')
         self.wsserver.stop()
       except Exception:
-        log.critical('Exception when stopping web socket server');
+        self.log.critical('Exception when stopping web socket server');
 
     if self.sslTunnel is not None:
       try:
-        log.info('Stopping ssltunnel')
+        self.log.info('Stopping ssltunnel')
         self.sslTunnel.stop()
       except Exception:
-        log.critical('Exception stopping ssltunnel');
+        self.log.critical('Exception stopping ssltunnel');
 
   def copyExtraFilesToProfile(self, options):
     "Copy extra files or dirs specified on the command line to the testing profile."
     for f in options.extraProfileFiles:
       abspath = self.getFullPath(f)
       if os.path.isfile(abspath):
         shutil.copy2(abspath, options.profilePath)
       elif os.path.isdir(abspath):
         dest = os.path.join(options.profilePath, os.path.basename(abspath))
         shutil.copytree(abspath, dest)
       else:
-        log.warning("runtests.py | Failed to copy %s to profile" % abspath)
+        self.log.warning("runtests.py | Failed to copy %s to profile" % abspath)
 
   def installChromeJar(self, chrome, options):
     """
       copy mochijar directory to profile as an extension so we have chrome://mochikit for all harness code
     """
     # Write chrome.manifest.
     with open(os.path.join(options.profilePath, "extensions", "staged", "mochikit@mozilla.org", "chrome.manifest"), "a") as mfile:
       mfile.write(chrome)
@@ -744,17 +762,17 @@ toolbar#nav-bar {
       manifestFile.write("content mochitests %s contentaccessible=yes\n" % chrometestDir)
 
       if options.testingModulesDir is not None:
         manifestFile.write("resource testing-common file:///%s\n" %
           options.testingModulesDir)
 
     # Call installChromeJar().
     if not os.path.isdir(os.path.join(SCRIPT_DIR, self.jarDir)):
-      log.error(message="TEST-UNEXPECTED-FAIL | invalid setup: missing mochikit extension")
+      self.log.error("TEST-UNEXPECTED-FAIL | invalid setup: missing mochikit extension")
       return None
 
     # Support Firefox (browser), B2G (shell), SeaMonkey (navigator), and Webapp
     # Runtime (webapp).
     chrome = ""
     if options.browserChrome or options.chrome or options.a11y or options.webapprtChrome:
       chrome += """
 overlay chrome://browser/content/browser.xul chrome://mochikit/content/browser-test-overlay.xul
@@ -787,17 +805,18 @@ overlay chrome://webapprt/content/webapp
             if os.path.isdir(path) or (os.path.isfile(path) and path.endswith(".xpi")):
               extensions.append(path)
 
     # append mochikit
     extensions.append(os.path.join(SCRIPT_DIR, self.jarDir))
     return extensions
 
 class SSLTunnel:
-  def __init__(self, options):
+  def __init__(self, options, logger):
+    self.log = logger
     self.process = None
     self.utilityPath = options.utilityPath
     self.xrePath = options.xrePath
     self.certPath = options.certPath
     self.sslPort = options.sslPort
     self.httpPort = options.httpPort
     self.webServer = options.webServer
     self.webSocketPort = options.webSocketPort
@@ -842,25 +861,25 @@ class SSLTunnel:
 
   def start(self):
     """ Starts the SSL Tunnel """
 
     # start ssltunnel to provide https:// URLs capability
     bin_suffix = mozinfo.info.get('bin_suffix', '')
     ssltunnel = os.path.join(self.utilityPath, "ssltunnel" + bin_suffix)
     if not os.path.exists(ssltunnel):
-      log.error("INFO | runtests.py | expected to find ssltunnel at %s", ssltunnel)
+      self.log.error("INFO | runtests.py | expected to find ssltunnel at %s" % ssltunnel)
       exit(1)
 
     env = environment(xrePath=self.xrePath)
     env["LD_LIBRARY_PATH"] = self.xrePath
     self.process = mozprocess.ProcessHandler([ssltunnel, self.configFile],
                                                env=env)
     self.process.run()
-    log.info("runtests.py | SSL tunnel pid: %d" % self.process.pid)
+    self.log.info("runtests.py | SSL tunnel pid: %d" % self.process.pid)
 
   def stop(self):
     """ Stops the SSL Tunnel and cleans up """
     if self.process is not None:
       self.process.kill()
     if os.path.exists(self.configFile):
       os.remove(self.configFile)
 
@@ -919,17 +938,17 @@ def checkAndConfigureV4l2loopback(device
 
   control.id = SUSTAIN_FRAMERATE
   control.value = 1
   libc.ioctl(fd, VIDIOC_S_CTRL, ctypes.byref(control))
   libc.close(fd)
 
   return True, vcap.card
 
-def findTestMediaDevices():
+def findTestMediaDevices(log):
   '''
   Find the test media devices configured on this system, and return a dict
   containing information about them. The dict will have keys for 'audio'
   and 'video', each containing the name of the media device to use.
 
   If audio and video devices could not be found, return None.
 
   This method is only currently implemented for Linux.
@@ -981,26 +1000,18 @@ class Mochitest(MochitestUtilsMixin):
   vmwareHelper = None
   DEFAULT_TIMEOUT = 60.0
   mediaDevices = None
 
   # XXX use automation.py for test name to avoid breaking legacy
   # TODO: replace this with 'runtests.py' or 'mochitest' or the like
   test_name = 'automation.py'
 
-  def __init__(self):
-    super(Mochitest, self).__init__()
-
-    # Structured logger
-    structured_log = StructuredLogger('mochitest')
-    stream_handler = StreamHandler(stream=sys.stdout, formatter=MochitestFormatter())
-    structured_log.add_handler(stream_handler)
-
-    # Structured logs parser
-    self.message_logger = MessageLogger(logger=structured_log)
+  def __init__(self, logger_options):
+    super(Mochitest, self).__init__(logger_options)
 
     # environment function for browserEnv
     self.environment = environment
 
     # Max time in seconds to wait for server startup before tests will fail -- if
     # this seems big, it's mostly for debug machines where cold startup
     # (particularly after a build) takes forever.
     self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get('debug') else 90
@@ -1135,17 +1146,17 @@ class Mochitest(MochitestUtilsMixin):
 
     manifest = self.addChromeToProfile(options)
     self.copyExtraFilesToProfile(options)
 
     # create certificate database for the profile
     # TODO: this should really be upstreamed somewhere, maybe mozprofile
     certificateStatus = self.fillCertificateDB(options)
     if certificateStatus:
-      log.error("TEST-UNEXPECTED-FAIL | runtests.py | Certificate integration failed")
+      self.log.error("TEST-UNEXPECTED-FAIL | runtests.py | Certificate integration failed")
       return None
 
     return manifest
 
   def getGMPPluginPath(self, options):
     # For local builds, gmp-fake will be under dist/bin.
     gmp_path = os.path.join(options.xrePath, 'gmp-fake')
     if os.path.isdir(gmp_path):
@@ -1171,17 +1182,17 @@ class Mochitest(MochitestUtilsMixin):
     # These variables are necessary for correct application startup; change
     # via the commandline at your own risk.
     browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
 
     # interpolate environment passed with options
     try:
       browserEnv.update(dict(parseKeyValue(options.environment, context='--setenv')))
     except KeyValueParseError, e:
-      log.error(str(e))
+      self.log.error(str(e))
       return None
 
     browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
 
     try:
       gmp_path = self.getGMPPluginPath(options)
       if gmp_path is not None:
           browserEnv["MOZ_GMP_PATH"] = gmp_path
@@ -1212,22 +1223,22 @@ class Mochitest(MochitestUtilsMixin):
       os.remove(self.manifest)
     del self.profile
     if options.pidFile != "":
       try:
         os.remove(options.pidFile)
         if os.path.exists(options.pidFile + ".xpcshell.pid"):
           os.remove(options.pidFile + ".xpcshell.pid")
       except:
-        log.warning("cleaning up pidfile '%s' was unsuccessful from the test harness" % options.pidFile)
+        self.log.warning("cleaning up pidfile '%s' was unsuccessful from the test harness" % options.pidFile)
     options.manifestFile = None
 
   def dumpScreen(self, utilityPath):
     if self.haveDumpedScreen:
-      log.info("Not taking screenshot here: see the one that was previously logged")
+      self.log.info("Not taking screenshot here: see the one that was previously logged")
       return
     self.haveDumpedScreen = True
     dumpScreen(utilityPath)
 
   def killAndGetStack(self, processPID, utilityPath, debuggerInfo, dump_screen=False):
     """
     Kill the process, preferrably in a way that gets us a stack trace.
     Also attempts to obtain a screenshot before killing the process
@@ -1246,79 +1257,79 @@ class Mochitest(MochitestUtilsMixin):
           printstatus(status, "crashinject")
           if status == 0:
             return
       else:
         try:
           os.kill(processPID, signal.SIGABRT)
         except OSError:
           # https://bugzilla.mozilla.org/show_bug.cgi?id=921509
-          log.info("Can't trigger Breakpad, process no longer exists")
+          self.log.info("Can't trigger Breakpad, process no longer exists")
         return
-    log.info("Can't trigger Breakpad, just killing process")
-    killPid(processPID)
+    self.log.info("Can't trigger Breakpad, just killing process")
+    killPid(processPID, self.log)
 
   def checkForZombies(self, processLog, utilityPath, debuggerInfo):
     """Look for hung processes"""
 
     if not os.path.exists(processLog):
-      log.info('Automation Error: PID log not found: %s' % processLog)
+      self.log.info('Automation Error: PID log not found: %s' % processLog)
       # Whilst no hung process was found, the run should still display as a failure
       return True
 
     # scan processLog for zombies
-    log.info('zombiecheck | Reading PID log: %s' % processLog)
+    self.log.info('zombiecheck | Reading PID log: %s' % processLog)
     processList = []
     pidRE = re.compile(r'launched child process (\d+)$')
     with open(processLog) as processLogFD:
       for line in processLogFD:
-        log.info(line.rstrip())
+        self.log.info(line.rstrip())
         m = pidRE.search(line)
         if m:
           processList.append(int(m.group(1)))
 
     # kill zombies
     foundZombie = False
     for processPID in processList:
-      log.info("zombiecheck | Checking for orphan process with PID: %d" % processPID)
+      self.log.info("zombiecheck | Checking for orphan process with PID: %d" % processPID)
       if isPidAlive(processPID):
         foundZombie = True
-        log.error("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown" % processPID)
+        self.log.error("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown" % processPID)
         self.killAndGetStack(processPID, utilityPath, debuggerInfo, dump_screen=not debuggerInfo)
 
     return foundZombie
 
   def startVMwareRecording(self, options):
     """ starts recording inside VMware VM using the recording helper dll """
     assert mozinfo.isWin
     from ctypes import cdll
     self.vmwareHelper = cdll.LoadLibrary(self.vmwareHelperPath)
     if self.vmwareHelper is None:
-      log.warning("runtests.py | Failed to load "
+      self.log.warning("runtests.py | Failed to load "
                    "VMware recording helper")
       return
-    log.info("runtests.py | Starting VMware recording.")
+    self.log.info("runtests.py | Starting VMware recording.")
     try:
       self.vmwareHelper.StartRecording()
     except Exception, e:
-      log.warning("runtests.py | Failed to start "
+      self.log.warning("runtests.py | Failed to start "
                   "VMware recording: (%s)" % str(e))
       self.vmwareHelper = None
 
   def stopVMwareRecording(self):
     """ stops recording inside VMware VM using the recording helper dll """
     try:
       assert mozinfo.isWin
       if self.vmwareHelper is not None:
-        log.info("runtests.py | Stopping VMware recording.")
+        self.log.info("runtests.py | Stopping VMware recording.")
         self.vmwareHelper.StopRecording()
     except Exception, e:
-      log.warning("runtests.py | Failed to stop "
+      self.log.warning("runtests.py | Failed to stop "
                   "VMware recording: (%s)" % str(e))
-      log.exception('Error stopping VMWare recording')
+      self.log.exception('Error stopping VMWare recording')
 
     self.vmwareHelper = None
 
   def runApp(self,
              testUrl,
              env,
              app,
              profile,
@@ -1379,22 +1390,22 @@ class Mochitest(MochitestUtilsMixin):
       # https://bugzilla.mozilla.org/show_bug.cgi?id=916512
       args.append('-foreground')
       if testUrl:
         if debuggerInfo and debuggerInfo['requiresEscapedArgs']:
           testUrl = testUrl.replace("&", "\\&")
         args.append(testUrl)
 
       if detectShutdownLeaks:
-        shutdownLeaks = ShutdownLeaks(log.info)
+        shutdownLeaks = ShutdownLeaks(self.log.info)
       else:
         shutdownLeaks = None
 
       if mozinfo.info["asan"] and (mozinfo.isLinux or mozinfo.isMac):
-        lsanLeaks = LSANLeaks(log.info)
+        lsanLeaks = LSANLeaks(self.log.info)
       else:
         lsanLeaks = None
 
       # create an instance to process the output
       outputHandler = self.OutputHandler(harness=self,
                                          utilityPath=utilityPath,
                                          symbolsPath=symbolsPath,
                                          dump_screen_on_timeout=not debuggerInfo,
@@ -1429,17 +1440,17 @@ class Mochitest(MochitestUtilsMixin):
                           process_class=mozprocess.ProcessHandlerMixin,
                           process_args=kp_kwargs)
 
       # start the runner
       runner.start(debug_args=debug_args,
                    interactive=interactive,
                    outputTimeout=timeout)
       proc = runner.process_handler
-      log.info("runtests.py | Application pid: %d" % proc.pid)
+      self.log.info("runtests.py | Application pid: %d" % proc.pid)
 
       if onLaunch is not None:
         # Allow callers to specify an onLaunch callback to be fired after the
         # app is launched.
         # We call onLaunch for b2g desktop mochitests so that we can
         # run a Marionette script after gecko has completed startup.
         onLaunch()
 
@@ -1458,21 +1469,21 @@ class Mochitest(MochitestUtilsMixin):
         didTimeout = proc.didTimeout
 
       # finalize output handler
       outputHandler.finish(didTimeout)
 
       # record post-test information
       if status:
         self.message_logger.dump_buffered()
-        log.error("TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s" % (self.lastTestSeen, status))
+        self.log.error("TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s" % (self.lastTestSeen, status))
       else:
         self.lastTestSeen = 'Main app process exited normally'
 
-      log.info("runtests.py | Application ran for: %s" % str(datetime.now() - startTime))
+      self.log.info("runtests.py | Application ran for: %s" % str(datetime.now() - startTime))
 
       # Do a final check for zombie child processes.
       zombieProcesses = self.checkForZombies(processLog, utilityPath, debuggerInfo)
 
       # check for crashes
       minidump_path = os.path.join(self.profile.profile, "minidumps")
       crashed = mozcrash.check_for_crashes(minidump_path,
                                            symbolsPath,
@@ -1537,17 +1548,17 @@ class Mochitest(MochitestUtilsMixin):
       assert pathAbs.startswith(self.testRootAbs)
       tp = pathAbs[len(self.testRootAbs):].replace('\\', '/').strip('/')
 
       # Filter out tests if we are using --test-path
       if testPath and not tp.startswith(testPath):
         continue
 
       if not self.isTest(options, tp):
-        log.warning('Warning: %s from manifest %s is not a valid test' % (test['name'], test['manifest']))
+        self.log.warning('Warning: %s from manifest %s is not a valid test' % (test['name'], test['manifest']))
         continue
 
       testob = {'path': tp}
       if test.has_key('disabled'):
         testob['disabled'] = test['disabled']
       paths.append(testob)
 
     def path_sort(ob1, ob2):
@@ -1562,17 +1573,17 @@ class Mochitest(MochitestUtilsMixin):
   def getTestsToRun(self, options):
     """
       This method makes a list of tests that are to be run. Required mainly for --bisect-chunk.
     """
     tests = self.getActiveTests(options)
     testsToRun = []
     for test in tests:
       if test.has_key('disabled'):
-        log.info('TEST-SKIPPED | %s | %s' % (test['path'], test['disabled']))
+        self.log.info('TEST-SKIPPED | %s | %s' % (test['path'], test['disabled']))
         continue
       testsToRun.append(test['path'])
 
     return testsToRun
 
   def runMochitests(self, options, onLaunch=None):
     "This is a base method for calling other methods in this class for --bisect-chunk."
     testsToRun = self.getTestsToRun(options)
@@ -1667,19 +1678,19 @@ class Mochitest(MochitestUtilsMixin):
     # TODO: use mozrunner.local.debugger_arguments:
     # https://github.com/mozilla/mozbase/blob/master/mozrunner/mozrunner/local.py#L42
     debuggerInfo = getDebuggerInfo(self.oldcwd,
                                    options.debugger,
                                    options.debuggerArgs,
                                    options.debuggerInteractive)
 
     if options.useTestMediaDevices:
-      devices = findTestMediaDevices()
+      devices = findTestMediaDevices(self.log)
       if not devices:
-        log.error("Could not find test media devices to use")
+        self.log.error("Could not find test media devices to use")
         return 1
       self.mediaDevices = devices
 
     # buildProfile sets self.profile .
     # This relies on sideeffects and isn't very stateful:
     # https://bugzilla.mozilla.org/show_bug.cgi?id=919300
     self.manifest = self.buildProfile(options)
     if self.manifest is None:
@@ -1736,17 +1747,17 @@ class Mochitest(MochitestUtilsMixin):
         timeout = 330.0 # default JS harness timeout is 300 seconds
 
       if options.vmwareRecording:
         self.startVMwareRecording(options);
 
       # detect shutdown leaks for m-bc runs
       detectShutdownLeaks = mozinfo.info["debug"] and options.browserChrome and not options.webapprtChrome
 
-      log.info("runtests.py | Running tests: start.\n")
+      self.log.info("runtests.py | Running tests: start.\n")
       try:
         status = self.runApp(testURL,
                              self.browserEnv,
                              options.app,
                              profile=self.profile,
                              extraArgs=options.browserArgs,
                              utilityPath=options.utilityPath,
                              debuggerInfo=debuggerInfo,
@@ -1755,54 +1766,54 @@ class Mochitest(MochitestUtilsMixin):
                              onLaunch=onLaunch,
                              detectShutdownLeaks=detectShutdownLeaks,
                              screenshotOnFail=options.screenshotOnFail,
                              testPath=options.testPath,
                              bisectChunk=options.bisectChunk,
                              quiet=options.quiet
         )
       except KeyboardInterrupt:
-        log.info("runtests.py | Received keyboard interrupt.\n");
+        self.log.info("runtests.py | Received keyboard interrupt.\n");
         status = -1
       except:
         traceback.print_exc()
-        log.error("Automation Error: Received unexpected exception while running application\n")
+        self.log.error("Automation Error: Received unexpected exception while running application\n")
         status = 1
 
     finally:
       if options.vmwareRecording:
         self.stopVMwareRecording();
       self.stopServers()
 
     processLeakLog(self.leak_report_file, options.leakThreshold)
 
     if self.nsprLogs:
       with zipfile.ZipFile("%s/nsprlog.zip" % browserEnv["MOZ_UPLOAD_DIR"], "w", zipfile.ZIP_DEFLATED) as logzip:
         for logfile in glob.glob("%s/nspr*.log*" % tempfile.gettempdir()):
           logzip.write(logfile)
           os.remove(logfile)
 
-    log.info("runtests.py | Running tests: end.")
+    self.log.info("runtests.py | Running tests: end.")
 
     if self.manifest is not None:
       self.cleanup(options)
 
     return status
 
   def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo, browserProcessId, testPath=None):
     """handle process output timeout"""
     # TODO: bug 913975 : _processOutput should call self.processOutputLine one more time one timeout (I think)
     if testPath:
       error_message = "TEST-UNEXPECTED-TIMEOUT | %s | application timed out after %d seconds with no output on %s" % (self.lastTestSeen, int(timeout), testPath)
     else:
       error_message = "TEST-UNEXPECTED-TIMEOUT | %s | application timed out after %d seconds with no output" % (self.lastTestSeen, int(timeout))
 
     self.message_logger.dump_buffered()
     self.message_logger.buffering = False
-    log.error(error_message)
+    self.log.error(error_message)
 
     browserProcessId = browserProcessId or proc.pid
     self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo, dump_screen=not debuggerInfo)
 
 
 
   class OutputHandler(object):
     """line output handler for mozrunner"""
@@ -1899,17 +1910,17 @@ class Mochitest(MochitestUtilsMixin):
 
       return (stackFixerFunction, stackFixerProcess)
 
     def finish(self, didTimeout):
       if self.stackFixerProcess:
         self.stackFixerProcess.communicate()
         status = self.stackFixerProcess.returncode
         if status and not didTimeout:
-          log.info("TEST-UNEXPECTED-FAIL | runtests.py | Stack fixer process exited with code %d during test run" % status)
+          self.harness.log.info("TEST-UNEXPECTED-FAIL | runtests.py | Stack fixer process exited with code %d during test run" % status)
 
       if self.shutdownLeaks:
         self.shutdownLeaks.process()
 
       if self.lsanLeaks:
         self.lsanLeaks.process()
 
     # output message handlers:
@@ -2063,25 +2074,26 @@ class Mochitest(MochitestUtilsMixin):
     for test in tests:
       rootdir = '/'.join(test['path'].split('/')[:-1])
       if rootdir not in dirlist:
         dirlist.append(rootdir)
 
     return dirlist
 
 def main():
-
   # parse command line options
-  mochitest = Mochitest()
   parser = MochitestOptions()
+  add_logging_group(parser)
   options, args = parser.parse_args()
-  options = parser.verifyOptions(options, mochitest)
   if options is None:
     # parsing error
     sys.exit(1)
+  logger_options = {key: value for key, value in vars(options).iteritems() if key.startswith('log')}
+  mochitest = Mochitest(logger_options)
+  options = parser.verifyOptions(options, mochitest)
 
   options.utilityPath = mochitest.getFullPath(options.utilityPath)
   options.certPath = mochitest.getFullPath(options.certPath)
   if options.symbolsPath and not isURL(options.symbolsPath):
     options.symbolsPath = mochitest.getFullPath(options.symbolsPath)
 
   return_code = mochitest.runTests(options)
   mochitest.message_logger.finish()
--- a/testing/mochitest/runtestsb2g.py
+++ b/testing/mochitest/runtestsb2g.py
@@ -11,49 +11,40 @@ import tempfile
 import threading
 import traceback
 
 here = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, here)
 
 from runtests import Mochitest
 from runtests import MochitestUtilsMixin
-from runtests import MessageLogger
-from runtests import MochitestFormatter
 from mochitest_options import B2GOptions, MochitestOptions
 from marionette import Marionette
 from mozprofile import Profile, Preferences
+from mozlog import structured
 import mozinfo
-from mozlog.structured.handlers import StreamHandler
-from mozlog.structured.structuredlog import StructuredLogger
-
-log = StructuredLogger('Mochitest')
-stream_handler = StreamHandler(stream=sys.stdout, formatter=MochitestFormatter())
-log.add_handler(stream_handler)
 
 class B2GMochitest(MochitestUtilsMixin):
     marionette = None
 
     def __init__(self, marionette_args,
+                       logger_options,
                        out_of_process=True,
                        profile_data_dir=None,
                        locations=os.path.join(here, 'server-locations.txt')):
-        super(B2GMochitest, self).__init__()
+        super(B2GMochitest, self).__init__(logger_options)
         self.marionette_args = marionette_args
         self.out_of_process = out_of_process
         self.locations_file = locations
         self.preferences = []
         self.webapps = None
         self.test_script = os.path.join(here, 'b2g_start_script.js')
         self.test_script_args = [self.out_of_process]
         self.product = 'b2g'
 
-        # structured logging
-        self.message_logger = MessageLogger(logger=log)
-
         if profile_data_dir:
             self.preferences = [os.path.join(profile_data_dir, f)
                                  for f in os.listdir(profile_data_dir) if f.startswith('pref')]
             self.webapps = [os.path.join(profile_data_dir, f)
                              for f in os.listdir(profile_data_dir) if f.startswith('webapp')]
 
         # mozinfo is populated by the parent class
         if mozinfo.info['debug']:
@@ -131,17 +122,17 @@ class B2GMochitest(MochitestUtilsMixin):
         else:
             if not options.timeout:
                 if mozinfo.info['debug']:
                     options.timeout = 420
                 else:
                     options.timeout = 300
             timeout = options.timeout + 30.0
 
-        log.info("runtestsb2g.py | Running tests: start.")
+        self.log.info("runtestsb2g.py | Running tests: start.")
         status = 0
         try:
             def on_output(line):
                 messages = self.message_logger.write(line)
                 for message in messages:
                     if message['action'] == 'test_start':
                         self.runner.last_test = message['test']
 
@@ -189,44 +180,44 @@ class B2GMochitest(MochitestUtilsMixin):
                 self.marionette.execute_script(self.test_script,
                                                script_args=self.test_script_args)
             status = self.runner.wait()
 
             if status is None:
                 # the runner has timed out
                 status = 124
         except KeyboardInterrupt:
-            log.info("runtests.py | Received keyboard interrupt.\n");
+            self.log.info("runtests.py | Received keyboard interrupt.\n");
             status = -1
         except:
             traceback.print_exc()
-            log.error("Automation Error: Received unexpected exception while running application\n")
+            self.log.error("Automation Error: Received unexpected exception while running application\n")
             if hasattr(self, 'runner'):
                 self.runner.check_for_crashes()
             status = 1
 
         self.stopServers()
 
-        log.info("runtestsb2g.py | Running tests: end.")
+        self.log.info("runtestsb2g.py | Running tests: end.")
 
         if manifest is not None:
             self.cleanup(manifest, options)
         return status
 
     def getGMPPluginPath(self, options):
         # TODO: bug 1043403
         return None
 
 
 class B2GDeviceMochitest(B2GMochitest, Mochitest):
     remote_log = None
 
-    def __init__(self, marionette_args, profile_data_dir,
+    def __init__(self, marionette_args, logger_options, profile_data_dir,
                  local_binary_dir, remote_test_root=None, remote_log_file=None):
-        B2GMochitest.__init__(self, marionette_args, out_of_process=True, profile_data_dir=profile_data_dir)
+        B2GMochitest.__init__(self, marionette_args, logger_options, out_of_process=True, profile_data_dir=profile_data_dir)
         self.local_log = None
         self.local_binary_dir = local_binary_dir
 
     def cleanup(self, manifest, options):
         if self.local_log:
             self.app_ctx.dm.getFile(self.remote_log, self.local_log)
             self.app_ctx.dm.removeFile(self.remote_log)
 
@@ -270,19 +261,19 @@ class B2GDeviceMochitest(B2GMochitest, M
         self.setup_common_options(options)
 
         options.profilePath = self.app_ctx.remote_profile
         options.logFile = self.local_log
 
 
 class B2GDesktopMochitest(B2GMochitest, Mochitest):
 
-    def __init__(self, marionette_args, profile_data_dir):
-        B2GMochitest.__init__(self, marionette_args, out_of_process=False, profile_data_dir=profile_data_dir)
-        Mochitest.__init__(self)
+    def __init__(self, marionette_args, logger_options, profile_data_dir):
+        B2GMochitest.__init__(self, marionette_args, logger_options, out_of_process=False, profile_data_dir=profile_data_dir)
+        Mochitest.__init__(self, logger_options)
         self.certdbNew = True
 
     def runMarionetteScript(self, marionette, test_script, test_script_args):
         assert(marionette.wait_for_port())
         marionette.start_session()
         marionette.set_context(marionette.CONTEXT_CHROME)
 
         if os.path.isfile(test_script):
@@ -339,18 +330,18 @@ def run_remote_mochitests(parser, option
         marionette_args['host'] = host
         marionette_args['port'] = int(port)
 
     options = parser.verifyRemoteOptions(options)
     if (options == None):
         print "ERROR: Invalid options specified, use --help for a list of valid options"
         sys.exit(1)
 
-    mochitest = B2GDeviceMochitest(marionette_args, options.profile_data_dir, options.xrePath,
-                                   remote_log_file=options.remoteLogFile)
+    mochitest = B2GDeviceMochitest(marionette_args, options, options.profile_data_dir,
+                                   options.xrePath, remote_log_file=options.remoteLogFile)
 
     options = parser.verifyOptions(options, mochitest)
     if (options == None):
         sys.exit(1)
 
     retVal = 1
     try:
         mochitest.cleanup(None, options)
@@ -371,39 +362,40 @@ def run_remote_mochitests(parser, option
 
 def run_desktop_mochitests(parser, options):
     # create our Marionette instance
     marionette_args = {}
     if options.marionette:
         host, port = options.marionette.split(':')
         marionette_args['host'] = host
         marionette_args['port'] = int(port)
-    mochitest = B2GDesktopMochitest(marionette_args, options.profile_data_dir)
 
     # add a -bin suffix if b2g-bin exists, but just b2g was specified
     if options.app[-4:] != '-bin':
         if os.path.isfile("%s-bin" % options.app):
             options.app = "%s-bin" % options.app
 
+    mochitest = B2GDesktopMochitest(marionette_args, options, options.profile_data_dir)
     options = MochitestOptions.verifyOptions(parser, options, mochitest)
     if options == None:
         sys.exit(1)
 
     if options.desktop and not options.profile:
         raise Exception("must specify --profile when specifying --desktop")
 
     options.browserArgs += ['-marionette']
 
     retVal = mochitest.runTests(options, onLaunch=mochitest.startTests)
     mochitest.message_logger.finish()
 
     sys.exit(retVal)
 
 def main():
     parser = B2GOptions()
+    structured.commandline.add_logging_group(parser)
     options, args = parser.parse_args()
 
     if options.desktop:
         run_desktop_mochitests(parser, options)
     else:
         run_remote_mochitests(parser, options)
 
 if __name__ == "__main__":
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -1,40 +1,36 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import base64
 import json
+import logging
 import math
 import os
 import re
 import shutil
 import sys
 import tempfile
 import traceback
 
 sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(__file__))))
 
 from automation import Automation
 from remoteautomation import RemoteAutomation, fennecLogcatFilters
-from runtests import Mochitest, MessageLogger, MochitestFormatter
+from runtests import Mochitest, MessageLogger
 from mochitest_options import MochitestOptions
+from mozlog import structured
 
 import devicemanager
 import droid
 import manifestparser
 import mozinfo
 import moznetwork
-from mozlog.structured.handlers import StreamHandler
-from mozlog.structured.structuredlog import StructuredLogger
-
-log = StructuredLogger('Mochi-Remote')
-stream_handler = StreamHandler(stream=sys.stdout, formatter=MochitestFormatter())
-log.add_handler(stream_handler)
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 
 class RemoteOptions(MochitestOptions):
 
     def __init__(self, automation, **kwargs):
         defaults = {}
         self._automation = automation or Automation()
@@ -120,89 +116,91 @@ class RemoteOptions(MochitestOptions):
         defaults["closeWhenDone"] = True
         defaults["testPath"] = ""
         defaults["app"] = None
         defaults["utilityPath"] = None
 
         self.set_defaults(**defaults)
 
     def verifyRemoteOptions(self, options, automation):
+        options_logger = logging.getLogger('MochitestRemote')
+
         if not options.remoteTestRoot:
             options.remoteTestRoot = automation._devicemanager.deviceRoot
 
         if options.remoteWebServer == None:
             if os.name != "nt":
                 options.remoteWebServer = moznetwork.get_ip()
             else:
-                log.error("you must specify a --remote-webserver=<ip address>")
+                options_logger.error("you must specify a --remote-webserver=<ip address>")
                 return None
 
         options.webServer = options.remoteWebServer
 
         if (options.deviceIP == None):
-            log.error("you must provide a device IP")
+            options_logger.error("you must provide a device IP")
             return None
 
         if (options.remoteLogFile == None):
             options.remoteLogFile = options.remoteTestRoot + '/logs/mochitest.log'
 
         if (options.remoteLogFile.count('/') < 1):
             options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
 
         # remoteAppPath or app must be specified to find the product to launch
         if (options.remoteAppPath and options.app):
-            log.error("You cannot specify both the remoteAppPath and the app setting")
+            options_logger.error("You cannot specify both the remoteAppPath and the app setting")
             return None
         elif (options.remoteAppPath):
             options.app = options.remoteTestRoot + "/" + options.remoteAppPath
         elif (options.app == None):
             # Neither remoteAppPath nor app are set -- error
-            log.error("You must specify either appPath or app")
+            options_logger.error("You must specify either appPath or app")
             return None
 
         # Only reset the xrePath if it wasn't provided
         if (options.xrePath == None):
             options.xrePath = options.utilityPath
 
         if (options.pidFile != ""):
             f = open(options.pidFile, 'w')
             f.write("%s" % os.getpid())
             f.close()
 
         # Robocop specific deprecated options.
         if options.robocop:
             if options.robocopIni:
-                log.error("can not use deprecated --robocop and replacement --robocop-ini together")
+                options_logger.error("can not use deprecated --robocop and replacement --robocop-ini together")
                 return None
             options.robocopIni = options.robocop
             del options.robocop
 
         if options.robocopPath:
             if options.robocopApk:
-                log.error("can not use deprecated --robocop-path and replacement --robocop-apk together")
+                options_logger.error("can not use deprecated --robocop-path and replacement --robocop-apk together")
                 return None
             options.robocopApk = os.path.join(options.robocopPath, 'robocop.apk')
             del options.robocopPath
 
         # Robocop specific options
         if options.robocopIni != "":
             if not os.path.exists(options.robocopIni):
-                log.error("Unable to find specified robocop .ini manifest '%s'" % options.robocopIni)
+                options_logger.error("Unable to find specified robocop .ini manifest '%s'" % options.robocopIni)
                 return None
             options.robocopIni = os.path.abspath(options.robocopIni)
 
         if options.robocopApk != "":
             if not os.path.exists(options.robocopApk):
-                log.error("Unable to find robocop APK '%s'" % options.robocopApk)
+                options_logger.error("Unable to find robocop APK '%s'" % options.robocopApk)
                 return None
             options.robocopApk = os.path.abspath(options.robocopApk)
 
         if options.robocopIds != "":
             if not os.path.exists(options.robocopIds):
-                log.error("Unable to find specified robocop IDs file '%s'" % options.robocopIds)
+                options_logger.error("Unable to find specified robocop IDs file '%s'" % options.robocopIds)
                 return None
             options.robocopIds = os.path.abspath(options.robocopIds)
 
         # allow us to keep original application around for cleanup while running robocop via 'am'
         options.remoteappname = options.app
         return options
 
     def verifyOptions(self, options, mochitest):
@@ -225,37 +223,35 @@ class RemoteOptions(MochitestOptions):
 
 class MochiRemote(Mochitest):
 
     _automation = None
     _dm = None
     localProfile = None
     logMessages = []
 
-    def __init__(self, automation, devmgr, options, message_logger=None):
+    def __init__(self, automation, devmgr, options):
+        Mochitest.__init__(self, options)
+
         self._automation = automation
-        Mochitest.__init__(self)
         self._dm = devmgr
         self.environment = self._automation.environment
         self.remoteProfile = options.remoteTestRoot + "/profile"
         self._automation.setRemoteProfile(self.remoteProfile)
         self.remoteLog = options.remoteLogFile
         self.localLog = options.logFile
         self._automation.deleteANRs()
         self.certdbNew = True
 
-        # structured logging
-        self.message_logger = message_logger or MessageLogger(logger=log)
-
     def cleanup(self, options):
         if self._dm.fileExists(self.remoteLog):
             self._dm.getFile(self.remoteLog, self.localLog)
             self._dm.removeFile(self.remoteLog)
         else:
-            log.warning("Unable to retrieve log file (%s) from remote device" % self.remoteLog)
+            self.log.warning("Unable to retrieve log file (%s) from remote device" % self.remoteLog)
         self._dm.removeDir(self.remoteProfile)
         Mochitest.cleanup(self, options)
 
     def findPath(self, paths, filename = None):
         for path in paths:
             p = path
             if filename:
                 p = os.path.join(p, filename)
@@ -295,36 +291,36 @@ class MochiRemote(Mochitest):
         paths = [
             options.xrePath,
             localAutomation.DIST_BIN,
             self._automation._product,
             os.path.join('..', self._automation._product)
         ]
         options.xrePath = self.findPath(paths)
         if options.xrePath == None:
-            log.error("unable to find xulrunner path for %s, please specify with --xre-path" % os.name)
+            self.log.error("unable to find xulrunner path for %s, please specify with --xre-path" % os.name)
             sys.exit(1)
 
         xpcshell = "xpcshell"
         if (os.name == "nt"):
             xpcshell += ".exe"
 
         if options.utilityPath:
             paths = [options.utilityPath, options.xrePath]
         else:
             paths = [options.xrePath]
         options.utilityPath = self.findPath(paths, xpcshell)
 
         if options.utilityPath == None:
-            log.error("unable to find utility path for %s, please specify with --utility-path" % os.name)
+            self.log.error("unable to find utility path for %s, please specify with --utility-path" % os.name)
             sys.exit(1)
 
         xpcshell_path = os.path.join(options.utilityPath, xpcshell)
         if localAutomation.elf_arm(xpcshell_path):
-            log.error('xpcshell at %s is an ARM binary; please use '
+            self.log.error('xpcshell at %s is an ARM binary; please use '
                       'the --utility-path argument to specify the path '
                       'to a desktop version.' % xpcshell_path)
             sys.exit(1)
 
         if self.localProfile:
             options.profilePath = self.localProfile
         else:
             options.profilePath = None
@@ -354,17 +350,17 @@ class MochiRemote(Mochitest):
             shutil.rmtree(os.path.join(options.profilePath, 'extensions', 'staged', 'mochikit@mozilla.org'))
             shutil.rmtree(os.path.join(options.profilePath, 'extensions', 'staged', 'worker-test@mozilla.org'))
             shutil.rmtree(os.path.join(options.profilePath, 'extensions', 'staged', 'workerbootstrap-test@mozilla.org'))
             os.remove(os.path.join(options.profilePath, 'userChrome.css'))
 
         try:
             self._dm.pushDir(options.profilePath, self.remoteProfile)
         except devicemanager.DMError:
-            log.error("Automation Error: Unable to copy profile to device.")
+            self.log.error("Automation Error: Unable to copy profile to device.")
             raise
 
         restoreRemotePaths()
         options.profilePath = self.remoteProfile
         return manifest
 
     def buildURLOptions(self, options, env):
         self.localLog = options.logFile
@@ -373,17 +369,17 @@ class MochiRemote(Mochitest):
         env["MOZ_HIDE_RESULTS_TABLE"] = "1"
         retVal = Mochitest.buildURLOptions(self, options, env)
 
         if not options.robocopIni:
             #we really need testConfig.js (for browser chrome)
             try:
                 self._dm.pushDir(options.profilePath, self.remoteProfile)
             except devicemanager.DMError:
-                log.error("Automation Error: Unable to copy profile to device.")
+                self.log.error("Automation Error: Unable to copy profile to device.")
                 raise
 
         options.profilePath = self.remoteProfile
         options.logFile = self.localLog
         return retVal
 
     def getTestsToRun(self, options):
         if options.robocopIni != "":
@@ -404,17 +400,17 @@ class MochiRemote(Mochitest):
         parts = options.app.split('/')
         if (parts[0] == options.app):
           return "NO_CHROME_ON_DROID"
         path = '/'.join(parts[:-1])
         manifest = path + "/chrome/" + os.path.basename(filename)
         try:
             self._dm.pushFile(filename, manifest)
         except devicemanager.DMError:
-            log.error("Automation Error: Unable to install Chrome files on device.")
+            self.log.error("Automation Error: Unable to install Chrome files on device.")
             raise
 
         return manifest
 
     def getLogFilePath(self, logFile):
         return logFile
 
     # In the future we could use LogParser: http://hg.mozilla.org/automation/logparser/
@@ -443,17 +439,17 @@ class MochiRemote(Mochitest):
             if message['action'] == 'test_start':
                 start_found = True
             if 'expected' in message:
                 fail_found = True
         result = 0
         if fail_found:
             result = 1
         if not end_found:
-            log.error("Automation Error: Missing end of test marker (process crashed?)")
+            self.log.error("Automation Error: Missing end of test marker (process crashed?)")
             result = 1
         return result
 
     def printLog(self):
         passed = 0
         failed = 0
         todo = 0
         incr = 1
@@ -488,43 +484,43 @@ class MochiRemote(Mochitest):
 
         if failed > 0:
             return 1
         return 0
 
     def printScreenshots(self, screenShotDir):
         # TODO: This can be re-written after completion of bug 749421
         if not self._dm.dirExists(screenShotDir):
-            log.info("SCREENSHOT: No ScreenShots directory available: " + screenShotDir)
+            self.log.info("SCREENSHOT: No ScreenShots directory available: " + screenShotDir)
             return
 
         printed = 0
         for name in self._dm.listFiles(screenShotDir):
             fullName = screenShotDir + "/" + name
-            log.info("SCREENSHOT: FOUND: [%s]" % fullName)
+            self.log.info("SCREENSHOT: FOUND: [%s]" % fullName)
             try:
                 image = self._dm.pullFile(fullName)
                 encoded = base64.b64encode(image)
-                log.info("SCREENSHOT: data:image/jpg;base64,%s" % encoded)
+                self.log.info("SCREENSHOT: data:image/jpg;base64,%s" % encoded)
                 printed += 1
             except:
-                log.info("SCREENSHOT: Could not be parsed")
+                self.log.info("SCREENSHOT: Could not be parsed")
                 pass
 
-        log.info("SCREENSHOT: TOTAL PRINTED: [%s]" % printed)
+        self.log.info("SCREENSHOT: TOTAL PRINTED: [%s]" % printed)
 
     def printDeviceInfo(self, printLogcat=False):
         try:
             if printLogcat:
                 logcat = self._dm.getLogcat(filterOutRegexps=fennecLogcatFilters)
-                log.info('\n' + ''.join(logcat).decode('utf-8', 'replace'))
-            log.info("Device info: %s" % self._dm.getInfo())
-            log.info("Test root: %s" % self._dm.deviceRoot)
+                self.log.info('\n' + ''.join(logcat).decode('utf-8', 'replace'))
+            self.log.info("Device info: %s" % self._dm.getInfo())
+            self.log.info("Test root: %s" % self._dm.deviceRoot)
         except devicemanager.DMError:
-            log.warn("Error getting device information")
+            self.log.warning("Error getting device information")
 
     def buildRobotiumConfig(self, options, browserEnv):
         deviceRoot = self._dm.deviceRoot
         fHandle = tempfile.NamedTemporaryFile(suffix='.config',
                                               prefix='robotium-',
                                               dir=os.getcwd(),
                                               delete=False)
         fHandle.write("profile=%s\n" % (self.remoteProfile))
@@ -533,18 +529,18 @@ class MochiRemote(Mochitest):
         fHandle.write("rawhost=http://%s:%s/tests\n" % (options.remoteWebServer, options.httpPort))
 
         if browserEnv:
             envstr = ""
             delim = ""
             for key, value in browserEnv.items():
                 try:
                     value.index(',')
-                    log.error("buildRobotiumConfig: browserEnv - Found a ',' in our value, unable to process value. key=%s,value=%s" % (key, value))
-                    log.error("browserEnv=%s" % browserEnv)
+                    self.log.error("buildRobotiumConfig: browserEnv - Found a ',' in our value, unable to process value. key=%s,value=%s" % (key, value))
+                    self.log.error("browserEnv=%s" % browserEnv)
                 except ValueError:
                     envstr += "%s%s=%s" % (delim, key, value)
                     delim = ","
 
             fHandle.write("envvars=%s\n" % envstr)
         fHandle.close()
 
         self._dm.removeFile(os.path.join(deviceRoot, "robotium.config"))
@@ -573,44 +569,52 @@ class MochiRemote(Mochitest):
         kwargs['runSSLTunnel'] = False
 
         if 'quiet' in kwargs:
             kwargs.pop('quiet')
 
         return self._automation.runApp(*args, **kwargs)
 
 def main():
-    message_logger = MessageLogger(logger=log)
+    message_logger = MessageLogger(logger=None)
     process_args = {'messageLogger': message_logger}
     auto = RemoteAutomation(None, "fennec", processArgs=process_args)
+
     parser = RemoteOptions(auto)
+    structured.commandline.add_logging_group(parser)
     options, args = parser.parse_args()
 
     if (options.dm_trans == "adb"):
         if (options.deviceIP):
             dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)
         else:
             dm = droid.DroidADB(deviceRoot=options.remoteTestRoot)
     else:
          dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)
     auto.setDeviceManager(dm)
     options = parser.verifyRemoteOptions(options, auto)
+
+    mochitest = MochiRemote(auto, dm, options)
+
+    log = mochitest.log
+    structured_logger = mochitest.structured_logger
+    message_logger.logger = mochitest.structured_logger
+    mochitest.message_logger = message_logger
+
     if (options == None):
         log.error("Invalid options specified, use --help for a list of valid options")
         sys.exit(1)
 
     productPieces = options.remoteProductName.split('.')
     if (productPieces != None):
         auto.setProduct(productPieces[0])
     else:
         auto.setProduct(options.remoteProductName)
     auto.setAppName(options.remoteappname)
 
-    mochitest = MochiRemote(auto, dm, options, message_logger)
-
     options = parser.verifyOptions(options, mochitest)
     if (options == None):
         sys.exit(1)
 
     logParent = os.path.dirname(options.remoteLogFile)
     dm.mkDir(logParent);
     auto.setRemoteLog(options.remoteLogFile)
     auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)
@@ -686,17 +690,17 @@ def main():
                 continue
 
             if 'disabled' in test:
                 log.info('TEST-INFO | skipping %s | %s' % (test['name'], test['disabled']))
                 continue
 
             active_tests.append(test)
 
-        log.suite_start([t['name'] for t in active_tests])
+        structured_logger.suite_start([t['name'] for t in active_tests])
 
         for test in active_tests:
             # When running in a loop, we need to create a fresh profile for each cycle
             if mochitest.localProfile:
                 options.profilePath = mochitest.localProfile
                 os.system("rm -Rf %s" % options.profilePath)
                 options.profilePath = None
                 mochitest.localProfile = options.profilePath
--- a/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
+++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
@@ -230,17 +230,17 @@ class DeviceManagerSUT(DeviceManager):
                     temp = ''
                     self._logger.debug("recv'ing...")
 
                     # Get our response
                     try:
                         # Wait up to a second for socket to become ready for reading...
                         if select.select([self._sock], [], [], select_timeout)[0]:
                             temp = self._sock.recv(1024)
-                            self._logger.debug("response: %s" % temp)
+                            self._logger.debug(u"response: %s" % temp.decode('utf8', 'replace'))
                             timer = 0
                             if not temp:
                                 socketClosed = True
                                 errStr = 'connection closed'
                         timer += select_timeout
                         if timer > timeout:
                             self._sock.close()
                             self._sock = None