Bug 909888 - Add mach commands for b2g reftests and crashtests, r=jgriffin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 29 Aug 2013 14:23:07 -0400
changeset 153037 6ec9bb4eedf8353cfd70265e3d63e1e266e99d59
parent 153036 0f939ca1606802376cd4193e8a96900e11f4fcc8
child 153038 545c76a2bf6f36eb4bfbb1daea05479fc1567553
push idunknown
push userunknown
push dateunknown
reviewersjgriffin
bugs909888
milestone26.0a1
Bug 909888 - Add mach commands for b2g reftests and crashtests, r=jgriffin
layout/tools/reftest/mach_commands.py
layout/tools/reftest/runreftestb2g.py
testing/mochitest/mach_commands.py
--- a/layout/tools/reftest/mach_commands.py
+++ b/layout/tools/reftest/mach_commands.py
@@ -2,38 +2,53 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import mozpack.path
 import os
 import re
+import sys
+import which
 
 from mozbuild.base import (
     MachCommandBase,
+    MachCommandConditions as conditions,
     MozbuildObject,
 )
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 
 DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'
 
+ADB_NOT_FOUND = """The %s command requires the adb binary to be on your path.
+This can be found in '%s/out/host/<platform>/bin'."""
 
 class ReftestRunner(MozbuildObject):
     """Easily run reftests.
 
     This currently contains just the basics for running reftests. We may want
     to hook up result parsing, etc.
     """
+    def __init__(self, *args, **kwargs):
+        MozbuildObject.__init__(self, *args, **kwargs)
+
+        # TODO Bug 794506 remove once mach integrates with virtualenv.
+        build_path = os.path.join(self.topobjdir, 'build')
+        if build_path not in sys.path:
+            sys.path.append(build_path)
+
+        self.tests_dir = os.path.join(self.topobjdir, '_tests')
+        self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
 
     def _manifest_file(self, suite):
         """Returns the manifest file used for a given test suite."""
         files = {
           'reftest': 'reftest.list',
           'reftest-ipc': 'reftest.list',
           'crashtest': 'crashtests.list',
           'crashtest-ipc': 'crashtests.list',
@@ -52,17 +67,82 @@ class ReftestRunner(MozbuildObject):
         if relpath.endswith('.list'):
             return relpath
 
         raise Exception('Running a single test is not currently supported')
 
     def _make_shell_string(self, s):
         return "'%s'" % re.sub("'", r"'\''", s)
 
-    def run_reftest_test(self, test_file=None, filter=None, suite=None,
+    def run_b2g_test(self, b2g_home, xre_path, test_file=None, suite=None, **kwargs):
+        """Runs a b2g reftest.
+
+        test_file is a path to a test file. It can be a relative path from the
+        top source directory, an absolute filename, or a directory containing
+        test files.
+
+        suite is the type of reftest to run. It can be one of ('reftest',
+        'crashtest').
+        """
+        if suite not in ('reftest', 'crashtest'):
+            raise Exception('None or unrecognized reftest suite type.')
+
+        try:
+            which.which('adb')
+        except which.WhichError:
+            # TODO Find adb automatically if it isn't on the path
+            raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))
+
+        # Find the manifest file
+        if not test_file:
+            if suite == 'reftest':
+                test_file = mozpack.path.join('layout', 'reftests')
+            elif suite == 'crashtest':
+                test_file = mozpack.path.join('testing', 'crashtest')
+
+        if not os.path.exists(os.path.join(self.topsrcdir, test_file)):
+            test_file = mozpack.path.relpath(os.path.abspath(test_file),
+                                             self.topsrcdir)
+
+        manifest = self._find_manifest(suite, test_file)
+        if not os.path.exists(mozpack.path.join(self.topsrcdir, manifest)):
+            raise Exception('No manifest file was found at %s.' % manifest)
+
+        # Need to chdir to reftest_dir otherwise imports fail below.
+        os.chdir(self.reftest_dir)
+
+        import imp
+        path = os.path.join(self.reftest_dir, 'runreftestb2g.py')
+        with open(path, 'r') as fh:
+            imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))
+        import reftest
+
+        # Set up the reftest options.
+        parser = reftest.B2GOptions()
+        options, args = parser.parse_args([])
+
+        options.b2gPath = b2g_home
+        options.logcat_dir = self.reftest_dir
+        options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
+        options.ignoreWindowSize = True
+        options.xrePath = xre_path
+
+        for k, v in kwargs.iteritems():
+            setattr(options, k, v)
+
+        # Tests need to be served from a subdirectory of the server. Symlink
+        # topsrcdir here to get around this.
+        tests = os.path.join(self.reftest_dir, 'tests')
+        if not os.path.isdir(tests):
+            os.symlink(self.topsrcdir, tests)
+        args.insert(0, os.path.join('tests', manifest))
+
+        return reftest.run_remote_reftests(parser, options, args)
+
+    def run_desktop_test(self, test_file=None, filter=None, suite=None,
             debugger=None):
         """Runs a reftest.
 
         test_file is a path to a test file. It can be a relative path from the
         top source directory, an absolute filename, or a directory containing
         test files.
 
         filter is a regular expression (in JS syntax, as could be passed to the
@@ -119,16 +199,57 @@ def ReftestCommand(func):
 
     path = CommandArgument('test_file', nargs='?', metavar='MANIFEST',
         help='Reftest manifest file, or a directory in which to select '
              'reftest.list. If omitted, the entire test suite is executed.')
     func = path(func)
 
     return func
 
+def B2GCommand(func):
+    """Decorator that adds shared command arguments to b2g mochitest commands."""
+
+    busybox = CommandArgument('--busybox', default=None,
+        help='Path to busybox binary to install on device')
+    func = busybox(func)
+
+    logcatdir = CommandArgument('--logcat-dir', default=None,
+        help='directory to store logcat dump files')
+    func = logcatdir(func)
+
+    geckopath = CommandArgument('--gecko-path', default=None,
+        help='the path to a gecko distribution that should \
+              be installed on the emulator prior to test')
+    func = geckopath(func)
+
+    sdcard = CommandArgument('--sdcard', default="10MB",
+        help='Define size of sdcard: 1MB, 50MB...etc')
+    func = sdcard(func)
+
+    emulator_res = CommandArgument('--emulator-res', default='800x1000',
+        help='Emulator resolution of the format \'<width>x<height>\'')
+    func = emulator_res(func)
+
+    emulator = CommandArgument('--emulator', default='arm',
+        help='Architecture of emulator to use: x86 or arm')
+    func = emulator(func)
+
+    marionette = CommandArgument('--marionette', default=None,
+        help='host:port to use when connecting to Marionette')
+    func = marionette(func)
+
+    path = CommandArgument('test_file', default=None, nargs='?',
+        metavar='TEST',
+        help='Test to run. Can be specified as a single file, a ' \
+            'directory, or omitted. If omitted, the entire test suite is ' \
+            'executed.')
+    func = path(func)
+
+    return func
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('reftest', category='testing', description='Run reftests.')
     @ReftestCommand
     def run_reftest(self, test_file, **kwargs):
         return self._run_reftest(test_file, suite='reftest', **kwargs)
 
@@ -145,15 +266,46 @@ class MachCommands(MachCommandBase):
         return self._run_reftest(test_file, suite='crashtest', **kwargs)
 
     @Command('crashtest-ipc', category='testing',
         description='Run IPC crashtests.')
     @ReftestCommand
     def run_crashtest_ipc(self, test_file, **kwargs):
         return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)
 
-    def _run_reftest(self, test_file=None, filter=None, suite=None,
-            debugger=None):
+    def _run_reftest(self, test_file=None, suite=None, **kwargs):
         reftest = self._spawn(ReftestRunner)
-        return reftest.run_reftest_test(test_file, filter=filter, suite=suite,
-            debugger=debugger)
+        return reftest.run_desktop_test(test_file, suite=suite, **kwargs)
+
+
+# TODO For now b2g commands will only work with the emulator,
+# they should be modified to work with all devices.
+def is_emulator(cls):
+    """Emulator needs to be configured."""
+    return cls.device_name in ('emulator', 'emulator-jb')
 
 
+@CommandProvider
+class B2GCommands(MachCommandBase):
+    def __init__(self, context):
+        MachCommandBase.__init__(self, context)
+
+        for attr in ('b2g_home', 'xre_path', 'device_name'):
+            setattr(self, attr, getattr(context, attr, None))
+
+    @Command('reftest-remote', category='testing',
+        description='Run a remote reftest.',
+        conditions=[conditions.is_b2g, is_emulator])
+    @B2GCommand
+    def run_reftest_remote(self, test_file, **kwargs):
+        return self._run_reftest(test_file, suite='reftest', **kwargs)
+
+    @Command('crashtest-remote', category='testing',
+        description='Run a remote crashtest.',
+        conditions=[conditions.is_b2g, is_emulator])
+    @B2GCommand
+    def run_crashtest_remote(self, test_file, **kwargs):
+        return self._run_reftest(test_file, suite='crashtest', **kwargs)
+
+    def _run_reftest(self, test_file=None, suite=None, **kwargs):
+        reftest = self._spawn(ReftestRunner)
+        return reftest.run_b2g_test(self.b2g_home, self.xre_path,
+            test_file, suite=suite, **kwargs)
--- a/layout/tools/reftest/runreftestb2g.py
+++ b/layout/tools/reftest/runreftestb2g.py
@@ -19,18 +19,21 @@ from runreftest import ReftestOptions
 from remotereftest import ReftestServer
 
 from mozdevice import DeviceManagerADB, DMError
 from marionette import Marionette
 
 
 class B2GOptions(ReftestOptions):
 
-    def __init__(self, automation, **kwargs):
+    def __init__(self, automation=None, **kwargs):
         defaults = {}
+        if not automation:
+            automation = B2GRemoteAutomation(None, "fennec", context_chrome=True)
+
         ReftestOptions.__init__(self, automation)
 
         self.add_option("--b2gpath", action="store",
                     type = "string", dest = "b2gPath",
                     help = "path to B2G repo or qemu dir")
         defaults["b2gPath"] = None
 
         self.add_option("--marionette", action="store",
@@ -465,21 +468,18 @@ class B2GReftest(RefTest):
             self._devicemanager.pushDir(profileDir, options.remoteProfile)
         except DMError:
             print "Automation Error: Failed to copy extra files to device"
             raise
 
     def getManifestPath(self, path):
         return path
 
-
-def main(args=sys.argv[1:]):
+def run_remote_reftests(parser, options, args):
     auto = B2GRemoteAutomation(None, "fennec", context_chrome=True)
-    parser = B2GOptions(auto)
-    options, args = parser.parse_args(args)
 
     # create our Marionette instance
     kwargs = {}
     if options.emulator:
         kwargs['emulator'] = options.emulator
         auto.setEmulator(True)
         if options.noWindow:
             kwargs['noWindow'] = True
@@ -503,16 +503,17 @@ def main(args=sys.argv[1:]):
     auto.marionette = marionette
 
     # create the DeviceManager
     kwargs = {'adbPath': options.adbPath,
               'deviceRoot': options.remoteTestRoot}
     if options.deviceIP:
         kwargs.update({'host': options.deviceIP,
                        'port': options.devicePort})
+
     dm = DeviceManagerADB(**kwargs)
     auto.setDeviceManager(dm)
 
     options = parser.verifyRemoteOptions(options)
 
     if (options == None):
         print "ERROR: Invalid options specified, use --help for a list of valid options"
         sys.exit(1)
@@ -576,11 +577,17 @@ def main(args=sys.argv[1:]):
             reftest.cleanup(None)
         except:
             pass
         return 1
 
     reftest.stopWebServer(options)
     return retVal
 
+def main(args=sys.argv[1:]):
+    parser = B2GOptions()
+    options, args = parser.parse_args(args)
+    return run_remote_reftests(parser, options, args)
+
+
 if __name__ == "__main__":
     sys.exit(main())
 
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -4,31 +4,34 @@
 
 from __future__ import unicode_literals
 
 import logging
 import mozpack.path
 import os
 import platform
 import sys
+import which
 
 from mozbuild.base import (
     MachCommandBase,
     MachCommandConditions as conditions,
     MozbuildObject,
 )
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 from mach.logging import StructuredHumanFormatter
 
+ADB_NOT_FOUND = """The %s command requires the adb binary to be on your path.
+This can be found in '%s/out/host/<platform>/bin'."""
 
 class UnexpectedFilter(logging.Filter):
     def filter(self, record):
         msg = getattr(record, 'params', {}).get('msg', '')
         return 'TEST-UNEXPECTED-' in msg
 
 
 class MochitestRunner(MozbuildObject):
@@ -51,16 +54,22 @@ class MochitestRunner(MozbuildObject):
 
     def run_b2g_test(self, b2g_home, xre_path, test_file=None, **kwargs):
         """Runs a b2g mochitest.
 
         test_file is a path to a test file. It can be a relative path from the
         top source directory, an absolute filename, or a directory containing
         test files.
         """
+        try:
+            which.which('adb')
+        except which.WhichError:
+            # TODO Find adb automatically if it isn't on the path
+            raise Exception(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))
+
         # TODO without os.chdir, chained imports fail below
         os.chdir(self.mochitest_dir)
 
         import imp
         path = os.path.join(self.mochitest_dir, 'runtestsb2g.py')
         with open(path, 'r') as fh:
             imp.load_module('mochitest', fh, path,
                 ('.py', 'r', imp.PY_SOURCE))