Bug 1228636 - Add mach support for running reftests on mulet, r=jgriffin draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 04 Jan 2016 16:27:37 -0500
changeset 320046 6efffc77ef2b108617214c5ead35f8c4e90fcd3f
parent 318640 0771c5eab32f0cee4f7d12bc382298a81e0eabb2
child 512679 0ab176b06770dadca28b22e243cb1edf06202990
push id9125
push userahalberstadt@mozilla.com
push dateFri, 08 Jan 2016 14:43:15 +0000
reviewersjgriffin
bugs1228636
milestone46.0a1
Bug 1228636 - Add mach support for running reftests on mulet, r=jgriffin Usage: mach reftest --profile path/to/gaia/profile path/to/test/dir
layout/tools/reftest/mach_commands.py
layout/tools/reftest/runreftestb2g.py
layout/tools/reftest/runreftestmulet.py
--- a/layout/tools/reftest/mach_commands.py
+++ b/layout/tools/reftest/mach_commands.py
@@ -1,73 +1,72 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
-import mozpack.path as mozpath
 import os
 import re
 import sys
 import warnings
 import which
 
 from mozbuild.base import (
     MachCommandBase,
     MachCommandConditions as conditions,
     MozbuildObject,
 )
 
 from mach.decorators import (
-    CommandArgument,
     CommandProvider,
     Command,
 )
 
 import reftestcommandline
 
 ADB_NOT_FOUND = '''
 The %s command requires the adb binary to be on your path.
 
 If you have a B2G build, this can be found in
 '%s/out/host/<platform>/bin'.
 '''.lstrip()
 
 GAIA_PROFILE_NOT_FOUND = '''
-The %s command requires a non-debug gaia profile. Either pass in --profile,
-or set the GAIA_PROFILE environment variable.
+The reftest command requires a non-debug gaia profile on Mulet.
+Either pass in --profile, or set the GAIA_PROFILE environment variable.
 
 If you do not have a non-debug gaia profile, you can build one:
     $ git clone https://github.com/mozilla-b2g/gaia
     $ cd gaia
     $ make
 
 The profile should be generated in a directory called 'profile'.
 '''.lstrip()
 
 GAIA_PROFILE_IS_DEBUG = '''
-The %s command requires a non-debug gaia profile. The specified profile,
-%s, is a debug profile.
+The reftest command requires a non-debug gaia profile on Mulet.
+The specified profile, %s, is a debug profile.
 
 If you do not have a non-debug gaia profile, you can build one:
     $ git clone https://github.com/mozilla-b2g/gaia
     $ cd gaia
     $ make
 
 The profile should be generated in a directory called 'profile'.
 '''.lstrip()
 
 MARIONETTE_DISABLED = '''
-The %s command requires a marionette enabled build.
+The reftest command requires a marionette enabled build on Mulet.
 
 Add 'ENABLE_MARIONETTE=1' to your mozconfig file and re-build the application.
 Your currently active mozconfig is %s.
 '''.lstrip()
 
+
 class ReftestRunner(MozbuildObject):
     """Easily run reftests.
 
     This currently contains just the basics for running reftests. We may want
     to hook up result parsing, etc.
     """
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
@@ -78,77 +77,108 @@ class ReftestRunner(MozbuildObject):
             sys.path.append(build_path)
 
         self.tests_dir = os.path.join(self.topobjdir, '_tests')
         self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
 
     def _make_shell_string(self, s):
         return "'%s'" % re.sub("'", r"'\''", s)
 
+    def _setup_objdir(self, **kwargs):
+        # reftest imports will happen from the objdir
+        sys.path.insert(0, self.reftest_dir)
+
+        if not kwargs["tests"]:
+            test_subdir = {
+                "reftest": os.path.join('layout', 'reftests'),
+                "crashtest": os.path.join('layout', 'crashtest'),
+            }[kwargs['suite']]
+            kwargs["tests"] = [test_subdir]
+
+        tests = os.path.join(self.reftest_dir, 'tests')
+        if not os.path.isdir(tests):
+            os.symlink(self.topsrcdir, tests)
+
     def run_b2g_test(self, b2g_home=None, xre_path=None, **kwargs):
         """Runs a b2g reftest.
 
         filter is a regular expression (in JS syntax, as could be passed to the
         RegExp constructor) to select which reftests to run from the manifest.
 
-        test_file is a path to a test file. It can be a relative path from the
+        tests is a list of paths. It can be a relative path from the
         top source directory, an absolute filename, or a directory containing
         test files.
 
         suite is the type of reftest to run. It can be one of ('reftest',
         'crashtest').
         """
         if kwargs["suite"] not in ('reftest', 'crashtest'):
             raise Exception('None or unrecognized reftest suite type.')
 
-        sys.path.insert(0, self.reftest_dir)
-
-        test_subdir = {"reftest": os.path.join('layout', 'reftests'),
-                       "crashtest": os.path.join('layout', 'crashtest')}[kwargs["suite"]]
-
-        # Find the manifest file
-        if not kwargs["tests"]:
-            if not os.path.exists(os.path.join(self.topsrcdir, test_subdir)):
-                test_file = mozpath.relpath(os.path.abspath(test_subdir),
-                                            self.topsrcdir)
-            kwargs["tests"] = [test_subdir]
-
-        tests = os.path.join(self.reftest_dir, 'tests')
-        if not os.path.isdir(tests):
-            os.symlink(self.topsrcdir, tests)
+        self._setup_objdir(**kwargs)
+        import runreftestb2g
 
         for i, path in enumerate(kwargs["tests"]):
             # Non-absolute paths are relative to the packaged directory, which
             # has an extra tests/ at the start
             if os.path.exists(os.path.abspath(path)):
                 path = os.path.relpath(path, os.path.join(self.topsrcdir))
             kwargs["tests"][i] = os.path.join('tests', path)
 
-        return self.run_b2g_remote(b2g_home, xre_path, **kwargs)
-
-    def run_b2g_remote(self, b2g_home, xre_path, **kwargs):
-        import runreftestb2g
-
         try:
             which.which('adb')
         except which.WhichError:
             # TODO Find adb automatically if it isn't on the path
             raise Exception(ADB_NOT_FOUND % ('%s-remote' % kwargs["suite"], b2g_home))
 
         kwargs["b2gPath"] = b2g_home
         kwargs["logdir"] = self.reftest_dir
         kwargs["httpdPath"] = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
         kwargs["xrePath"] = xre_path
         kwargs["ignoreWindowSize"] = True
 
         # Don't enable oop for crashtest until they run oop in automation
         if kwargs["suite"] == 'reftest':
             kwargs["oop"] = True
 
-        return runreftestb2g.run_remote(**kwargs)
+        return runreftestb2g.run(**kwargs)
+
+    def run_mulet_test(self, **kwargs):
+        """Runs a mulet reftest."""
+        self._setup_objdir(**kwargs)
+        import runreftestmulet
+
+        if self.substs.get('ENABLE_MARIONETTE') != '1':
+            print(MARIONETTE_DISABLED % self.mozconfig['path'])
+            return 1
+
+        if not kwargs["profile"]:
+            gaia_profile = os.environ.get('GAIA_PROFILE')
+            if not gaia_profile:
+                print(GAIA_PROFILE_NOT_FOUND)
+                return 1
+            kwargs["profile"] = gaia_profile
+
+        if os.path.isfile(os.path.join(kwargs["profile"], 'extensions',
+                                       'httpd@gaiamobile.org')):
+            print(GAIA_PROFILE_IS_DEBUG % kwargs["profile"])
+            return 1
+
+        kwargs['app'] = self.get_binary_path()
+        kwargs['mulet'] = True
+        kwargs['oop'] = True
+
+        if kwargs['oop']:
+            kwargs['browser_arg'] = '-oop'
+        if not kwargs['app'].endswith('-bin'):
+            kwargs['app'] = '%s-bin' % kwargs['app']
+        if not os.path.isfile(kwargs['app']):
+            kwargs['app'] = kwargs['app'][:-len('-bin')]
+
+        return runreftestmulet.run(**kwargs)
 
     def run_desktop_test(self, **kwargs):
         """Runs a reftest, in desktop Firefox."""
         import runreftest
 
         if kwargs["suite"] not in ('reftest', 'crashtest', 'jstestbrowser'):
             raise Exception('None or unrecognized reftest suite type.')
 
@@ -168,17 +198,17 @@ class ReftestRunner(MozbuildObject):
         if kwargs["suite"] == "jstestbrowser":
             kwargs["extraProfileFiles"].append(os.path.join(self.topobjdir, "dist",
                                                             "test-stage", "jsreftest",
                                                             "tests", "user.js"))
 
         if not kwargs["runTestsInParallel"]:
             kwargs["logFile"] = "%s.log" % kwargs["suite"]
 
-        #Remove the stdout handler from the internal logger and let mach deal with it
+        # Remove the stdout handler from the internal logger and let mach deal with it
         runreftest.log.removeHandler(runreftest.log.handlers[0])
         self.log_manager.enable_unstructured()
         try:
             rv = runreftest.run(**kwargs)
         finally:
             self.log_manager.disable_unstructured()
 
         return rv
@@ -255,35 +285,40 @@ class ReftestRunner(MozbuildObject):
         self.log_manager.enable_unstructured()
         try:
             rv = reftest.run(**kwargs)
         finally:
             self.log_manager.disable_unstructured()
 
         return rv
 
+
 def process_test_objects(kwargs):
     """|mach test| works by providing a test_objects argument, from
     which the test path must be extracted and converted into a normal
     reftest tests argument."""
 
     if "test_objects" in kwargs:
         if kwargs["tests"] is None:
             kwargs["tests"] = []
         kwargs["tests"].extend(item["path"] for item in kwargs["test_objects"])
         del kwargs["test_objects"]
 
+
 def get_parser():
     here = os.path.abspath(os.path.dirname(__file__))
     build_obj = MozbuildObject.from_environment(cwd=here)
     if conditions.is_android(build_obj):
         return reftestcommandline.RemoteArgumentsParser()
+    elif conditions.is_mulet(build_obj):
+        return reftestcommandline.B2GArgumentParser()
     else:
         return reftestcommandline.DesktopArgumentsParser()
 
+
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('reftest',
              category='testing',
              description='Run reftests (layout and graphics correctness).',
              parser=get_parser)
     def run_reftest(self, **kwargs):
         kwargs["suite"] = "reftest"
@@ -328,16 +363,18 @@ class MachCommands(MachCommandBase):
 
     def _run_reftest(self, **kwargs):
         process_test_objects(kwargs)
         reftest = self._spawn(ReftestRunner)
         if conditions.is_android(self):
             from mozrunner.devices.android_device import verify_android_device
             verify_android_device(self, install=True, xre=True)
             return reftest.run_android_test(**kwargs)
+        elif conditions.is_mulet(self):
+            return reftest.run_mulet_test(**kwargs)
         return reftest.run_desktop_test(**kwargs)
 
 
 # TODO For now b2g commands will only work with the emulator,
 # they should be modified to work with all devices.
 def is_emulator(cls):
     """Emulator needs to be configured."""
     return cls.device_name.startswith('emulator')
--- a/layout/tools/reftest/runreftestb2g.py
+++ b/layout/tools/reftest/runreftestb2g.py
@@ -10,17 +10,17 @@ import traceback
 
 # We need to know our current directory so that we can serve our test files from it.
 here = os.path.abspath(os.path.dirname(__file__))
 if here not in sys.path:
     sys.path.insert(0, here)
 
 from automation import Automation
 from b2gautomation import B2GRemoteAutomation
-from runreftestmulet import run_test_harness as run_mulet
+from runreftestmulet import run_test_harness as run_mulet_reftests
 from remotereftest import RemoteReftestResolver, ReftestServer
 from runreftest import RefTest
 import reftestcommandline
 
 from mozdevice import DeviceManagerADB, DMError
 from marionette import Marionette
 
 class ProfileConfigParser(ConfigParser.RawConfigParser):
@@ -413,28 +413,29 @@ def run_remote_reftests(parser, options)
             reftest.cleanup(None)
         except:
             pass
         return 1
 
     reftest.stopWebServer(options)
     return retVal
 
-def run_remote(**kwargs):
-    # Tests need to be served from a subdirectory of the server. Symlink
-    # topsrcdir here to get around this.
+
+def run(**kwargs):
+    # Mach gives us kwargs; this is a way to turn them back into an
+    # options object
     parser = reftestcommandline.B2GArgumentParser()
     parser.set_defaults(**kwargs)
     options = parser.parse_args(kwargs["tests"])
     return run_remote_reftests(parser, options)
 
-def main():
+
+def main(args=sys.argv[1:]):
     parser = reftestcommandline.B2GArgumentParser()
-    options = parser.parse_args()
+    options = parser.parse_args(args)
 
     if options.mulet:
-        return run_mulet(parser, options)
+        return run_mulet_reftests(parser, options)
     return run_remote_reftests(parser, options)
 
 
 if __name__ == "__main__":
     sys.exit(main())
-
--- a/layout/tools/reftest/runreftestmulet.py
+++ b/layout/tools/reftest/runreftestmulet.py
@@ -4,28 +4,30 @@
 from __future__ import print_function, unicode_literals
 
 import os
 import signal
 import sys
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-from runreftest import RefTest
-
 from marionette_driver import expected
 from marionette_driver.by import By
 from marionette_driver.marionette import Marionette
 from marionette_driver.wait import Wait
 
 from mozprocess import ProcessHandler
 from mozrunner import FirefoxRunner
 import mozinfo
 import mozlog
 
+from runreftest import RefTest
+import reftestcommandline
+
+
 log = mozlog.unstructured.getLogger('REFTEST')
 
 class MuletReftest(RefTest):
     build_type = "mulet"
     marionette = None
 
     def __init__(self, marionette_args):
         RefTest.__init__(self)
@@ -202,8 +204,17 @@ def run_test_harness(parser, options):
 
     if options.xrePath is None:
         options.xrePath = os.path.dirname(options.app)
 
     if options.mulet and not options.profile:
         raise Exception("must specify --profile when specifying --mulet")
 
     sys.exit(reftest.run_tests(options.tests, options))
+
+
+def run(**kwargs):
+    # Mach gives us kwargs; this is a way to turn them back into an
+    # options object
+    parser = reftestcommandline.B2GArgumentParser()
+    parser.set_defaults(**kwargs)
+    options = parser.parse_args(kwargs['tests'])
+    return run_test_harness(parser, options)