Bug 1228636 - Add mach support for running reftests on mulet, r=jgriffin draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 04 Jan 2016 16:27:37 -0500
changeset 319290 52a0e293f43e90054a459d3e89c978e5452417fa
parent 318640 0771c5eab32f0cee4f7d12bc382298a81e0eabb2
child 512568 8d52838540285f9c3be0edbf49837d0b8e90517f
push id9000
push userahalberstadt@mozilla.com
push dateWed, 06 Jan 2016 14:45:40 +0000
reviewersjgriffin
bugs1228636
milestone46.0a1
Bug 1228636 - Add mach support for running reftests on mulet, r=jgriffin Usage: mach reftest --profile path/to/gaia/profile path/to/test/dir
layout/tools/reftest/mach_commands.py
layout/tools/reftest/runreftestb2g.py
layout/tools/reftest/runreftestmulet.py
--- a/layout/tools/reftest/mach_commands.py
+++ b/layout/tools/reftest/mach_commands.py
@@ -28,30 +28,30 @@ import reftestcommandline
 ADB_NOT_FOUND = '''
 The %s command requires the adb binary to be on your path.
 
 If you have a B2G build, this can be found in
 '%s/out/host/<platform>/bin'.
 '''.lstrip()
 
 GAIA_PROFILE_NOT_FOUND = '''
-The %s command requires a non-debug gaia profile. Either pass in --profile,
-or set the GAIA_PROFILE environment variable.
+The reftest command requires a non-debug gaia profile on Mulet.
+Either pass in --profile, or set the GAIA_PROFILE environment variable.
 
 If you do not have a non-debug gaia profile, you can build one:
     $ git clone https://github.com/mozilla-b2g/gaia
     $ cd gaia
     $ make
 
 The profile should be generated in a directory called 'profile'.
 '''.lstrip()
 
 GAIA_PROFILE_IS_DEBUG = '''
-The %s command requires a non-debug gaia profile. The specified profile,
-%s, is a debug profile.
+The reftest command requires a non-debug gaia profile on Mulet.
+The specified profile, %s, is a debug profile.
 
 If you do not have a non-debug gaia profile, you can build one:
     $ git clone https://github.com/mozilla-b2g/gaia
     $ cd gaia
     $ make
 
 The profile should be generated in a directory called 'profile'.
 '''.lstrip()
@@ -78,77 +78,113 @@ class ReftestRunner(MozbuildObject):
             sys.path.append(build_path)
 
         self.tests_dir = os.path.join(self.topobjdir, '_tests')
         self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
 
     def _make_shell_string(self, s):
         return "'%s'" % re.sub("'", r"'\''", s)
 
+    def _setup_objdir(self, **kwargs):
+        # reftest imports will happen from the objdir
+        sys.path.insert(0, self.reftest_dir)
+
+        if not kwargs["tests"]:
+            test_subdir = {
+                "reftest": os.path.join('layout', 'reftests'),
+                "crashtest": os.path.join('layout', 'crashtest'),
+            }[kwargs['suite']]
+
+            if not os.path.exists(os.path.join(self.topsrcdir, test_subdir)):
+                test_file = mozpath.relpath(os.path.abspath(test_subdir),
+                                            self.topsrcdir)
+            kwargs["tests"] = [test_subdir]
+
+        tests = os.path.join(self.reftest_dir, 'tests')
+        if not os.path.isdir(tests):
+            os.symlink(self.topsrcdir, tests)
+
     def run_b2g_test(self, b2g_home=None, xre_path=None, **kwargs):
         """Runs a b2g reftest.
 
         filter is a regular expression (in JS syntax, as could be passed to the
         RegExp constructor) to select which reftests to run from the manifest.
 
         test_file is a path to a test file. It can be a relative path from the
         top source directory, an absolute filename, or a directory containing
         test files.
 
         suite is the type of reftest to run. It can be one of ('reftest',
         'crashtest').
         """
         if kwargs["suite"] not in ('reftest', 'crashtest'):
             raise Exception('None or unrecognized reftest suite type.')
 
-        sys.path.insert(0, self.reftest_dir)
-
-        test_subdir = {"reftest": os.path.join('layout', 'reftests'),
-                       "crashtest": os.path.join('layout', 'crashtest')}[kwargs["suite"]]
-
-        # Find the manifest file
-        if not kwargs["tests"]:
-            if not os.path.exists(os.path.join(self.topsrcdir, test_subdir)):
-                test_file = mozpath.relpath(os.path.abspath(test_subdir),
-                                            self.topsrcdir)
-            kwargs["tests"] = [test_subdir]
-
-        tests = os.path.join(self.reftest_dir, 'tests')
-        if not os.path.isdir(tests):
-            os.symlink(self.topsrcdir, tests)
+        self._setup_objdir(**kwargs)
+        import runreftestb2g
 
         for i, path in enumerate(kwargs["tests"]):
             # Non-absolute paths are relative to the packaged directory, which
             # has an extra tests/ at the start
             if os.path.exists(os.path.abspath(path)):
                 path = os.path.relpath(path, os.path.join(self.topsrcdir))
             kwargs["tests"][i] = os.path.join('tests', path)
 
-        return self.run_b2g_remote(b2g_home, xre_path, **kwargs)
-
-    def run_b2g_remote(self, b2g_home, xre_path, **kwargs):
-        import runreftestb2g
-
         try:
             which.which('adb')
         except which.WhichError:
             # TODO Find adb automatically if it isn't on the path
             raise Exception(ADB_NOT_FOUND % ('%s-remote' % kwargs["suite"], b2g_home))
 
         kwargs["b2gPath"] = b2g_home
         kwargs["logdir"] = self.reftest_dir
         kwargs["httpdPath"] = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
         kwargs["xrePath"] = xre_path
         kwargs["ignoreWindowSize"] = True
 
         # Don't enable oop for crashtest until they run oop in automation
         if kwargs["suite"] == 'reftest':
             kwargs["oop"] = True
 
-        return runreftestb2g.run_remote(**kwargs)
+        return runreftestb2g.run(**kwargs)
+
+    def run_mulet_test(self, **kwargs):
+        """Runs a mulet reftest."""
+        self._setup_objdir(**kwargs)
+        import runreftestmulet
+
+        if self.substs.get('ENABLE_MARIONETTE') != '1':
+            print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
+                                         self.mozconfig['path']))
+            return 1
+
+        if not kwargs["profile"]:
+            gaia_profile = os.environ.get('GAIA_PROFILE')
+            if not gaia_profile:
+                print(GAIA_PROFILE_NOT_FOUND)
+                return 1
+            kwargs["profile"] = gaia_profile
+
+        if os.path.isfile(os.path.join(kwargs["profile"], 'extensions',
+                                       'httpd@gaiamobile.org')):
+            print(GAIA_PROFILE_IS_DEBUG % kwargs["profile"])
+            return 1
+
+        kwargs['app'] = self.get_binary_path()
+        kwargs['mulet'] = True
+        kwargs['oop'] = True
+
+        if kwargs['oop']:
+            kwargs['browser_arg'] = '-oop'
+        if not kwargs['app'].endswith('-bin'):
+            kwargs['app'] = '%s-bin' % kwargs['app']
+        if not os.path.isfile(kwargs['app']):
+            options.app = kwargs['app'][:-len('-bin')]
+
+        return runreftestmulet.run(**kwargs)
 
     def run_desktop_test(self, **kwargs):
         """Runs a reftest, in desktop Firefox."""
         import runreftest
 
         if kwargs["suite"] not in ('reftest', 'crashtest', 'jstestbrowser'):
             raise Exception('None or unrecognized reftest suite type.')
 
@@ -271,16 +307,18 @@ def process_test_objects(kwargs):
         kwargs["tests"].extend(item["path"] for item in kwargs["test_objects"])
         del kwargs["test_objects"]
 
 def get_parser():
     here = os.path.abspath(os.path.dirname(__file__))
     build_obj = MozbuildObject.from_environment(cwd=here)
     if conditions.is_android(build_obj):
         return reftestcommandline.RemoteArgumentsParser()
+    elif conditions.is_mulet(build_obj):
+        return reftestcommandline.B2GArgumentParser()
     else:
         return reftestcommandline.DesktopArgumentsParser()
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('reftest',
              category='testing',
              description='Run reftests (layout and graphics correctness).',
@@ -328,16 +366,18 @@ class MachCommands(MachCommandBase):
 
     def _run_reftest(self, **kwargs):
         process_test_objects(kwargs)
         reftest = self._spawn(ReftestRunner)
         if conditions.is_android(self):
             from mozrunner.devices.android_device import verify_android_device
             verify_android_device(self, install=True, xre=True)
             return reftest.run_android_test(**kwargs)
+        elif conditions.is_mulet(self):
+            return reftest.run_mulet_test(**kwargs)
         return reftest.run_desktop_test(**kwargs)
 
 
 # TODO For now b2g commands will only work with the emulator,
 # they should be modified to work with all devices.
 def is_emulator(cls):
     """Emulator needs to be configured."""
     return cls.device_name.startswith('emulator')
--- a/layout/tools/reftest/runreftestb2g.py
+++ b/layout/tools/reftest/runreftestb2g.py
@@ -10,17 +10,17 @@ import traceback
 
 # We need to know our current directory so that we can serve our test files from it.
 here = os.path.abspath(os.path.dirname(__file__))
 if here not in sys.path:
     sys.path.insert(0, here)
 
 from automation import Automation
 from b2gautomation import B2GRemoteAutomation
-from runreftestmulet import run_test_harness as run_mulet
+from runreftestmulet import run_test_harness as run_mulet_reftests
 from remotereftest import RemoteReftestResolver, ReftestServer
 from runreftest import RefTest
 import reftestcommandline
 
 from mozdevice import DeviceManagerADB, DMError
 from marionette import Marionette
 
 class ProfileConfigParser(ConfigParser.RawConfigParser):
@@ -413,28 +413,29 @@ def run_remote_reftests(parser, options)
             reftest.cleanup(None)
         except:
             pass
         return 1
 
     reftest.stopWebServer(options)
     return retVal
 
-def run_remote(**kwargs):
-    # Tests need to be served from a subdirectory of the server. Symlink
-    # topsrcdir here to get around this.
+
+def run(**kwargs):
+    # Mach gives us kwargs; this is a way to turn them back into an
+    # options object
     parser = reftestcommandline.B2GArgumentParser()
     parser.set_defaults(**kwargs)
     options = parser.parse_args(kwargs["tests"])
     return run_remote_reftests(parser, options)
 
-def main():
+
+def main(args=sys.argv[1:]):
     parser = reftestcommandline.B2GArgumentParser()
-    options = parser.parse_args()
+    options = parser.parse_args(args)
 
     if options.mulet:
-        return run_mulet(parser, options)
+        return run_mulet_reftests(parser, options)
     return run_remote_reftests(parser, options)
 
 
 if __name__ == "__main__":
     sys.exit(main())
-
--- a/layout/tools/reftest/runreftestmulet.py
+++ b/layout/tools/reftest/runreftestmulet.py
@@ -4,28 +4,30 @@
 from __future__ import print_function, unicode_literals
 
 import os
 import signal
 import sys
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-from runreftest import RefTest
-
 from marionette_driver import expected
 from marionette_driver.by import By
 from marionette_driver.marionette import Marionette
 from marionette_driver.wait import Wait
 
 from mozprocess import ProcessHandler
 from mozrunner import FirefoxRunner
 import mozinfo
 import mozlog
 
+from runreftest import RefTest
+import reftestcommandline
+
+
 log = mozlog.unstructured.getLogger('REFTEST')
 
 class MuletReftest(RefTest):
     build_type = "mulet"
     marionette = None
 
     def __init__(self, marionette_args):
         RefTest.__init__(self)
@@ -202,8 +204,17 @@ def run_test_harness(parser, options):
 
     if options.xrePath is None:
         options.xrePath = os.path.dirname(options.app)
 
     if options.mulet and not options.profile:
         raise Exception("must specify --profile when specifying --mulet")
 
     sys.exit(reftest.run_tests(options.tests, options))
+
+
+def run(**kwargs):
+    # Mach gives us kwargs; this is a way to turn them back into an
+    # options object
+    parser = reftestcommandline.B2GArgumentParser()
+    parser.set_defaults(**kwargs)
+    options = parser.parse_args(kwargs['tests'])
+    return run_test_harness(parser, options)