Bug 790465 - Refactor Marionette testrunner to make it easier to write Gaia tests, r=mdas
authorJonathan Griffin <jgriffin@mozilla.com>
Fri, 14 Sep 2012 15:04:34 -0700
changeset 107348 8d96a5392ff4caa0e611a2efae13fa94df431330
parent 107347 ae2e45d25a1aab728a7e4a498db6b65c22d9e0ba
child 107349 fee9427c15c83da1412850e5f1593bd77c674f9e
push id74
push usershu@rfrn.org
push dateTue, 18 Sep 2012 19:23:47 +0000
reviewersmdas
bugs790465
milestone18.0a1
Bug 790465 - Refactor Marionette testrunner to make it easier to write Gaia tests, r=mdas
testing/marionette/client/marionette/__init__.py
testing/marionette/client/marionette/marionette_test.py
testing/marionette/client/marionette/runtests.py
testing/marionette/client/setup.py
--- a/testing/marionette/client/marionette/__init__.py
+++ b/testing/marionette/client/marionette/__init__.py
@@ -1,8 +1,9 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from marionette import Marionette, HTMLElement
-from marionette_test import MarionetteTestCase
+from marionette_test import MarionetteTestCase, CommonTestCase
 from emulator import Emulator
+from runtests import MarionetteTestRunner
 
--- a/testing/marionette/client/marionette/marionette_test.py
+++ b/testing/marionette/client/marionette/marionette_test.py
@@ -1,62 +1,58 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import imp
 import os
 import re
 import sys
 import types
 import unittest
+import weakref
 
 from errors import *
 from marionette import HTMLElement, Marionette
 
 def skip_if_b2g(target):
     def wrapper(self, *args, **kwargs):
         if not hasattr(self.marionette, 'b2g') or not self.marionette.b2g:
             return target(self, *args, **kwargs)
         else:
             sys.stderr.write('skipping ... ')
     return wrapper
 
 class CommonTestCase(unittest.TestCase):
 
+    match_re = None
+
     def __init__(self, methodName):
         unittest.TestCase.__init__(self, methodName)
         self.loglines = None
         self.perfdata = None
 
-    def kill_gaia_app(self, url):
-        self.marionette.execute_script("""
-window.wrappedJSObject.getApplicationManager().kill("%s");
-return(true);
-""" % url)
-
-    def kill_gaia_apps(self):
-        # shut down any running Gaia apps
-        # XXX there's no API to do this currently
-        pass
+    @classmethod
+    def match(cls, filename):
+        """
+        Determines if the specified filename should be handled by this
+        test class; this is done by looking for a match for the filename
+        using cls.match_re.
+        """
+        if not cls.match_re:
+            return False
+        m = cls.match_re.match(filename)
+        return m is not None
 
-    def launch_gaia_app(self, url):
-        # launch the app using Gaia's AppManager
-        self.marionette.set_script_timeout(30000)
-        frame = self.marionette.execute_async_script("""
-var frame = window.wrappedJSObject.getApplicationManager().launch("%s").element;
-window.addEventListener('message', function frameload(e) {
-    if (e.data == 'appready') {
-        window.removeEventListener('message', frameload);
-        marionetteScriptFinished(frame);
-    }
-});
-    """ % url)
-
-        self.assertTrue(isinstance(frame, HTMLElement))
-        return frame
+    @classmethod
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette):
+        """
+        Adds all the tests in the specified file to the specified suite.
+        """
+        raise NotImplementedError
 
     def set_up_test_page(self, emulator, url="test.html", permissions=None):
         emulator.set_context("content")
         url = emulator.absolute_url(url)
         emulator.navigate(url)
 
         if not permissions:
             return
@@ -85,25 +81,41 @@ permissions.forEach(function (perm) {
         if self.marionette.session is not None:
             self.loglines = self.marionette.get_logs()
             self.perfdata = self.marionette.get_perf_data()
             self.marionette.delete_session()
         self.marionette = None
 
 class MarionetteTestCase(CommonTestCase):
 
+    match_re = re.compile(r"test_(.*)\.py$")
+
     def __init__(self, marionette_weakref, methodName='runTest',
                  filepath='', **kwargs):
         self._marionette_weakref = marionette_weakref
         self.marionette = None
         self.extra_emulator_index = -1
         self.methodName = methodName
         self.filepath = filepath
         CommonTestCase.__init__(self, methodName, **kwargs)
 
+    @classmethod
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette):
+        test_mod = imp.load_source(mod_name, filepath)
+
+        for name in dir(test_mod):
+            obj = getattr(test_mod, name)
+            if (isinstance(obj, (type, types.ClassType)) and
+                issubclass(obj, unittest.TestCase)):
+                testnames = testloader.getTestCaseNames(obj)
+                for testname in testnames:
+                    suite.addTest(obj(weakref.ref(marionette),
+                                  methodName=testname,
+                                  filepath=filepath))
+
     def setUp(self):
         CommonTestCase.setUp(self)
         self.marionette.execute_script("log('TEST-START: %s:%s')" % 
                                        (self.filepath.replace('\\', '\\\\'), self.methodName))
 
     def tearDown(self):
         self.marionette.set_context("content")
         self.marionette.execute_script("log('TEST-END: %s:%s')" % 
@@ -124,25 +136,29 @@ class MarionetteTestCase(CommonTestCase)
             qemu = self.marionette.extra_emulators[self.extra_emulator_index]
         return qemu
 
 
 class MarionetteJSTestCase(CommonTestCase):
 
     context_re = re.compile(r"MARIONETTE_CONTEXT(\s*)=(\s*)['|\"](.*?)['|\"];")
     timeout_re = re.compile(r"MARIONETTE_TIMEOUT(\s*)=(\s*)(\d+);")
-    launch_re = re.compile(r"MARIONETTE_LAUNCH_APP(\s*)=(\s*)['|\"](.*?)['|\"];")
+    match_re = re.compile(r"test_(.*)\.js$")
 
     def __init__(self, marionette_weakref, methodName='runTest', jsFile=None):
         assert(jsFile)
         self.jsFile = jsFile
         self._marionette_weakref = marionette_weakref
         self.marionette = None
         CommonTestCase.__init__(self, methodName)
 
+    @classmethod
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette):
+        suite.addTest(cls(weakref.ref(marionette), jsFile=filepath))
+
     def runTest(self):
         if self.marionette.session is None:
             self.marionette.start_session()
         self.marionette.execute_script("log('TEST-START: %s');" % self.jsFile.replace('\\', '\\\\'))
         f = open(self.jsFile, 'r')
         js = f.read()
         args = []
 
@@ -168,28 +184,19 @@ class MarionetteJSTestCase(CommonTestCas
             page = self.marionette.absolute_url("empty.html")
             self.marionette.navigate(page)
 
         timeout = self.timeout_re.search(js)
         if timeout:
             timeout = timeout.group(3)
             self.marionette.set_script_timeout(timeout)
 
-        launch_app = self.launch_re.search(js)
-        if launch_app:
-            launch_app = launch_app.group(3)
-            frame = self.launch_gaia_app(launch_app)
-            args.append({'__marionetteArgs': {'appframe': frame}})
-
         try:
             results = self.marionette.execute_js_script(js, args, special_powers=True)
 
-            if launch_app:
-                self.kill_gaia_app(launch_app)
-
             self.assertTrue(not 'timeout' in self.jsFile,
                             'expected timeout not triggered')
 
             if 'fail' in self.jsFile:
                 self.assertTrue(results['failed'] > 0,
                                 "expected test failures didn't occur")
             else:
                 fails = []
--- a/testing/marionette/client/marionette/runtests.py
+++ b/testing/marionette/client/marionette/runtests.py
@@ -23,17 +23,17 @@ try:
 except ImportError:
     print "manifestparser or mozhttpd not found!  Please install mozbase:\n"
     print "\tgit clone git://github.com/mozilla/mozbase.git"
     print "\tpython setup_development.py\n"
     import sys
     sys.exit(1)
 
 from marionette import Marionette
-from marionette_test import MarionetteJSTestCase
+from marionette_test import MarionetteJSTestCase, MarionetteTestCase
 
 
 class MarionetteTestResult(unittest._TextTestResult):
 
     def __init__(self, *args):
         super(MarionetteTestResult, self).__init__(*args)
         self.passed = 0
         self.perfdata = None
@@ -78,16 +78,21 @@ class MarionetteTestResult(unittest._Tex
                 else:
                     self.perfdata.join_results(testcase.perfdata)
 
 
 class MarionetteTextTestRunner(unittest.TextTestRunner):
 
     resultclass = MarionetteTestResult
 
+    def __init__(self, **kwargs):
+        self.perf = kwargs['perf']
+        del kwargs['perf']
+        unittest.TextTestRunner.__init__(self, **kwargs)
+
     def _makeResult(self):
         return self.resultclass(self.stream, self.descriptions, self.verbosity)
 
     def run(self, test):
         "Run the given test case or test suite."
         result = self._makeResult()
         if hasattr(self, 'failfast'):
             result.failfast = self.failfast
@@ -102,17 +107,17 @@ class MarionetteTextTestRunner(unittest.
         finally:
             stopTestRun = getattr(result, 'stopTestRun', None)
             if stopTestRun is not None:
                 stopTestRun()
         stopTime = time.time()
         timeTaken = stopTime - startTime
         result.printErrors()
         result.printLogs(test)
-        if options.perf:
+        if self.perf:
             result.getPerfData(test)
         if hasattr(result, 'separator2'):
             self.stream.writeln(result.separator2)
         run = result.testsRun
         self.stream.writeln("Ran %d test%s in %.3fs" %
                             (run, run != 1 and "s" or "", timeTaken))
         self.stream.writeln()
 
@@ -151,17 +156,17 @@ class MarionetteTextTestRunner(unittest.
 
 class MarionetteTestRunner(object):
 
     def __init__(self, address=None, emulator=None, emulatorBinary=None,
                  emulatorImg=None, emulator_res='480x800', homedir=None,
                  bin=None, profile=None, autolog=False, revision=None,
                  es_server=None, rest_server=None, logger=None,
                  testgroup="marionette", noWindow=False, logcat_dir=None,
-                 xml_output=None):
+                 xml_output=None, repeat=0, perf=False, perfserv=None):
         self.address = address
         self.emulator = emulator
         self.emulatorBinary = emulatorBinary
         self.emulatorImg = emulatorImg
         self.emulator_res = emulator_res
         self.homedir = homedir
         self.bin = bin
         self.profile = profile
@@ -173,16 +178,23 @@ class MarionetteTestRunner(object):
         self.logger = logger
         self.noWindow = noWindow
         self.httpd = None
         self.baseurl = None
         self.marionette = None
         self.logcat_dir = logcat_dir
         self.perfrequest = None
         self.xml_output = xml_output
+        self.repeat = repeat
+        self.perf = perf
+        self.perfserv = perfserv
+
+        # set up test handlers
+        self.test_handlers = []
+        self.register_handlers()
 
         self.reset_test_stats()
 
         if self.logger is None:
             self.logger = logging.getLogger('Marionette')
             self.logger.setLevel(logging.INFO)
             self.logger.addHandler(logging.StreamHandler())
 
@@ -288,20 +300,20 @@ class MarionetteTestRunner(object):
         for f in self.failures:
             testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
 
         testgroup.submit()
 
     def run_tests(self, tests, testtype=None):
         self.reset_test_stats()
         starttime = datetime.utcnow()
-        while options.repeat >=0 :
+        while self.repeat >=0:
             for test in tests:
                 self.run_test(test, testtype)
-            options.repeat -= 1
+            self.repeat -= 1
         self.logger.info('\nSUMMARY\n-------')
         self.logger.info('passed: %d' % self.passed)
         self.logger.info('failed: %d' % self.failed)
         self.logger.info('todo: %d' % self.todo)
         self.elapsedtime = datetime.utcnow() - starttime
         if self.autolog:
             self.post_to_autolog(self.elapsedtime)
         if self.perfrequest and options.perf:
@@ -349,29 +361,29 @@ class MarionetteTestRunner(object):
                     elif atype.startswith('-'):
                         testargs.update({ atype[1:]: 'false' })
                     else:
                         testargs.update({ atype: 'true' })
 
             manifest = TestManifest()
             manifest.read(filepath)
 
-            if options.perf:
-                if options.perfserv is None:
-                    options.perfserv = manifest.get("perfserv")[0]
+            if self.perf:
+                if self.perfserv is None:
+                    self.perfserv = manifest.get("perfserv")[0]
                 machine_name = socket.gethostname()
                 try:
                     manifest.has_key("machine_name")
                     machine_name = manifest.get("machine_name")[0]
                 except:
                     self.logger.info("Using machine_name: %s" % machine_name)
                 os_name = platform.system()
                 os_version = platform.release()
                 self.perfrequest = datazilla.DatazillaRequest(
-                             server=options.perfserv,
+                             server=self.perfserv,
                              machine_name=machine_name,
                              os=os_name,
                              os_version=os_version,
                              platform=manifest.get("platform")[0],
                              build_name=manifest.get("build_name")[0],
                              version=manifest.get("version")[0],
                              revision=self.revision,
                              branch=manifest.get("branch")[0],
@@ -381,49 +393,41 @@ class MarionetteTestRunner(object):
             manifest_tests = manifest.get(**testargs)
 
             for i in manifest_tests:
                 self.run_test(i["path"], testtype)
             return
 
         self.logger.info('TEST-START %s' % os.path.basename(test))
 
-        if file_ext == '.py':
-            test_mod = imp.load_source(mod_name, filepath)
-
-            for name in dir(test_mod):
-                obj = getattr(test_mod, name)
-                if (isinstance(obj, (type, types.ClassType)) and
-                    issubclass(obj, unittest.TestCase)):
-                    testnames = testloader.getTestCaseNames(obj)
-                    for testname in testnames:
-                        suite.addTest(obj(weakref.ref(self.marionette),
-                                      methodName=testname,
-                                      filepath=filepath))
-
-        elif file_ext == '.js':
-            suite.addTest(MarionetteJSTestCase(weakref.ref(self.marionette), jsFile=filepath))
+        for handler in self.test_handlers:
+            if handler.match(os.path.basename(test)):
+                handler.add_tests_to_suite(mod_name, filepath, suite, testloader, self.marionette)
+                break
 
         if suite.countTestCases():
-            results = MarionetteTextTestRunner(verbosity=3).run(suite)
+            results = MarionetteTextTestRunner(verbosity=3, perf=self.perf).run(suite)
             self.results.append(results)
 
             self.failed += len(results.failures) + len(results.errors)
             if results.perfdata and options.perf:
                 self.perfrequest.add_datazilla_result(results.perfdata)
             if hasattr(results, 'skipped'):
                 self.todo += len(results.skipped) + len(results.expectedFailures)
             self.passed += results.passed
             for failure in results.failures + results.errors:
                 self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
             if hasattr(results, 'unexpectedSuccess'):
                 self.failed += len(results.unexpectedSuccesses)
                 for failure in results.unexpectedSuccesses:
                     self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
 
+    def register_handlers(self):
+        self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase])
+
     def cleanup(self):
         if self.httpd:
             self.httpd.stop()
 
     __del__ = cleanup
 
     def generate_xml(self, results_list):
 
@@ -510,17 +514,17 @@ class MarionetteTestRunner(object):
 
             for cls in classes.itervalues():
                 assembly.appendChild(cls)
 
         doc.appendChild(assembly)
         return doc.toxml(encoding='utf-8')
 
 
-if __name__ == "__main__":
+def parse_options():
     parser = OptionParser(usage='%prog [options] test_file_or_dir <test_file_or_dir> ...')
     parser.add_option("--autolog",
                       action = "store_true", dest = "autolog",
                       default = False,
                       help = "send test results to autolog")
     parser.add_option("--revision",
                       action = "store", dest = "revision",
                       help = "git revision for autolog/perfdata submissions")
@@ -605,31 +609,46 @@ if __name__ == "__main__":
         import datazilla
 
     # check for valid resolution string, strip whitespaces
     try:
         dims = options.emulator_res.split('x')
         assert len(dims) == 2
         width = str(int(dims[0]))
         height = str(int(dims[1]))
-        res = 'x'.join([width, height])
+        options.emulator_res = 'x'.join([width, height])
     except:
         raise ValueError('Invalid emulator resolution format. '
-                         'Should be like "480x800".\n')
+                         'Should be like "480x800".')
+
+    return (options, tests)
 
-    runner = MarionetteTestRunner(address=options.address,
-                                  emulator=options.emulator,
-                                  emulatorBinary=options.emulatorBinary,
-                                  emulatorImg=options.emulatorImg,
-                                  emulator_res=res,
-                                  homedir=options.homedir,
-                                  logcat_dir=options.logcat_dir,
-                                  bin=options.bin,
-                                  profile=options.profile,
-                                  noWindow=options.noWindow,
-                                  revision=options.revision,
-                                  testgroup=options.testgroup,
-                                  autolog=options.autolog,
-                                  xml_output=options.xml_output)
+def startTestRunner(runner_class, options, tests):
+    runner = runner_class(address=options.address,
+                          emulator=options.emulator,
+                          emulatorBinary=options.emulatorBinary,
+                          emulatorImg=options.emulatorImg,
+                          emulator_res=options.emulator_res,
+                          homedir=options.homedir,
+                          logcat_dir=options.logcat_dir,
+                          bin=options.bin,
+                          profile=options.profile,
+                          noWindow=options.noWindow,
+                          revision=options.revision,
+                          testgroup=options.testgroup,
+                          autolog=options.autolog,
+                          xml_output=options.xml_output,
+                          repeat=options.repeat,
+                          perf=options.perf,
+                          perfserv=options.perfserv)
     runner.run_tests(tests, testtype=options.type)
+    return runner
+
+def cli(runner_class=MarionetteTestRunner):
+    options, tests = parse_options()
+    runner = startTestRunner(runner_class, options, tests)
     if runner.failed > 0:
         sys.exit(10)
 
+if __name__ == "__main__":
+    cli()
+
+
--- a/testing/marionette/client/setup.py
+++ b/testing/marionette/client/setup.py
@@ -1,12 +1,12 @@
 import os
 from setuptools import setup, find_packages
 
-version = '0.3'
+version = '0.4'
 
 # get documentation from the README
 try:
     here = os.path.dirname(os.path.abspath(__file__))
     description = file(os.path.join(here, 'README.md')).read()
 except (OSError, IOError):
     description = ''
 
@@ -22,11 +22,16 @@ setup(name='marionette',
       keywords='mozilla',
       author='Jonathan Griffin',
       author_email='jgriffin@mozilla.com',
       url='https://wiki.mozilla.org/Auto-tools/Projects/Marionette',
       license='MPL',
       packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
       include_package_data=True,
       zip_safe=False,
+      entry_points="""
+      # -*- Entry points: -*-
+      [console_scripts]
+      marionette = marionette.runtests:cli
+      """,
       install_requires=deps,
       )