Bug 1212608 - Add parts of firefox_ui_harness to Marionette runner; r=automatedtester
authorMaja Frydrychowicz <mjzffr@gmail.com>
Fri, 23 Oct 2015 16:43:40 -0400
changeset 306941 1cb956020202df4f65eeac7b68794c24610b7a7d
parent 306940 84a9236266cac6f561908c7c717c05c486305e3c
child 306942 f6ec673b3b76e61dfc10a8cf70e5efe861107fa9
push id7225
push usermjzffr@gmail.com
push dateThu, 05 Nov 2015 20:58:39 +0000
reviewersautomatedtester
bugs1212608
milestone45.0a1
Bug 1212608 - Add parts of firefox_ui_harness to Marionette runner; r=automatedtester * BaseMarionetteRunner: any change to self.bin resets self.marionette and self.tests, so you change binaries between calls to run_tests() * Refactor runtests.py to make it easier to extend/customize the harness for different test suites. Add more error handling.
testing/marionette/client/marionette/runner/base.py
testing/marionette/client/marionette/runtests.py
--- a/testing/marionette/client/marionette/runner/base.py
+++ b/testing/marionette/client/marionette/runner/base.py
@@ -564,17 +564,17 @@ class BaseMarionetteTestRunner(object):
 
             # In the event we're gathering debug without starting a session, skip marionette commands
             if marionette.session is not None:
                 try:
                     with marionette.using_context(marionette.CONTEXT_CHROME):
                         rv['screenshot'] = marionette.screenshot()
                     with marionette.using_context(marionette.CONTEXT_CONTENT):
                         rv['source'] = marionette.page_source
-                except:
+                except Exception:
                     logger = get_default_logger()
                     logger.warning('Failed to gather test failure debug.', exc_info=True)
             return rv
 
         self.result_callbacks.append(gather_debug)
 
         def update(d, u):
             """ Update a dictionary that may contain nested dictionaries. """
@@ -634,16 +634,35 @@ class BaseMarionetteTestRunner(object):
     @property
     def appName(self):
         if self._appName:
             return self._appName
 
         self._appName = self.capabilities.get('browserName')
         return self._appName
 
+    @property
+    def bin(self):
+        return self._bin
+
+    @bin.setter
+    def bin(self, path):
+        """
+        Set binary and reset parts of runner accordingly
+
+        Intended use: to change binary between calls to run_tests
+        """
+        self._bin = path
+        self.tests = []
+        if hasattr(self, 'marionette') and self.marionette:
+            self.marionette.cleanup()
+            if self.marionette.instance:
+                self.marionette.instance = None
+        self.marionette = None
+
     def reset_test_stats(self):
         self.passed = 0
         self.failed = 0
         self.unexpected_successes = 0
         self.todo = 0
         self.skipped = 0
         self.failures = []
 
@@ -893,17 +912,17 @@ setReq.onerror = function() {
             self.logger.info("Using seed where seed is:%d" % self.shuffle_seed)
 
         self.logger.suite_end()
 
     def start_httpd(self, need_external_ip):
         warnings.warn("start_httpd has been deprecated in favour of create_httpd",
             DeprecationWarning)
         self.httpd = self.create_httpd(need_external_ip)
-        
+
     def create_httpd(self, need_external_ip):
         host = "127.0.0.1"
         if need_external_ip:
             host = moznetwork.get_ip()
         root = self.server_root or os.path.join(os.path.dirname(here), "www")
         rv = httpd.FixtureServer(root, host=host)
         rv.start()
         return rv
--- a/testing/marionette/client/marionette/runtests.py
+++ b/testing/marionette/client/marionette/runtests.py
@@ -23,42 +23,61 @@ class MarionetteTestRunner(BaseMarionett
 
 
 class MarionetteArguments(BaseMarionetteArguments):
     def __init__(self, **kwargs):
         BaseMarionetteArguments.__init__(self, **kwargs)
         self.register_argument_container(BrowserMobProxyArguments())
 
 
-def startTestRunner(runner_class, args):
-    if args.pydebugger:
-        MarionetteTestCase.pydebugger = __import__(args.pydebugger)
+class MarionetteHarness(object):
+    def __init__(self,
+                 runner_class=MarionetteTestRunner,
+                 parser_class=MarionetteArguments):
+        self._runner_class = runner_class
+        self._parser_class = parser_class
+        self.args = self.parse_args()
+
+    def parse_args(self, logger_defaults=None):
+        parser = self._parser_class(
+            usage='%(prog)s [options] test_file_or_dir <test_file_or_dir> ...',
+            version="%(prog)s {version}"
+                    " (using marionette-driver: {driver_version}, "
+                    "marionette-transport: {transport_version})".format(
+                        version=__version__,
+                        driver_version=driver_version,
+                        transport_version=transport_version
+                    )
+        )
+        mozlog.commandline.add_logging_group(parser)
+        args = parser.parse_args()
+        parser.verify_usage(args)
 
-    args = vars(args)
-    tests = args.pop('tests')
-    runner = runner_class(**args)
-    runner.run_tests(tests)
-    return runner
+        logger = mozlog.commandline.setup_logging(
+            args.logger_name, args, logger_defaults or {"tbpl": sys.stdout})
+
+        args.logger = logger
+        return args
+
+    def process_args(self):
+        if self.args.pydebugger:
+            MarionetteTestCase.pydebugger = __import__(self.args.pydebugger)
+
+    def run(self):
+        try:
+            self.process_args()
+            args_dict = vars(self.args)
+            tests = args_dict.pop('tests')
+            runner = self._runner_class(**args_dict)
+            runner.run_tests(tests)
+            if runner.failed > 0:
+                sys.exit(10)
+        except Exception:
+            self.args.logger.error('Failure during test execution.',
+                                   exc_info=True)
+            sys.exit(1)
+
 
 def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments):
-    parser = parser_class(
-        usage='%(prog)s [options] test_file_or_dir <test_file_or_dir> ...',
-        version="%(prog)s {version} (using marionette-driver: {driver_version}"
-                ", marionette-transport: {transport_version})".format(
-                    version=__version__,
-                    driver_version=driver_version,
-                    transport_version=transport_version)
-    )
-    mozlog.commandline.add_logging_group(parser)
-    args = parser.parse_args()
-    parser.verify_usage(args)
-
-    logger = mozlog.commandline.setup_logging(
-        args.logger_name, args, {"tbpl": sys.stdout})
-
-    args.logger = logger
-
-    runner = startTestRunner(runner_class, args)
-    if runner.failed > 0:
-        sys.exit(10)
+    MarionetteHarness(runner_class, parser_class).run()
 
 if __name__ == "__main__":
     cli()