Bug 1223429 - Return exit code 10 when Marionette harness has failing tests; r=automatedtester
authorMaja Frydrychowicz <mjzffr@gmail.com>
Tue, 10 Nov 2015 13:09:13 -0500
changeset 308721 c308db03b24298978ae3f3adf6c5f096c7d2ed7c
parent 308720 bc555a3298d3c8d93a670fddff06e31b8a5f2a06
child 308722 bed0a37b59a689c7db3d035f04707e11cad6f9c4
push id7515
push usermjzffr@gmail.com
push dateFri, 13 Nov 2015 14:42:42 +0000
reviewersautomatedtester
bugs1223429
milestone45.0a1
Bug 1223429 - Return exit code 10 when Marionette harness has failing tests; r=automatedtester
testing/marionette/client/marionette/runtests.py
testing/marionette/mach_commands.py
--- a/testing/marionette/client/marionette/runtests.py
+++ b/testing/marionette/client/marionette/runtests.py
@@ -64,19 +64,35 @@ class MarionetteHarness(object):
 
     def run(self):
         try:
             self.process_args()
             args_dict = vars(self.args)
             tests = args_dict.pop('tests')
             runner = self._runner_class(**args_dict)
             runner.run_tests(tests)
-            return runner
+            return runner.failed
         except Exception:
             self.args.logger.error('Failure during test execution.',
                                    exc_info=True)
-            sys.exit(1)
+            raise
+
+
+def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
+        harness_class=MarionetteHarness):
+    """
+    Call the harness to parse args and run tests.
 
-def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments):
-    MarionetteHarness(runner_class, parser_class).run()
+    The following exit codes are expected:
+    - Test failures: 10
+    - Harness/other failures: 1
+    - Success: 0
+    """
+    try:
+        failed = harness_class(runner_class, parser_class).run()
+        if failed > 0:
+            sys.exit(10)
+    except Exception:
+        sys.exit(1)
+    sys.exit(0)
 
 if __name__ == "__main__":
     cli()
--- a/testing/marionette/mach_commands.py
+++ b/testing/marionette/mach_commands.py
@@ -62,20 +62,18 @@ def run_marionette(tests, b2g_path=None,
     for k, v in kwargs.iteritems():
         setattr(args, k, v)
 
     parser.verify_usage(args)
 
     args.logger = commandline.setup_logging("Marionette Unit Tests",
                                             args,
                                             {"mach": sys.stdout})
-
-    results = MarionetteHarness(MarionetteTestRunner, args=args).run()
-
-    if results.failed > 0:
+    failed = MarionetteHarness(MarionetteTestRunner, args=args).run()
+    if failed > 0:
         return 1
     else:
         return 0
 
 @CommandProvider
 class B2GCommands(MachCommandBase):
     def __init__(self, context):
         MachCommandBase.__init__(self, context)