Bug 940924 - Marionette should list the number of unexpected/expected fails at the end of the test run. r=jgriffin
authorJuien Pagès <j.parkouss@gmail.com>
Tue, 27 May 2014 10:01:00 +0200
changeset 185251 99d60f78b54366a62d149d0ab62184d11a1a6e9b
parent 185250 98644e8a0e128dde6f80f71bc7fac91d240d6a09
child 185252 8dff41f0ca9c545e98085ed909ff13d5c627979d
push id26846
push usercbook@mozilla.com
push dateWed, 28 May 2014 12:22:26 +0000
treeherdermozilla-central@96838c90ebac [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgriffin
bugs940924
milestone32.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 940924 - Marionette should list the number of unexpected/expected fails at the end of the test run. r=jgriffin
testing/marionette/client/marionette/runner/base.py
--- a/testing/marionette/client/marionette/runner/base.py
+++ b/testing/marionette/client/marionette/runner/base.py
@@ -665,17 +665,19 @@ class BaseMarionetteTestRunner(object):
             return self._appName
 
         self._appName = self.capabilities.get('browserName')
         return self._appName
 
     def reset_test_stats(self):
         self.passed = 0
         self.failed = 0
+        self.unexpected_successes = 0
         self.todo = 0
+        self.skipped = 0
         self.failures = []
 
     def start_httpd(self, need_external_ip):
         host = "127.0.0.1"
         if need_external_ip:
             host = moznetwork.get_ip()
         docroot = self.server_root or os.path.join(os.path.dirname(os.path.dirname(__file__)), 'www')
         if not os.path.isdir(docroot):
@@ -815,18 +817,24 @@ class BaseMarionetteTestRunner(object):
         while counter >=0:
             round = self.repeat - counter
             if round > 0:
                 self.logger.info('\nREPEAT %d\n-------' % round)
             self.run_test_sets()
             counter -= 1
         self.logger.info('\nSUMMARY\n-------')
         self.logger.info('passed: %d' % self.passed)
-        self.logger.info('failed: %d' % self.failed)
-        self.logger.info('todo: %d' % self.todo)
+        if self.unexpected_successes == 0:
+            self.logger.info('failed: %d' % self.failed)
+        else:
+            self.logger.info('failed: %d (unexpected sucesses: %d)', self.failed, self.unexpected_successes)
+        if self.skipped == 0:
+            self.logger.info('todo: %d', self.todo)
+        else:
+            self.logger.info('todo: %d (skipped: %d)', self.todo, self.skipped)
 
         if self.failed > 0:
             self.logger.info('\nFAILED TESTS\n-------')
             for failed_test in self.failures:
                 self.logger.info('%s' % failed_test[0])
 
         try:
             self.marionette.check_for_crash()
@@ -983,26 +991,28 @@ class BaseMarionetteTestRunner(object):
             runner = self.textrunnerclass(verbosity=3,
                                           marionette=self.marionette,
                                           capabilities=self.capabilities)
             results = runner.run(suite)
             self.results.append(results)
 
             self.failed += len(results.failures) + len(results.errors)
             if hasattr(results, 'skipped'):
+                self.skipped += len(results.skipped)
                 self.todo += len(results.skipped)
             self.passed += results.passed
             for failure in results.failures + results.errors:
                 self.failures.append((results.getInfo(failure), failure.output, 'TEST-UNEXPECTED-FAIL'))
             if hasattr(results, 'unexpectedSuccesses'):
                 self.failed += len(results.unexpectedSuccesses)
+                self.unexpected_successes += len(results.unexpectedSuccesses)
                 for failure in results.unexpectedSuccesses:
                     self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
             if hasattr(results, 'expectedFailures'):
-                self.passed += len(results.expectedFailures)
+                self.todo += len(results.expectedFailures)
 
     def run_test_set(self, tests):
         if self.shuffle:
             random.shuffle(tests)
 
         for test in tests:
             self.run_test(test['filepath'], test['expected'], test['oop'])
             if self.marionette.check_for_crash():