Bug 1340551 - Log tests by manifest from suite_start in marionette-harness, r=jgraham
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 22 Feb 2017 15:09:02 -0500
changeset 345168 762d295b35b9c5e305861fada0ad8b4f76d2375d
parent 345167 b2b8e0f9fe10f3307ef4bb180fd5d422ca744b2b
child 345169 3026fb03c97d8c1223770a46f4bc86a731ac3edc
push id38078
push userahalberstadt@mozilla.com
push dateTue, 28 Feb 2017 14:27:25 +0000
treeherderautoland@3026fb03c97d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1340551
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1340551 - Log tests by manifest from suite_start in marionette-harness, r=jgraham MozReview-Commit-ID: 95XZAO4CfWP
testing/marionette/harness/marionette_harness/runner/base.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
--- a/testing/marionette/harness/marionette_harness/runner/base.py
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -8,16 +8,17 @@ import random
 import re
 import socket
 import sys
 import time
 import traceback
 import unittest
 
 from argparse import ArgumentParser
+from collections import defaultdict
 from copy import deepcopy
 
 import mozinfo
 import moznetwork
 import mozprofile
 import mozversion
 import serve
 
@@ -839,17 +840,21 @@ class BaseMarionetteTestRunner(object):
         appinfo_e10s = self.appinfo.get('browserTabsRemoteAutostart', False)
         self.logger.info("e10s is {}".format("enabled" if appinfo_e10s else "disabled"))
         if self.e10s != appinfo_e10s:
             message_e10s = ("BaseMarionetteTestRunner configuration (self.e10s) does "
                             "not match browser appinfo")
             self.cleanup()
             raise AssertionError(message_e10s)
 
-        self.logger.suite_start(self.tests,
+        tests_by_group = defaultdict(list)
+        for test in self.tests:
+            tests_by_group[test['group']].append(test['filepath'])
+
+        self.logger.suite_start(tests_by_group,
                                 version_info=self.version_info,
                                 device_info=device_info)
 
         self._log_skipped_tests()
 
         interrupted = None
         try:
             counter = self.repeat
@@ -913,17 +918,17 @@ class BaseMarionetteTestRunner(object):
 
     def start_fixture_servers(self):
         root = self.server_root or os.path.join(os.path.dirname(here), "www")
         if self.appName == "fennec":
             return serve.start(root, host=moznetwork.get_ip())
         else:
             return serve.start(root)
 
-    def add_test(self, test, expected='pass'):
+    def add_test(self, test, expected='pass', group='default'):
         filepath = os.path.abspath(test)
 
         if os.path.isdir(filepath):
             for root, dirs, files in os.walk(filepath):
                 for filename in files:
                     if filename.endswith('.ini'):
                         msg_tmpl = ("Ignoring manifest '{0}'; running all tests in '{1}'."
                                     " See --help for details.")
@@ -932,16 +937,18 @@ class BaseMarionetteTestRunner(object):
                     elif self._is_filename_valid(filename):
                         test_file = os.path.join(root, filename)
                         self.add_test(test_file)
             return
 
         file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
 
         if file_ext == '.ini':
+            group = filepath
+
             manifest = TestManifest()
             manifest.read(filepath)
 
             json_path = update_mozinfo(filepath)
             self.logger.info("mozinfo updated from: {}".format(json_path))
             self.logger.info("mozinfo is: {}".format(mozinfo.info))
 
             filters = []
@@ -970,22 +977,20 @@ class BaseMarionetteTestRunner(object):
                     self.manifest_skipped_tests.append(test)
                 else:
                     target_tests.append(test)
 
             for i in target_tests:
                 if not os.path.exists(i["path"]):
                     raise IOError("test file: {} does not exist".format(i["path"]))
 
-                file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
-
-                self.add_test(i["path"], i["expected"])
+                self.add_test(i["path"], i["expected"], group=group)
             return
 
-        self.tests.append({'filepath': filepath, 'expected': expected})
+        self.tests.append({'filepath': filepath, 'expected': expected, 'group': group})
 
     def run_test(self, filepath, expected):
         testloader = unittest.TestLoader()
         suite = unittest.TestSuite()
         self.test_kwargs['expected'] = expected
         mod_name = os.path.splitext(os.path.split(filepath)[-1])[0]
         for handler in self.test_handlers:
             if handler.match(os.path.basename(filepath)):
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -268,17 +268,17 @@ def test_record_crash(runner, has_crashe
 
 def test_add_test_module(runner):
     tests = ['test_something.py', 'testSomething.js', 'bad_test.py']
     assert len(runner.tests) == 0
     for test in tests:
         with patch('os.path.abspath', return_value=test) as abspath:
             runner.add_test(test)
         assert abspath.called
-        expected = {'filepath': test, 'expected': 'pass'}
+        expected = {'filepath': test, 'expected': 'pass', 'group': 'default'}
         assert expected in runner.tests
     # add_test doesn't validate module names; 'bad_test.py' gets through
     assert len(runner.tests) == 3
 
 
 def test_add_test_directory(runner):
     test_dir = 'path/to/tests'
     dir_contents = [