Bug 1340551 - Log tests by manifest from suite_start in mochitest harness, r=jgraham
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 22 Feb 2017 14:36:31 -0500
changeset 394144 b2b8e0f9fe10f3307ef4bb180fd5d422ca744b2b
parent 394143 1fb190001b3ecc45514688c5628239cc2e221fc8
child 394145 762d295b35b9c5e305861fada0ad8b4f76d2375d
push id1468
push userasasaki@mozilla.com
push dateMon, 05 Jun 2017 19:31:07 +0000
treeherdermozilla-release@0641fc6ee9d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1340551
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1340551 - Log tests by manifest from suite_start in mochitest harness, r=jgraham MozReview-Commit-ID: 1lcw62fmofa
testing/mochitest/runrobocop.py
testing/mochitest/runtests.py
--- a/testing/mochitest/runrobocop.py
+++ b/testing/mochitest/runrobocop.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import sys
 import tempfile
 import traceback
+from collections import defaultdict
 
 sys.path.insert(
     0, os.path.abspath(
         os.path.realpath(
             os.path.dirname(__file__))))
 
 from automation import Automation
 from remoteautomation import RemoteAutomation, fennecLogcatFilters
@@ -506,17 +507,22 @@ class RobocopTestRunner(MochitestDesktop
         for test in robocop_tests:
             if self.options.test_paths and test['name'] not in self.options.test_paths:
                 continue
             if 'disabled' in test:
                 self.log.info('TEST-INFO | skipping %s | %s' %
                               (test['name'], test['disabled']))
                 continue
             active_tests.append(test)
-        self.log.suite_start([t['name'] for t in active_tests])
+
+        tests_by_manifest = defaultdict(list)
+        for test in active_tests:
+            tests_by_manifest[test['manifest']].append(test['name'])
+        self.log.suite_start(tests_by_manifest)
+
         worstTestResult = None
         for test in active_tests:
             result = self.runSingleTest(test)
             if worstTestResult is None or worstTestResult == 0:
                 worstTestResult = result
         if worstTestResult is None:
             self.log.warning(
                 "No tests run. Did you pass an invalid TEST_PATH?")
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -8,16 +8,17 @@ Runs the Mochitest test harness.
 
 from __future__ import with_statement
 import os
 import sys
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 sys.path.insert(0, SCRIPT_DIR)
 
 from argparse import Namespace
+from collections import defaultdict
 import ctypes
 import glob
 import json
 import mozcrash
 import mozdebug
 import mozinfo
 import mozprocess
 import mozrunner
@@ -793,16 +794,17 @@ class MochitestDesktop(object):
     test_name = 'automation.py'
 
     def __init__(self, logger_options, quiet=False):
         self.update_mozinfo()
         self.server = None
         self.wsserver = None
         self.websocketProcessBridge = None
         self.sslTunnel = None
+        self.tests_by_manifest = defaultdict(list)
         self._active_tests = None
         self._locations = None
 
         self.marionette = None
         self.start_script = None
         self.mozLogs = None
         self.start_script_kwargs = {}
         self.urlOpts = []
@@ -1300,17 +1302,17 @@ toolbar#nav-bar {
                                 os.path.isfile(path) and path.endswith(".xpi")):
                             extensions.append(path)
         extensions.extend(options.extensionsToInstall)
         return extensions
 
     def logPreamble(self, tests):
         """Logs a suite_start message and test_start/test_end at the beginning of a run.
         """
-        self.log.suite_start([t['path'] for t in tests])
+        self.log.suite_start(self.tests_by_manifest)
         for test in tests:
             if 'disabled' in test:
                 self.log.test_start(test['path'])
                 self.log.test_end(
                     test['path'],
                     'SKIP',
                     message=test['disabled'])
 
@@ -1435,32 +1437,32 @@ toolbar#nav-bar {
             if len(tests) == 0:
                 self.log.error(NO_TESTS_FOUND.format(options.flavor, manifest.fmt_filters()))
 
         paths = []
 
         # When running mochitest locally the manifest is based on topsrcdir,
         # but when running in automation it is based on the test root.
         manifest_root = build_obj.topsrcdir if build_obj else self.testRootAbs
-        manifests = set()
         for test in tests:
             if len(tests) == 1 and 'disabled' in test:
                 del test['disabled']
 
             pathAbs = os.path.abspath(test['path'])
             assert pathAbs.startswith(self.testRootAbs)
             tp = pathAbs[len(self.testRootAbs):].replace('\\', '/').strip('/')
 
             if not self.isTest(options, tp):
                 self.log.warning(
                     'Warning: %s from manifest %s is not a valid test' %
                     (test['name'], test['manifest']))
                 continue
 
-            manifests.add(os.path.relpath(test['manifest'], manifest_root))
+            manifest_relpath = os.path.relpath(test['manifest'], manifest_root)
+            self.tests_by_manifest[manifest_relpath].append(tp)
 
             testob = {'path': tp}
             if 'disabled' in test:
                 testob['disabled'] = test['disabled']
             if 'expected' in test:
                 testob['expected'] = test['expected']
             if 'scheme' in test:
                 testob['scheme'] = test['scheme']
@@ -1486,17 +1488,17 @@ toolbar#nav-bar {
 
             self.log.info("Dumping active_tests to %s file." % options.dump_tests)
             sys.exit()
 
         # Upload a list of test manifests that were executed in this run.
         if 'MOZ_UPLOAD_DIR' in os.environ:
             artifact = os.path.join(os.environ['MOZ_UPLOAD_DIR'], 'manifests.list')
             with open(artifact, 'a') as fh:
-                fh.write('\n'.join(sorted(manifests)))
+                fh.write('\n'.join(sorted(self.tests_by_manifest.keys())))
 
         self._active_tests = paths
         return self._active_tests
 
     def getTestManifest(self, options):
         if isinstance(options.manifestFile, TestManifest):
             manifest = options.manifestFile
         elif options.manifestFile and os.path.isfile(options.manifestFile):