Bug 1572794 - Add summary and filter-values regex to 'mach test-info report'; r=jmaher
authorGeoff Brown <gbrown@mozilla.com>
Tue, 20 Aug 2019 16:19:40 +0000
changeset 488994 23bfe455aac9daefced32301d62d574f4732dd69
parent 488993 f5da79194d06eaeb2e0b64b5779da0feb01fe864
child 488995 a7651a98ea517d8f223a28bce39e205296eb808f
push id36460
push usermalexandru@mozilla.com
push dateTue, 20 Aug 2019 21:52:47 +0000
treeherdermozilla-central@0dc91d87721b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1572794, 1568638
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1572794 - Add summary and filter-values regex to 'mach test-info report'; r=jmaher Optional summary provides a convenient synopsis of matching tests. filter-values regex was requested for in bug 1568638, like --filter-values='(?<!!)fission'. Differential Revision: https://phabricator.services.mozilla.com/D42570
testing/mach_commands.py
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -1163,72 +1163,81 @@ class TestInfoCommand(MachCommandBase):
     @CommandArgument('--subsuite',
                      help='Limit results to tests of the specified subsuite (eg. "devtools").')
     @CommandArgument('paths', nargs=argparse.REMAINDER,
                      help='File system paths of interest.')
     @CommandArgument('--show-manifests', action='store_true',
                      help='Include test manifests in report.')
     @CommandArgument('--show-tests', action='store_true',
                      help='Include individual tests in report.')
+    @CommandArgument('--show-summary', action='store_true',
+                     help='Include summary in report.')
     @CommandArgument('--filter-values',
-                     help='Comma-separated list of values to filter on; '
+                     help='Comma-separated list of value regular expressions to filter on; '
                           'displayed tests contain all specified values.')
     @CommandArgument('--filter-keys',
                      help='Comma-separated list of test keys to filter on, '
                           'like "skip-if"; only these fields will be searched '
                           'for filter-values.')
     @CommandArgument('--no-component-report', action='store_false',
                      dest="show_components", default=True,
                      help='Do not categorize by bugzilla component.')
     @CommandArgument('--output-file',
                      help='Path to report file.')
     def test_report(self, components, flavor, subsuite, paths,
-                    show_manifests, show_tests,
+                    show_manifests, show_tests, show_summary,
                     filter_values, filter_keys, show_components, output_file):
         import mozpack.path as mozpath
+        import re
         from moztest.resolve import TestResolver
 
         def matches_filters(test):
             '''
                Return True if all of the requested filter_values are found in this test;
                if filter_keys are specified, restrict search to those test keys.
             '''
             for value in filter_values:
                 value_found = False
                 for key in test:
                     if not filter_keys or key in filter_keys:
-                        if value in test[key]:
+                        if re.search(value, test[key]):
                             value_found = True
                             break
                 if not value_found:
                     return False
             return True
 
-        if not show_manifests and not show_tests:
+        # Ensure useful report by default
+        if not show_manifests and not show_tests and not show_summary:
             show_manifests = True
+            show_summary = True
+
         by_component = {}
         if components:
             components = components.split(',')
         if filter_keys:
             filter_keys = filter_keys.split(',')
         if filter_values:
             filter_values = filter_values.split(',')
         else:
             filter_values = []
 
         print("Finding tests...")
         resolver = self._spawn(TestResolver)
         tests = list(resolver.resolve_tests(paths=paths, flavor=flavor,
                                             subsuite=subsuite))
+
+        manifest_paths = set()
+        for t in tests:
+            manifest_paths.add(t['manifest'])
+        manifest_count = len(manifest_paths)
+        print("Resolver found {} tests, {} manifests".format(len(tests), manifest_count))
+
         if show_manifests:
             by_component['manifests'] = {}
-            manifest_paths = set()
-            for t in tests:
-                manifest_paths.add(t['manifest'])
-            print("{} tests, {} manifests".format(len(tests), len(manifest_paths)))
             manifest_paths = list(manifest_paths)
             manifest_paths.sort()
             for manifest_path in manifest_paths:
                 relpath = mozpath.relpath(manifest_path, self.topsrcdir)
                 print("  {}".format(relpath))
                 if mozpath.commonprefix((manifest_path, self.topsrcdir)) != self.topsrcdir:
                     continue
                 reader = self.mozbuild_reader(config_mode='empty')
@@ -1254,38 +1263,60 @@ class TestInfoCommand(MachCommandBase):
                             manifest_info['tests'] += 1
                             if t.get('skip-if'):
                                 manifest_info['skipped'] += 1
             for key in by_component['manifests']:
                 by_component['manifests'][key].sort()
 
         if show_tests:
             by_component['tests'] = {}
+
+        if show_tests or show_summary:
+            test_count = 0
+            failed_count = 0
+            skipped_count = 0
+            component_set = set()
             for t in tests:
                 reader = self.mozbuild_reader(config_mode='empty')
                 if not matches_filters(t):
                     continue
+                test_count += 1
                 relpath = t.get('srcdir_relpath')
                 for info_path, info in reader.files_info([relpath]).items():
                     bug_component = info.get('BUG_COMPONENT')
                     key = "{}::{}".format(bug_component.product, bug_component.component)
                     if (info_path == relpath) and ((not components) or (key in components)):
+                        component_set.add(key)
                         test_info = {'test': relpath}
                         for test_key in ['skip-if', 'fail-if']:
                             value = t.get(test_key)
                             if value:
                                 test_info[test_key] = value
-                        rkey = key if show_components else 'all'
-                        if rkey in by_component['tests']:
-                            by_component['tests'][rkey].append(test_info)
-                        else:
-                            by_component['tests'][rkey] = [test_info]
+                        if t.get('fail-if'):
+                            failed_count += 1
+                        if t.get('skip-if'):
+                            skipped_count += 1
+                        if show_tests:
+                            rkey = key if show_components else 'all'
+                            if rkey in by_component['tests']:
+                                by_component['tests'][rkey].append(test_info)
+                            else:
+                                by_component['tests'][rkey] = [test_info]
                         break
-            for key in by_component['tests']:
-                by_component['tests'][key].sort()
+            if show_tests:
+                for key in by_component['tests']:
+                    by_component['tests'][key].sort()
+
+        if show_summary:
+            by_component['summary'] = {}
+            by_component['summary']['components'] = len(component_set)
+            by_component['summary']['manifests'] = manifest_count
+            by_component['summary']['tests'] = test_count
+            by_component['summary']['failed tests'] = failed_count
+            by_component['summary']['skipped tests'] = skipped_count
 
         json_report = json.dumps(by_component, indent=2, sort_keys=True)
         if output_file:
             with open(output_file, 'w') as f:
                 f.write(json_report)
         else:
             print(json_report)