Bug 1595257 - Add annotations report to 'mach test-info report'; r=jmaher
authorGeoff Brown <gbrown@mozilla.com>
Thu, 14 Nov 2019 19:11:07 +0000
changeset 502017 1e54f02ec937bc439dfd0241d8f9f6d677ebdad9
parent 502016 62bdc923737702055fba46dfd3bc7778cb005f39
child 502018 a89e03dd6b3c3917d6c784d57e53fc0ef055d581
push id114172
push userdluca@mozilla.com
push dateTue, 19 Nov 2019 11:31:10 +0000
treeherdermozilla-inbound@b5c5ba07d3db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1595257
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1595257 - Add annotations report to 'mach test-info report'; r=jmaher This "annotations" report provides a different view of manifest data, concentrating on the manifest annotations, like "skip-if". What conditions are used in manifests? How many times does each occur? Differential Revision: https://phabricator.services.mozilla.com/D52926
testing/mach_commands.py
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -1191,30 +1191,32 @@ class TestInfoCommand(MachCommandBase):
     @CommandArgument('paths', nargs=argparse.REMAINDER,
                      help='File system paths of interest.')
     @CommandArgument('--show-manifests', action='store_true',
                      help='Include test manifests in report.')
     @CommandArgument('--show-tests', action='store_true',
                      help='Include individual tests in report.')
     @CommandArgument('--show-summary', action='store_true',
                      help='Include summary in report.')
+    @CommandArgument('--show-annotations', action='store_true',
+                     help='Include list of manifest annotation conditions in report.')
     @CommandArgument('--filter-values',
                      help='Comma-separated list of value regular expressions to filter on; '
                           'displayed tests contain all specified values.')
     @CommandArgument('--filter-keys',
                      help='Comma-separated list of test keys to filter on, '
                           'like "skip-if"; only these fields will be searched '
                           'for filter-values.')
     @CommandArgument('--no-component-report', action='store_false',
                      dest="show_components", default=True,
                      help='Do not categorize by bugzilla component.')
     @CommandArgument('--output-file',
                      help='Path to report file.')
     def test_report(self, components, flavor, subsuite, paths,
-                    show_manifests, show_tests, show_summary,
+                    show_manifests, show_tests, show_summary, show_annotations,
                     filter_values, filter_keys, show_components, output_file):
         import mozpack.path as mozpath
         import re
         from mozbuild.build_commands import Build
         from moztest.resolve import TestResolver
 
         def matches_filters(test):
             '''
@@ -1228,17 +1230,17 @@ class TestInfoCommand(MachCommandBase):
                         if re.search(value, test[key]):
                             value_found = True
                             break
                 if not value_found:
                     return False
             return True
 
         # Ensure useful report by default
-        if not show_manifests and not show_tests and not show_summary:
+        if not show_manifests and not show_tests and not show_summary and not show_annotations:
             show_manifests = True
             show_summary = True
 
         by_component = {}
         if components:
             components = components.split(',')
         if filter_keys:
             filter_keys = filter_keys.split(',')
@@ -1306,30 +1308,63 @@ class TestInfoCommand(MachCommandBase):
                             if t.get('skip-if'):
                                 manifest_info['skipped'] += 1
             for key in by_component['manifests']:
                 by_component['manifests'][key].sort()
 
         if show_tests:
             by_component['tests'] = {}
 
-        if show_tests or show_summary:
+        if show_tests or show_summary or show_annotations:
             test_count = 0
             failed_count = 0
             skipped_count = 0
+            annotation_count = 0
+            condition_count = 0
             component_set = set()
             relpaths = []
+            conditions = {}
+            known_unconditional_annotations = ['skip', 'fail', 'asserts', 'random']
+            known_conditional_annotations = ['skip-if', 'fail-if', 'run-if',
+                                             'fails-if', 'fuzzy-if', 'random-if', 'asserts-if']
             for t in tests:
                 relpath = t.get('srcdir_relpath')
                 relpaths.append(relpath)
             reader = self.mozbuild_reader(config_mode='empty')
             files_info = reader.files_info(relpaths)
             for t in tests:
                 if not matches_filters(t):
                     continue
+                if 'referenced-test' in t:
+                    # Avoid double-counting reftests: disregard reference file entries
+                    continue
+                if show_annotations:
+                    for key in t:
+                        if key in known_unconditional_annotations:
+                            annotation_count += 1
+                        if key in known_conditional_annotations:
+                            annotation_count += 1
+                            # Here 'key' is a manifest annotation type like 'skip-if' and t[key]
+                            # is the associated condition. For example, the manifestparser
+                            # manifest annotation, "skip-if = os == 'win'", is expected to be
+                            # encoded as t['skip-if'] = "os == 'win'".
+                            # To allow for reftest manifests, t[key] may have multiple entries
+                            # separated by ';', each corresponding to a condition for that test
+                            # and annotation type. For example,
+                            # "skip-if(Android&&webrender) skip-if(OSX)", would be
+                            # encoded as t['skip-if'] = "Android&&webrender;OSX".
+                            annotation_conditions = t[key].split(';')
+                            for condition in annotation_conditions:
+                                condition_count += 1
+                                # Trim reftest fuzzy-if ranges: everything after the first comma
+                                # eg. "Android,0-2,1-3" -> "Android"
+                                condition = condition.split(',')[0]
+                                if condition not in conditions:
+                                    conditions[condition] = 0
+                                conditions[condition] += 1
                 test_count += 1
                 relpath = t.get('srcdir_relpath')
                 if relpath in files_info:
                     bug_component = files_info[relpath].get('BUG_COMPONENT')
                     key = "{}::{}".format(bug_component.product, bug_component.component)
                     if (not components) or (key in components):
                         component_set.add(key)
                         test_info = {'test': relpath}
@@ -1356,16 +1391,23 @@ class TestInfoCommand(MachCommandBase):
         if show_summary:
             by_component['summary'] = {}
             by_component['summary']['components'] = len(component_set)
             by_component['summary']['manifests'] = manifest_count
             by_component['summary']['tests'] = test_count
             by_component['summary']['failed tests'] = failed_count
             by_component['summary']['skipped tests'] = skipped_count
 
+        if show_annotations:
+            by_component['annotations'] = {}
+            by_component['annotations']['total annotations'] = annotation_count
+            by_component['annotations']['total conditions'] = condition_count
+            by_component['annotations']['unique conditions'] = len(conditions)
+            by_component['annotations']['conditions'] = conditions
+
         json_report = json.dumps(by_component, indent=2, sort_keys=True)
         if output_file:
             output_file = os.path.abspath(output_file)
             output_dir = os.path.dirname(output_file)
             if not os.path.isdir(output_dir):
                 os.makedirs(output_dir)
 
             with open(output_file, 'w') as f: