Bug 1592608 - Add more fields to ReftestManifest test descriptions; r=ahal,froydnj
authorGeoff Brown <gbrown@mozilla.com>
Tue, 05 Nov 2019 17:09:41 +0000
changeset 500627 082bcb14a8c5796ea53715de00b37852f9d45b35
parent 500626 c33c9703069bc316367059a3277575db1503441f
child 500628 26d95900951d4b7b4875456ba0990176956fc02e
push id36768
push usershindli@mozilla.com
push dateTue, 05 Nov 2019 22:07:34 +0000
treeherdermozilla-central@e96c1ca93d25 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal, froydnj
bugs1592608
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1592608 - Add more fields to ReftestManifest test descriptions; r=ahal,froydnj Add test annotations to the ReftestManifest and TestResolver. For example, a test description from the TestResolver might now contain 'skip-if': 'skiaContent'; similar to the content provided for manifestparser tests; this will allow 'mach test-info report' to filter tests based on reftest manifest test annotations. Also add the referenced-test field which identifies the test file associated with test entries for reference files; this will allow test-verify to run the correct reftest when only the reference file has been modified. Differential Revision: https://phabricator.services.mozilla.com/D51113
layout/tools/reftest/reftest/__init__.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/frontend/reader.py
--- a/layout/tools/reftest/reftest/__init__.py
+++ b/layout/tools/reftest/reftest/__init__.py
@@ -28,34 +28,68 @@ FAILURE_TYPES = (
     'asserts',
     'asserts-if',
 )
 PREF_ITEMS = (
     'pref',
     'test-pref',
     'ref-pref',
 )
+RE_ANNOTATION = re.compile(r'(.*)\((.*)\)')
 
 
 class ReftestManifest(object):
     """Represents a parsed reftest manifest.
-
-    We currently only capture file information because that is the only thing
-    tools require.
     """
     def __init__(self, finder=None):
         self.path = None
         self.dirs = set()
         self.files = set()
         self.manifests = set()
-        self.tests = set()
+        self.tests = []
         self.finder = finder
 
     def load(self, path):
         """Parse a reftest manifest file."""
+
+        def add_test(file, annotations, referenced_test=None):
+            # We can't package about:, data:, or chrome: URIs.
+            # Discarding data isn't correct for a parser. But retaining
+            # all data isn't currently a requirement.
+            if RE_PROTOCOL.match(file):
+                return
+            test = os.path.normpath(os.path.join(mdir, urlprefix + file))
+            if test in self.files:
+                # if test path has already been added, make no changes, to
+                # avoid duplicate paths in self.tests
+                return
+            self.files.add(test)
+            self.dirs.add(os.path.dirname(test))
+            test_dict = {
+                'path': test,
+                'here': os.path.dirname(test),
+                'manifest': normalized_path,
+                'name': os.path.basename(test),
+                'head': '',
+                'support-files': '',
+                'subsuite': '',
+            }
+            if referenced_test:
+                test_dict['referenced-test'] = referenced_test
+            for annotation in annotations:
+                m = RE_ANNOTATION.match(annotation)
+                if m:
+                    if m.group(1) not in test_dict:
+                        test_dict[m.group(1)] = m.group(2)
+                    else:
+                        test_dict[m.group(1)] += ";" + m.group(2)
+                else:
+                    test_dict[annotation] = None
+            self.tests.append(test_dict)
+
         normalized_path = os.path.normpath(os.path.abspath(path))
         self.manifests.add(normalized_path)
         if not self.path:
             self.path = normalized_path
 
         mdir = os.path.dirname(normalized_path)
         self.dirs.add(mdir)
 
@@ -77,24 +111,23 @@ class ReftestManifest(object):
             m = RE_COMMENT.search(line)
             if m:
                 line = line[:m.start()]
             line = line.strip()
             if not line:
                 continue
 
             items = line.split()
-            tests = []
+            annotations = []
 
             for i in range(len(items)):
                 item = items[i]
 
-                if item.startswith(FAILURE_TYPES):
-                    continue
-                if item.startswith(PREF_ITEMS):
+                if item.startswith(FAILURE_TYPES) or item.startswith(PREF_ITEMS):
+                    annotations += [item]
                     continue
                 if item == 'HTTP':
                     continue
 
                 m = RE_HTTP.match(item)
                 if m:
                     # Need to package the referenced directory.
                     self.dirs.add(os.path.normpath(os.path.join(
@@ -108,26 +141,15 @@ class ReftestManifest(object):
                 if item == 'default-preferences':
                     break
 
                 if item == 'include':
                     self.load(os.path.join(mdir, items[i+1]))
                     break
 
                 if item == 'load' or item == 'script':
-                    tests.append(items[i+1])
+                    add_test(items[i+1], annotations)
                     break
 
                 if item == '==' or item == '!=' or item == 'print':
-                    tests.extend(items[i+1:i+3])
+                    add_test(items[i+1], annotations)
+                    add_test(items[i+2], annotations, items[i+1])
                     break
-
-            for f in tests:
-                # We can't package about:, data:, or chrome: URIs.
-                # Discarding data isn't correct for a parser. But retaining
-                # all data isn't currently a requirement.
-                if RE_PROTOCOL.match(f):
-                    continue
-
-                test = os.path.normpath(os.path.join(mdir, urlprefix + f))
-                self.files.add(test)
-                self.dirs.add(os.path.dirname(test))
-                self.tests.add((test, normalized_path))
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -1595,27 +1595,17 @@ class TreeMetadataEmitter(LoggingMixin):
 
         # reftest manifests don't come from manifest parser. But they are
         # similar enough that we can use the same emitted objects. Note
         # that we don't perform any installs for reftests.
         obj = TestManifest(context, manifest_full_path, manifest,
                            flavor=flavor, install_prefix='%s/' % flavor,
                            relpath=mozpath.join(manifest_reldir,
                                                 mozpath.basename(manifest_path)))
-
-        for test, source_manifest in sorted(manifest.tests):
-            obj.tests.append({
-                'path': test,
-                'here': mozpath.dirname(test),
-                'manifest': source_manifest,
-                'name': mozpath.basename(test),
-                'head': '',
-                'support-files': '',
-                'subsuite': '',
-            })
+        obj.tests = list(sorted(manifest.tests, key=lambda t: t['path']))
 
         yield obj
 
     def _process_jar_manifests(self, context):
         jar_manifests = context.get('JAR_MANIFESTS', [])
         if len(jar_manifests) > 1:
             raise SandboxValidationError('While JAR_MANIFESTS is a list, '
                                          'it is currently limited to one value.', context)
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -1413,16 +1413,15 @@ class BuildReader(object):
                 for paths, obj in ctx[key]:
                     if isinstance(paths, tuple):
                         path, tests_root = paths
                         tests_root = mozpath.join(ctx.relsrcdir, tests_root)
                         for t in (mozpath.join(tests_root, it[0]) for it in obj):
                             result_context.test_files.add(mozpath.dirname(t) + '/**')
                     else:
                         for t in obj.tests:
-                            if isinstance(t, tuple):
-                                path, _ = t
-                                relpath = mozpath.relpath(path,
+                            if 'relpath' in t:
+                                relpath = t['relpath']
+                            else:
+                                relpath = mozpath.relpath(t['path'],
                                                           self.config.topsrcdir)
-                            else:
-                                relpath = t['relpath']
                             result_context.test_files.add(mozpath.dirname(relpath) + '/**')
         return result_context