Bug 1583353 - [moztest.resolve] Implement a TestLoader that doesn't rely on the build system, r=gbrown
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 06 Nov 2019 17:13:20 +0000
changeset 501586 cc2575c342028b2ccc4a686bce0ebb20b13246ff
parent 501585 b96f1cf028b61ef4d8d647dba6a48b7b48213ec3
child 501587 ca54f70e0297399b460682c0d189ba17b55560e7
push id100300
push userahalberstadt@mozilla.com
push dateTue, 12 Nov 2019 15:57:39 +0000
treeherderautoland@7ea00823df0a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1583353
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1583353 - [moztest.resolve] Implement a TestLoader that doesn't rely on the build system, r=gbrown This loader uses 'reader.find_variables_from_ast' to parse all *_MANIFESTS variables from moz.build files using the abstract syntax tree. This means it will find all such variables regardless of the current buildconfig. Differential Revision: https://phabricator.services.mozilla.com/D51834
python/mozbuild/mozbuild/frontend/reader.py
testing/mozbase/moztest/moztest/resolve.py
testing/mozbase/moztest/tests/data/srcdir/apple/mochitest.ini
testing/mozbase/moztest/tests/data/srcdir/apple/moz.build
testing/mozbase/moztest/tests/data/srcdir/banana/moz.build
testing/mozbase/moztest/tests/data/srcdir/banana/xpcshell.ini
testing/mozbase/moztest/tests/data/srcdir/dragonfruit/elderberry/xpcshell_updater.ini
testing/mozbase/moztest/tests/data/srcdir/dragonfruit/moz.build
testing/mozbase/moztest/tests/data/srcdir/dragonfruit/xpcshell.ini
testing/mozbase/moztest/tests/data/srcdir/fig/grape/instrumentation.ini
testing/mozbase/moztest/tests/data/srcdir/fig/huckleberry/instrumentation.ini
testing/mozbase/moztest/tests/data/srcdir/fig/moz.build
testing/mozbase/moztest/tests/data/srcdir/juniper/browser.ini
testing/mozbase/moztest/tests/data/srcdir/kiwi/browser.ini
testing/mozbase/moztest/tests/data/srcdir/moz.build
testing/mozbase/moztest/tests/test_resolve.py
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -836,16 +836,17 @@ class BuildReader(object):
         self._read_files = set()
         self._execution_stack = []
         self.finder = finder
 
         # Finder patterns to ignore when searching for moz.build files.
         ignores = {
             # Ignore fake moz.build files used for testing moz.build.
             'python/mozbuild/mozbuild/test',
+            'testing/mozbase/moztest/tests/data',
 
             # Ignore object directories.
             'obj*',
         }
 
         self._relevant_mozbuild_finder = FileFinder(self.config.topsrcdir,
                                                     ignore=ignores)
 
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -3,22 +3,25 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import fnmatch
 import os
 import pickle
 import sys
+from abc import ABCMeta, abstractmethod
 from collections import defaultdict
 
-import manifestparser
 import mozpack.path as mozpath
+from manifestparser import combine_fields, TestManifest
 from mozbuild.base import MozbuildObject
+from mozbuild.testing import TEST_MANIFESTS, REFTEST_FLAVORS
 from mozbuild.util import OrderedDefaultDict
+from mozpack.files import FileFinder
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 MOCHITEST_CHUNK_BY_DIR = 4
 MOCHITEST_TOTAL_CHUNKS = 5
 
 
 def WebglSuite(name):
@@ -315,17 +318,24 @@ def rewrite_test_base(test, new_base, ho
                                     test['install-to-subdir'], manifest_relpath)
     else:
         test['path'] = mozpath.join(new_base, test['file_relpath'])
 
     return test
 
 
 class TestLoader(MozbuildObject):
+    __metaclass__ = ABCMeta
 
+    @abstractmethod
+    def __call__(self):
+        """Generate test metadata."""
+
+
+class BuildBackendLoader(TestLoader):
     def __call__(self):
         """Loads the test metadata generated by the TestManifest build backend.
 
         The data is stored in two files:
 
             - <objdir>/all-tests.pkl
             - <objdir>/test-defaults.pkl
 
@@ -363,37 +373,82 @@ class TestLoader(MozbuildObject):
                     # contains the defaults of the included manifest, so
                     # use it instead of [metadata['manifest']].
                     defaults_manifests[0] = (ancestor_manifest, metadata['manifest'])
                     defaults_manifests.append(ancestor_manifest)
 
                 for manifest in defaults_manifests:
                     manifest_defaults = defaults.get(manifest)
                     if manifest_defaults:
-                        metadata = manifestparser.combine_fields(manifest_defaults,
-                                                                 metadata)
+                        metadata = combine_fields(manifest_defaults, metadata)
 
                 yield metadata
 
 
+class TestManifestLoader(TestLoader):
+    def __init__(self, *args, **kwargs):
+        super(TestManifestLoader, self).__init__(*args, **kwargs)
+        self.finder = FileFinder(self.topsrcdir)
+        self.reader = self.mozbuild_reader(config_mode="empty")
+        self.variables = {'{}_MANIFESTS'.format(k): v[0] for k, v in TEST_MANIFESTS.items()}
+        self.variables.update({'{}_MANIFESTS'.format(f.upper()): f for f in REFTEST_FLAVORS})
+
+    def _load_manifestparser_manifest(self, mpath):
+        mp = TestManifest(manifests=[mpath], strict=True, rootdir=self.topsrcdir,
+                          finder=self.finder, handle_defaults=True)
+        return (test for test in mp.tests)
+
+    def _load_reftest_manifest(self, mpath):
+        import reftest
+        manifest = reftest.ReftestManifest(finder=self.finder)
+        manifest.load(mpath)
+
+        for test in sorted(manifest.tests):
+            test['manifest_relpath'] = test['manifest'][len(self.topsrcdir)+1:]
+            yield test
+
+    def __call__(self):
+        for path, name, key, value in self.reader.find_variables_from_ast(self.variables):
+            mpath = os.path.join(self.topsrcdir, os.path.dirname(path), value)
+            flavor = self.variables[name]
+
+            if name.rsplit('_', 1)[0].lower() in REFTEST_FLAVORS:
+                tests = self._load_reftest_manifest(mpath)
+            else:
+                tests = self._load_manifestparser_manifest(mpath)
+
+            for test in tests:
+                path = mozpath.normpath(test['path'])
+                assert mozpath.basedir(path, [self.topsrcdir])
+                relpath = path[len(self.topsrcdir)+1:]
+
+                # Add these keys for compatibility with the build backend loader.
+                test['flavor'] = flavor
+                test['file_relpath'] = relpath
+                test['srcdir_relpath'] = relpath
+                test['dir_relpath'] = mozpath.dirname(relpath)
+
+                yield test
+
+
 class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
-
     test_rewrites = {
         'a11y': '_tests/testing/mochitest/a11y',
         'browser-chrome': '_tests/testing/mochitest/browser',
         'chrome': '_tests/testing/mochitest/chrome',
         'mochitest': '_tests/testing/mochitest/tests',
         'xpcshell': '_tests/xpcshell',
     }
 
     def __init__(self, *args, **kwargs):
-        MozbuildObject.__init__(self, *args, **kwargs)
-        self.load_tests = self._spawn(TestLoader)
+        loader_cls = kwargs.pop('loader_cls', BuildBackendLoader)
+        super(TestResolver, self).__init__(*args, **kwargs)
 
+        self.load_tests = self._spawn(loader_cls)
         self._tests = []
         self._reset_state()
 
         # These suites aren't registered in moz.build so require special handling.
         self._puppeteer_loaded = False
         self._tests_loaded = False
         self._wpt_loaded = False
 
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/apple/mochitest.ini
@@ -0,0 +1,1 @@
+[test_a11y.html]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/apple/moz.build
@@ -0,0 +1,1 @@
+A11Y_MANIFESTS += ["mochitest.ini"]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/banana/moz.build
@@ -0,0 +1,1 @@
+XPCSHELL_TESTS_MANIFESTS += ["xpcshell.ini"]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/banana/xpcshell.ini
@@ -0,0 +1,2 @@
+[currant/test_xpcshell_A.js]
+[currant/test_xpcshell_B.js]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/dragonfruit/elderberry/xpcshell_updater.ini
@@ -0,0 +1,7 @@
+[DEFAULT]
+support-files =
+    data/**
+    xpcshell_updater.ini
+
+[test_xpcshell_C.js]
+head=head_updates.js head2.js
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/dragonfruit/moz.build
@@ -0,0 +1,1 @@
+XPCSHELL_TESTS_MANIFESTS += ['xpcshell.ini']
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/dragonfruit/xpcshell.ini
@@ -0,0 +1,4 @@
+[include:elderberry/xpcshell_updater.ini]
+
+[elderberry/test_xpcshell_C.js]
+head=head_update.js
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/fig/grape/instrumentation.ini
@@ -0,0 +1,2 @@
+[src/TestInstrumentationA.java]
+subsuite=background
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/fig/huckleberry/instrumentation.ini
@@ -0,0 +1,2 @@
+[src/TestInstrumentationB.java]
+subsuite=browser
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/fig/moz.build
@@ -0,0 +1,4 @@
+ANDROID_INSTRUMENTATION_MANIFESTS += [
+    'grape/instrumentation.ini',
+    'huckleberry/instrumentation.ini',
+]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/juniper/browser.ini
@@ -0,0 +1,1 @@
+[browser_chrome.js]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/kiwi/browser.ini
@@ -0,0 +1,3 @@
+[browser_devtools.js]
+subsuite=devtools
+tags=devtools
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/tests/data/srcdir/moz.build
@@ -0,0 +1,4 @@
+BROWSER_CHROME_MANIFESTS += [
+    'juniper/browser.ini',
+    'kiwi/browser.ini',
+]
--- a/testing/mozbase/moztest/tests/test_resolve.py
+++ b/testing/mozbase/moztest/tests/test_resolve.py
@@ -12,46 +12,57 @@ import re
 import shutil
 import tempfile
 from collections import defaultdict
 
 import pytest
 import mozpack.path as mozpath
 import mozunit
 from mozbuild.base import MozbuildObject
+from mozbuild.frontend.reader import BuildReader
+from mozbuild.test.common import MockConfig
 from mozfile import NamedTemporaryFile
 
 from moztest.resolve import (
+    BuildBackendLoader,
+    TestManifestLoader,
     TestResolver,
     TEST_SUITES,
 )
 
+here = os.path.abspath(os.path.dirname(__file__))
+data_path = os.path.join(here, 'data')
+
 
 @pytest.fixture(scope='module')
-def create_tests():
-    sourcedir = '/firefox'
+def topsrcdir():
+    return mozpath.join(data_path, 'srcdir')
+
+
+@pytest.fixture(scope='module')
+def create_tests(topsrcdir):
 
     def inner(*paths, **defaults):
         tests = defaultdict(list)
         for path in paths:
             if isinstance(path, tuple):
                 path, kwargs = path
             else:
                 kwargs = {}
 
             path = mozpath.normpath(path)
             manifest_name = kwargs.get('flavor', defaults.get('flavor', 'manifest'))
             manifest = kwargs.pop('manifest', defaults.pop('manifest',
                                   mozpath.join(mozpath.dirname(path), manifest_name + '.ini')))
 
-            manifest_abspath = mozpath.join(sourcedir, manifest)
+            manifest_abspath = mozpath.join(topsrcdir, manifest)
             relpath = mozpath.relpath(path, mozpath.dirname(manifest))
             test = {
                 'name': relpath,
-                'path': mozpath.join(sourcedir, path),
+                'path': mozpath.join(topsrcdir, path),
                 'relpath': relpath,
                 'file_relpath': path,
                 'dir_relpath': mozpath.dirname(path),
                 'here': mozpath.dirname(manifest_abspath),
                 'manifest': manifest_abspath,
                 'manifest_relpath': manifest,
             }
             test.update(**defaults)
@@ -82,17 +93,17 @@ def all_tests(create_tests):
             "firefox-appdir": "browser",
             "flavor": "xpcshell",
             "head": "head_global.js head_helpers.js head_http.js",
          }),
         ("dragonfruit/elderberry/test_xpcshell_C.js", {
             "flavor": "xpcshell",
             "generated-files": "head_update.js",
             "head": "head_update.js",
-            "manifest": "dragonfruit/elderberry/xpcshell_updater.ini",
+            "manifest": "dragonfruit/xpcshell.ini",
             "reason": "busted",
             "run-sequentially": "Launches application.",
             "skip-if": "os == 'android'",
          }),
         ("dragonfruit/elderberry/test_xpcshell_C.js", {
             "flavor": "xpcshell",
             "generated-files": "head_update.js",
             "head": "head_update.js head2.js",
@@ -110,53 +121,53 @@ def all_tests(create_tests):
             "flavor": "instrumentation",
             "manifest": "fig/huckleberry/instrumentation.ini",
             "subsuite": "browser",
          }),
         ("juniper/browser_chrome.js", {
             "flavor": "browser-chrome",
             "manifest": "juniper/browser.ini",
             "skip-if": "e10s  # broken",
-            "subsuite": "",
          }),
         ("kiwi/browser_devtools.js", {
             "flavor": "browser-chrome",
             "manifest": "kiwi/browser.ini",
             "subsuite": "devtools",
             "tags": "devtools",
          }),
     ])
 
 
 @pytest.fixture(scope='module')
-def defaults():
+def defaults(topsrcdir):
     return {
-        "/firefox/dragonfruit/elderberry/xpcshell_updater.ini": {
+        mozpath.join(topsrcdir, "dragonfruit/elderberry/xpcshell_updater.ini"): {
             "support-files": "\ndata/**\nxpcshell_updater.ini"
         }
     }
 
 
-@pytest.fixture
-def resolver(tmpdir, all_tests, defaults):
+@pytest.fixture(params=[BuildBackendLoader, TestManifestLoader])
+def resolver(request, tmpdir, topsrcdir, all_tests, defaults):
     topobjdir = tmpdir.mkdir("objdir").strpath
+    loader_cls = request.param
 
-    with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
-        pickle.dump(all_tests, fh)
-    with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
-        pickle.dump(defaults, fh)
+    if loader_cls == BuildBackendLoader:
+        with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
+            pickle.dump(all_tests, fh)
+        with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
+            pickle.dump(defaults, fh)
 
-    o = MozbuildObject('/firefox', None, None, topobjdir=topobjdir)
-
-    # Monkey patch the test resolver to avoid tests failing to find make
-    # due to our fake topscrdir.
-    TestResolver._run_make = lambda *a, **b: None
-    resolver = o._spawn(TestResolver)
+    resolver = TestResolver(topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls)
     resolver._puppeteer_loaded = True
     resolver._wpt_loaded = True
+
+    if loader_cls == TestManifestLoader:
+        config = MockConfig(topsrcdir)
+        resolver.load_tests.reader = BuildReader(config)
     return resolver
 
 
 def test_load(resolver):
     assert len(resolver.tests_by_path) == 8
 
     assert len(resolver.tests_by_flavor['xpcshell']) == 3
     assert len(resolver.tests_by_flavor['mochitest-plain']) == 0
@@ -181,21 +192,24 @@ def test_resolve_under_path(resolver):
 
 def test_resolve_multiple_paths(resolver):
     result = list(resolver.resolve_tests(paths=['banana', 'dragonfruit']))
     assert len(result) == 4
 
 
 def test_resolve_support_files(resolver):
     expected_support_files = "\ndata/**\nxpcshell_updater.ini"
-    result = list(resolver.resolve_tests(paths=['dragonfruit']))
-    assert len(result) == 2
+    tests = list(resolver.resolve_tests(paths=['dragonfruit']))
+    assert len(tests) == 2
 
-    for test in result:
-        assert test['support-files'] == expected_support_files
+    for test in tests:
+        if test['manifest'].endswith('xpcshell_updater.ini'):
+            assert test['support-files'] == expected_support_files
+        else:
+            assert 'support-files' not in test
 
 
 def test_resolve_path_prefix(resolver):
     result = list(resolver._resolve(paths=['juniper']))
     assert len(result) == 1
 
 
 def test_cwd_children_only(resolver):