Bug 1320194 - Generate all-tests.pkl and related files when resolving tests r=mshal
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 01 Feb 2017 09:56:09 -0500
changeset 343050 b4344af7fc7c68032a1468798882bbcc9f73a227
parent 343049 b5bcf4edfbb07186d0d86ae9e9186e5f7e563bfa
child 343051 620f06960400e4d95da0ed5c9682edda3aa136ec
push id37423
push userahalberstadt@mozilla.com
push dateWed, 15 Feb 2017 15:16:15 +0000
treeherderautoland@b4344af7fc7c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1320194
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1320194 - Generate all-tests.pkl and related files when resolving tests r=mshal This replaces the 'run-tests-deps' make target with a python function that will directly read moz.build files, emit them with TestManifestEmitter, then consume them with TestManifestBackend. Because the TestResolver is the only place that actually reads the test metadata files, we can remove this logic from the CommonBackend as well. MozReview-Commit-ID: DXgMoeH5dKf MozReview-Commit-ID: HstZ57qkqf2
Makefile.in
build/gen_test_backend.py
python/moz.build
python/mozbuild/mozbuild/backend/base.py
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/testing.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -181,21 +181,16 @@ endif
 	$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
 
 # Dummy wrapper rule to allow the faster backend to piggy back
 $(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
 
 .PHONY: install-tests
 install-tests: install-test-files
 
-# We no longer run "make install-tests" directly before running tests, but we still
-# want to depend on things like config.status, hence this target.
-.PHONY: run-tests-deps
-run-tests-deps: $(install_manifest_depends)
-
 # Force --no-remove, because $objdir/_tests is handled by multiple manifests.
 .PHONY: install-test-files
 install-test-files:
 	$(call py_action,process_install_manifest,--no-remove _tests _build_manifests/install/_test_files)
 
 include $(topsrcdir)/build/moz-automation.mk
 
 # dist and _tests should be purged during cleaning. However, we don't want them
new file mode 100644
--- /dev/null
+++ b/build/gen_test_backend.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+from mozbuild.backend.test_manifest import TestManifestBackend
+from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+from mozbuild.frontend.emitter import TreeMetadataEmitter
+from mozbuild.frontend.reader import BuildReader, EmptyConfig
+
+
+def gen_test_backend():
+    build_obj = MozbuildObject.from_environment()
+    try:
+        config = build_obj.config_environment
+    except BuildEnvironmentNotFoundException:
+        print("No build detected, test metadata may be incomplete.")
+        config = EmptyConfig(build_obj.topsrcdir)
+        config.topobjdir = build_obj.topobjdir
+
+    reader = BuildReader(config)
+    emitter = TreeMetadataEmitter(config)
+    backend = TestManifestBackend(config)
+
+    context = reader.read_topsrcdir()
+    data = emitter.emit(context, emitfn=emitter._process_test_manifests)
+    backend.consume(data)
+
+
+if __name__ == '__main__':
+    sys.exit(gen_test_backend())
--- a/python/moz.build
+++ b/python/moz.build
@@ -18,12 +18,16 @@ SPHINX_PYTHON_PACKAGE_DIRS += [
     'mozversioncontrol/mozversioncontrol',
 ]
 
 SPHINX_TREES['mach'] = 'mach/docs'
 
 PYTHON_UNITTEST_MANIFESTS += [
     'mach/mach/test/python.ini',
     'mozbuild/dumbmake/test/python.ini',
-    'mozbuild/mozbuild/test/python.ini',
-    'mozbuild/mozpack/test/python.ini',
     'mozlint/test/python.ini',
 ]
+
+if CONFIG['MOZ_BUILD_APP']:
+    PYTHON_UNITTEST_MANIFESTS += [
+        'mozbuild/mozbuild/test/python.ini',
+        'mozbuild/mozpack/test/python.ini',
+    ]
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -21,33 +21,33 @@ from mach.mixin.logging import LoggingMi
 import mozpack.path as mozpath
 from ..preprocessor import Preprocessor
 from ..pythonutil import iter_modules_in_path
 from ..util import (
     FileAvoidWrite,
     simple_diff,
 )
 from ..frontend.data import ContextDerived
+from ..frontend.reader import EmptyConfig
 from .configenvironment import ConfigEnvironment
 from mozbuild.base import ExecutionSummary
 
 
 class BuildBackend(LoggingMixin):
     """Abstract base class for build backends.
 
     A build backend is merely a consumer of the build configuration (the output
     of the frontend processing). It does something with said data. What exactly
     is the discretion of the specific implementation.
     """
 
     __metaclass__ = ABCMeta
 
     def __init__(self, environment):
-        assert isinstance(environment, ConfigEnvironment)
-
+        assert isinstance(environment, (ConfigEnvironment, EmptyConfig))
         self.populate_logger()
 
         self.environment = environment
 
         # Files whose modification should cause a new read and backend
         # generation.
         self.backend_input_files = set()
 
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -6,17 +6,16 @@ from __future__ import absolute_import, 
 
 import itertools
 import json
 import os
 
 import mozpack.path as mozpath
 
 from mozbuild.backend.base import BuildBackend
-from mozbuild.backend.test_manifest import TestManifestBackend
 
 from mozbuild.frontend.context import (
     Context,
     Path,
     RenamedSourcePath,
     VARIABLES,
 )
 from mozbuild.frontend.data import (
@@ -27,31 +26,28 @@ from mozbuild.frontend.data import (
     IPDLFile,
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     GeneratedEventWebIDLFile,
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     SharedLibrary,
-    TestManifest,
     TestWebIDLFile,
     UnifiedSources,
     XPIDLFile,
     WebIDLFile,
 )
 from mozbuild.jar import (
     DeprecatedJarManifest,
     JarManifestParser,
 )
 from mozbuild.preprocessor import Preprocessor
 from mozpack.chrome.manifest import parse_manifest_line
 
-from collections import defaultdict
-
 from mozbuild.util import group_unified_files
 
 class XPIDLManager(object):
     """Helps manage XPCOM IDLs in the context of the build system."""
     def __init__(self, config):
         self.config = config
         self.topsrcdir = config.topsrcdir
         self.topobjdir = config.topobjdir
@@ -179,27 +175,20 @@ class CommonBackend(BuildBackend):
 
     def _init(self):
         self._idl_manager = XPIDLManager(self.environment)
         self._webidls = WebIDLCollection()
         self._binaries = BinariesCollection()
         self._configs = set()
         self._ipdl_sources = set()
 
-        # Temporarily compose a partial TestManifestBackend, soon test manifest
-        # processing will no longer be part of the build and this will be removed.
-        self._test_backend = TestManifestBackend(self.environment)
-
     def consume_object(self, obj):
         self._configs.add(obj.config)
 
-        if isinstance(obj, TestManifest):
-            self._test_backend.consume_object(obj)
-
-        elif isinstance(obj, XPIDLFile):
+        if isinstance(obj, XPIDLFile):
             # TODO bug 1240134 tracks not processing XPIDL files during
             # artifact builds.
             self._idl_manager.register_idl(obj)
 
         elif isinstance(obj, ConfigFileSubstitution):
             # Do not handle ConfigFileSubstitution for Makefiles. Leave that
             # to other
             if mozpath.basename(obj.output_path) == 'Makefile':
@@ -322,19 +311,16 @@ class CommonBackend(BuildBackend):
                                                           files_per_unified_file=16))
 
         self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
         self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
 
         for config in self._configs:
             self.backend_input_files.add(config.source)
 
-        # Write out a machine-readable file describing every test.
-        self._test_backend.consume_finished()
-
         # Write out a machine-readable file describing binaries.
         topobjdir = self.environment.topobjdir
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
             d = {
                 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
                 'programs': [p.to_dict() for p in self._binaries.programs],
             }
             json.dump(d, fh, sort_keys=True, indent=4)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -438,25 +438,22 @@ class RecursiveMakeBackend(CommonBackend
 
         if not isinstance(obj, ContextDerived):
             return False
 
         backend_file = self._get_backend_file_for(obj)
 
         consumed = CommonBackend.consume_object(self, obj)
 
-        # CommonBackend handles XPIDLFile and TestManifest, but we want to do
+        # CommonBackend handles XPIDLFile, but we want to do
         # some extra things for them.
         if isinstance(obj, XPIDLFile):
             backend_file.xpt_name = '%s.xpt' % obj.module
             self._idl_dirs.add(obj.relobjdir)
 
-        elif isinstance(obj, TestManifest):
-            self._process_test_manifest(obj, backend_file)
-
         # If CommonBackend acknowledged the object, we're done with it.
         if consumed:
             return True
 
         if not isinstance(obj, Defines):
             self.consume_object(obj.defines)
 
         if isinstance(obj, Linkable):
@@ -652,16 +649,19 @@ class RecursiveMakeBackend(CommonBackend
         elif isinstance(obj, AndroidExtraPackages):
             # Order does not matter.
             for p in sorted(set(obj.packages)):
                 backend_file.write('ANDROID_EXTRA_PACKAGES += %s\n' % p)
 
         elif isinstance(obj, ChromeManifestEntry):
             self._process_chrome_manifest_entry(obj, backend_file)
 
+        elif isinstance(obj, TestManifest):
+            self._process_test_manifest(obj, backend_file)
+
         else:
             return False
 
         return True
 
     def _fill_root_mk(self):
         """
         Create two files, root.mk and root-deps.mk, the first containing
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -13,16 +13,17 @@ from mozpack.manifests import (
     InstallManifest,
 )
 from mozunit import main
 
 from mozbuild.backend.recursivemake import (
     RecursiveMakeBackend,
     RecursiveMakeTraversal,
 )
+from mozbuild.backend.test_manifest import TestManifestBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
 
 import mozpack.path as mozpath
 
 
@@ -555,31 +556,39 @@ class TestRecursiveMakeBackend(BackendTe
         # done.
         entries = [e for e in m._dests.keys() if '**' in e]
         self.assertEqual(len(entries), 1)
         self.assertIn('support/**', entries[0])
 
     def test_test_manifest_deffered_installs_written(self):
         """Shared support files are written to their own data file by the backend."""
         env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
+
+        # First, read the generated for ini manifest contents.
+        test_files_manifest = mozpath.join(env.topobjdir,
+                                           '_build_manifests',
+                                           'install',
+                                           '_test_files')
+        m = InstallManifest(path=test_files_manifest)
+
+        # Then, synthesize one from the test-installs.pkl file. This should
+        # allow us to re-create a subset of the above.
+        env = self._consume('test-manifest-shared-support', TestManifestBackend)
         test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl')
 
         with open(test_installs_path, 'r') as fh:
             test_installs = pickle.load(fh)
 
-        test_files_manifest = mozpath.join(env.topobjdir,
-                                           '_build_manifests',
-                                           'install',
-                                           '_test_files')
+        self.assertEqual(set(test_installs.keys()),
+                         set(['child/test_sub.js',
+                              'child/data/**',
+                              'child/another-file.sjs']))
+        for key in test_installs.keys():
+            self.assertIn(key, test_installs)
 
-        # First, read the generated for ini manifest contents.
-        m = InstallManifest(path=test_files_manifest)
-
-        # Then, synthesize one from the test-installs.pkl file. This should
-        # allow us to re-create a subset of the above.
         synthesized_manifest = InstallManifest()
         for item, installs in test_installs.items():
             for install_info in installs:
                 if len(install_info) == 3:
                     synthesized_manifest.add_pattern_symlink(*install_info)
                 if len(install_info) == 2:
                     synthesized_manifest.add_symlink(*install_info)
 
@@ -936,24 +945,16 @@ class TestRecursiveMakeBackend(BackendTe
             # Destination and install manifest are relative to topobjdir.
             stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name)
             self.assertIn(command_template % (stem, stem), lines)
 
     def test_install_manifests_package_tests(self):
         """Ensure test suites honor package_tests=False."""
         env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
 
-        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
-        self.assertTrue(os.path.exists(all_tests_path))
-
-        with open(all_tests_path, 'rb') as fh:
-            o = pickle.load(fh)
-            self.assertIn('mochitest.js', o)
-            self.assertIn('not_packaged.java', o)
-
         man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
         self.assertTrue(os.path.isdir(man_dir))
 
         full = mozpath.join(man_dir, '_test_files')
         self.assertTrue(os.path.exists(full))
 
         m = InstallManifest(path=full)
 
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -180,18 +180,26 @@ class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
         # If installing tests is going to result in re-generating the build
         # backend, we need to do this here, so that the updated contents of
         # all-tests.pkl make it to the set of tests to run.
-        self._run_make(target='run-tests-deps', pass_thru=True,
-                       print_directory=False)
+        self._run_make(
+            target='backend.TestManifestBackend', pass_thru=True, print_directory=False,
+            filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'),
+            append_env={
+                b'PYTHON': self.virtualenv_manager.python_path,
+                b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
+                b'BACKEND_GENERATION_SCRIPT': mozpath.join(
+                    self.topsrcdir, 'build', 'gen_test_backend.py'),
+            },
+        )
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
                                                 'all-tests.pkl'),
                                    test_defaults=os.path.join(self.topobjdir,
                                                               'test-defaults.pkl'))
 
         self._test_rewrites = {
             'a11y': os.path.join(self.topobjdir, '_tests', 'testing',