Bug 1242051 - Install test files to the objdir lazily rather than with each invocation of mach. r=gps
authorChris Manchester <cmanchester@mozilla.com>
Mon, 04 Apr 2016 14:56:52 -0700
changeset 316728 79b6b01de089fe7b54de10eb65c5690d66d0a658
parent 316727 e174b6bf5439095c7b28c48ff43fbd812c797882
child 316729 4271ef91a68621e3de091f455a18777fae0f7a0d
push id6048
push userkmoir@mozilla.com
push dateMon, 06 Jun 2016 19:02:08 +0000
treeherdermozilla-esr52@46d72a56c57d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1242051
milestone48.0a1
Bug 1242051 - Install test files to the objdir lazily rather than with each invocation of mach. r=gps This moves test installation for test files out of the monolithic install manifest for $objdir/_tests, and determines the test and support files to install based on the object derived from all-tests.json. Additionally, the files resulting from TEST_HARNESS_FILES are installed, as some tests will depend on them. As a result, the time to install tests when invoking the test runner will scale with the number of tests requested to run rather than the entire set of tests in the tree, resulting in significantly less overhead. MozReview-Commit-ID: LeIrUVh1yD4
Makefile.in
build/docs/test_manifests.rst
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/another-file.sjs
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/browser.ini
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/data/one.txt
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/data/two.txt
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/test_sub.js
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/mochitest.ini
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/moz.build
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/support-file.txt
python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/test_foo.js
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/another-file.sjs
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/browser.ini
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/data/one.txt
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/data/two.txt
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/test_sub.js
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/mochitest.ini
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/moz.build
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/support-file.txt
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/test_foo.js
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/another-file.sjs
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/browser.ini
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/data/one.txt
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/data/two.txt
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/test_sub.js
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/mochitest.ini
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/moz.build
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/support-file.txt
python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/test_foo.js
python/mozbuild/mozbuild/test/frontend/test_emitter.py
python/mozbuild/mozbuild/test/test_testing.py
python/mozbuild/mozbuild/testing.py
testing/mach_commands.py
testing/mochitest/mach_commands.py
testing/testsuite-targets.mk
testing/xpcshell/mach_commands.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -188,19 +188,28 @@ ifneq (,$(filter FasterMake+RecursiveMak
 	@# same directory, because that would blow up
 	$(if $(wildcard _build_manifests/install/$(subst /,_,$*)),$(if $(wildcard faster/install_$(subst /,_,$*)*),$(error FasterMake and RecursiveMake ends of the hybrid build system want to handle $*)))
 endif
 	$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
 
 # Dummy wrapper rule to allow the faster backend to piggy back
 $(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
 
-# For compatibility
 .PHONY: install-tests
-install-tests: install-_tests
+install-tests: install-test-files
+
+# We no longer run "make install-tests" directly before running tests, but we still
+# want to depend on things like config.status, hence this target.
+.PHONY: run-tests-deps
+run-tests-deps: $(install_manifest_depends)
+
+# Force --no-remove, because $objdir/_tests is handled by multiple manifests.
+.PHONY: install-test-files
+install-test-files:
+	$(call py_action,process_install_manifest,--no-remove _tests _build_manifests/install/_test_files)
 
 include $(topsrcdir)/build/moz-automation.mk
 
 # dist and _tests should be purged during cleaning. However, we don't want them
 # purged during PGO builds because they contain some auto-generated files.
 ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)
 GARBAGE_DIRS += dist _tests
 endif
--- a/build/docs/test_manifests.rst
+++ b/build/docs/test_manifests.rst
@@ -94,16 +94,28 @@ support-files
    ``data/**`` will match ``data/foo`` and ``data/subdir/bar``.
 
    Support files starting with ``/`` are placed in a root directory, rather
    than a location determined by the manifest location. For mochitests,
    this allows for the placement of files at the server root. The source
    file is selected from the base name (e.g., ``foo`` for ``/path/foo``).
    Files starting with ``/`` cannot be selected using globbing.
 
+   Some support files are used by tests across multiple directories. In
+   this case, a test depending on a support file from another directory
+   must note that dependency with the path to the required support file
+   in its own **support-files** entry. These use a syntax where paths
+   starting with ``!/`` will indicate the beginning of the path to a
+   shared support file starting from the root of the srcdir. For example,
+   if a manifest at ``dom/base/test/mochitest.ini`` has a support file,
+   ``dom/base/test/server-script.sjs``, and a mochitest in
+   ``dom/workers/test`` depends on that support file, the test manifest
+   at ``dom/workers/test/mochitest.ini`` must include
+   ``!/dom/base/test/server-script.sjs`` in its **support-files** entry.
+
 generated-files
    List of files that are generated as part of the build and don't exist in
    the source tree.
 
    The build system assumes that each manifest file, test file, and file
    listed in **head**, **tail**, and **support-files** is static and
    provided by the source tree (and not automatically generated as part
    of the build). This variable tells the build system not to make this
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -169,35 +169,42 @@ class WebIDLCollection(object):
 class TestManager(object):
     """Helps hold state related to tests."""
 
     def __init__(self, config):
         self.config = config
         self.topsrcdir = mozpath.normpath(config.topsrcdir)
 
         self.tests_by_path = defaultdict(list)
+        self.installs_by_path = defaultdict(list)
+        self.deferred_installs = set()
 
-    def add(self, t, flavor=None, topsrcdir=None):
+    def add(self, t, flavor, topsrcdir):
         t = dict(t)
         t['flavor'] = flavor
 
-        if topsrcdir is None:
-            topsrcdir = self.topsrcdir
-        else:
-            topsrcdir = mozpath.normpath(topsrcdir)
-
         path = mozpath.normpath(t['path'])
         assert mozpath.basedir(path, [topsrcdir])
 
         key = path[len(topsrcdir)+1:]
         t['file_relpath'] = key
         t['dir_relpath'] = mozpath.dirname(key)
 
         self.tests_by_path[key].append(t)
 
+    def add_installs(self, obj, topsrcdir):
+        for src, (dest, _) in obj.installs.iteritems():
+            key = src[len(topsrcdir)+1:]
+            self.installs_by_path[key].append((src, dest))
+        for src, pat, dest in obj.pattern_installs:
+            key = mozpath.join(src[len(topsrcdir)+1:], pat)
+            self.installs_by_path[key].append((src, pat, dest))
+        for path in obj.deferred_installs:
+            self.deferred_installs.add(path[2:])
+
 
 class BinariesCollection(object):
     """Tracks state of binaries produced by the build."""
 
     def __init__(self):
         self.shared_libraries = []
         self.programs = []
 
@@ -213,18 +220,18 @@ class CommonBackend(BuildBackend):
         self._configs = set()
         self._ipdl_sources = set()
 
     def consume_object(self, obj):
         self._configs.add(obj.config)
 
         if isinstance(obj, TestManifest):
             for test in obj.tests:
-                self._test_manager.add(test, flavor=obj.flavor,
-                    topsrcdir=obj.topsrcdir)
+                self._test_manager.add(test, obj.flavor, obj.topsrcdir)
+            self._test_manager.add_installs(obj, obj.topsrcdir)
 
         elif isinstance(obj, XPIDLFile):
             # TODO bug 1240134 tracks not processing XPIDL files during
             # artifact builds.
             self._idl_manager.register_idl(obj)
 
         elif isinstance(obj, ConfigFileSubstitution):
             # Do not handle ConfigFileSubstitution for Makefiles. Leave that
@@ -354,16 +361,24 @@ class CommonBackend(BuildBackend):
         for config in self._configs:
             self.backend_input_files.add(config.source)
 
         # Write out a machine-readable file describing every test.
         topobjdir = self.environment.topobjdir
         with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
             json.dump(self._test_manager.tests_by_path, fh)
 
+        path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
+        with self._write_file(path) as fh:
+            json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
+                       if k in self._test_manager.deferred_installs},
+                      fh,
+                      sort_keys=True,
+                      indent=4)
+
         # Write out a machine-readable file describing binaries.
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
             d = {
                 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
                 'programs': [p.to_dict() for p in self._binaries.programs],
             }
             json.dump(d, fh, sort_keys=True, indent=4)
 
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1037,32 +1037,32 @@ class RecursiveMakeBackend(CommonBackend
         self.backend_input_files.add(mozpath.join(obj.topsrcdir,
             obj.manifest_relpath))
 
         # Don't allow files to be defined multiple times unless it is allowed.
         # We currently allow duplicates for non-test files or test files if
         # the manifest is listed as a duplicate.
         for source, (dest, is_test) in obj.installs.items():
             try:
-                self._install_manifests['_tests'].add_symlink(source, dest)
+                self._install_manifests['_test_files'].add_symlink(source, dest)
             except ValueError:
                 if not obj.dupe_manifest and is_test:
                     raise
 
         for base, pattern, dest in obj.pattern_installs:
             try:
-                self._install_manifests['_tests'].add_pattern_symlink(base,
+                self._install_manifests['_test_files'].add_pattern_symlink(base,
                     pattern, dest)
             except ValueError:
                 if not obj.dupe_manifest:
                     raise
 
         for dest in obj.external_installs:
             try:
-                self._install_manifests['_tests'].add_optional_exists(dest)
+                self._install_manifests['_test_files'].add_optional_exists(dest)
             except ValueError:
                 if not obj.dupe_manifest:
                     raise
 
         m = self._test_manifests.setdefault(obj.flavor,
             (obj.install_prefix, set()))
         m[1].add(obj.manifest_obj_relpath)
 
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -21,18 +21,22 @@ from collections import (
 
 try:
     import psutil
 except Exception:
     psutil = None
 
 from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
 
+import mozpack.path as mozpath
+
 from ..base import MozbuildObject
 
+from ..testing import install_test_files
+
 from ..compilation.warnings import (
     WarningsCollector,
     WarningsDatabase,
 )
 
 from textwrap import TextWrapper
 
 INSTALL_TESTS_CLOBBER = ''.join([TextWrapper().fill(line) + '\n' for line in
@@ -648,22 +652,24 @@ class CCacheStats(object):
             return '%.1f Mbytes' % (float(v) / CCacheStats.MiB)
         else:
             return '%.1f Kbytes' % (float(v) / CCacheStats.KiB)
 
 
 class BuildDriver(MozbuildObject):
     """Provides a high-level API for build actions."""
 
-    def install_tests(self, remove=True):
-        """Install test files (through manifest)."""
+    def install_tests(self, test_objs):
+        """Install test files."""
 
         if self.is_clobber_needed():
             print(INSTALL_TESTS_CLOBBER.format(
                   clobber_file=os.path.join(self.topobjdir, 'CLOBBER')))
             sys.exit(1)
 
-        env = {}
-        if not remove:
-            env[b'NO_REMOVE'] = b'1'
-
-        self._run_make(target='install-tests', append_env=env, pass_thru=True,
-            print_directory=False)
+        if not test_objs:
+            # If we don't actually have a list of tests to install we install
+            # test and support files wholesale.
+            self._run_make(target='install-test-files', pass_thru=True,
+                           print_directory=False)
+        else:
+            install_test_files(mozpath.normpath(self.topsrcdir), self.topobjdir,
+                               '_tests', test_objs)
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -556,16 +556,20 @@ class TestManifest(ContextDerived):
 
         # Where all files for this manifest flavor are installed in the unified
         # test package directory.
         'install_prefix',
 
         # Set of files provided by an external mechanism.
         'external_installs',
 
+        # Set of files required by multiple test directories, whose installation
+        # will be resolved when running tests.
+        'deferred_installs',
+
         # The full path of this manifest file.
         'path',
 
         # The directory where this manifest is defined.
         'directory',
 
         # The parsed manifestparser.TestManifest instance.
         'manifest',
@@ -597,16 +601,17 @@ class TestManifest(ContextDerived):
         self.install_prefix = install_prefix
         self.manifest_relpath = relpath
         self.manifest_obj_relpath = relpath
         self.dupe_manifest = dupe_manifest
         self.installs = {}
         self.pattern_installs = []
         self.tests = []
         self.external_installs = set()
+        self.deferred_installs = set()
 
 
 class LocalInclude(ContextDerived):
     """Describes an individual local include path."""
 
     __slots__ = (
         'path',
     )
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -78,17 +78,17 @@ from mozpack.chrome.manifest import (
 )
 
 from .reader import SandboxValidationError
 
 from ..testing import (
     TEST_MANIFESTS,
     REFTEST_FLAVORS,
     WEB_PLATFORM_TESTS_FLAVORS,
-    convert_support_files,
+    SupportFilesConverter,
 )
 
 from .context import (
     Context,
     SourcePath,
     ObjDirPath,
     Path,
     SubContext,
@@ -135,16 +135,17 @@ class TreeMetadataEmitter(LoggingMixin):
         if os.path.exists(subconfigures):
             paths = open(subconfigures).read().splitlines()
         self._external_paths = set(mozpath.normsep(d) for d in paths)
         # Add security/nss manually, since it doesn't have a subconfigure.
         self._external_paths.add('security/nss')
 
         self._emitter_time = 0.0
         self._object_count = 0
+        self._test_files_converter = SupportFilesConverter()
 
     def summary(self):
         return ExecutionSummary(
             'Processed into {object_count:d} build config descriptors in '
             '{execution_time:.2f}s',
             execution_time=self._emitter_time,
             object_count=self._object_count)
 
@@ -1092,33 +1093,34 @@ class TreeMetadataEmitter(LoggingMixin):
             out_dir = mozpath.join(install_prefix, manifest_reldir)
             if 'install-to-subdir' in defaults:
                 # This is terrible, but what are you going to do?
                 out_dir = mozpath.join(out_dir, defaults['install-to-subdir'])
                 obj.manifest_obj_relpath = mozpath.join(manifest_reldir,
                                                         defaults['install-to-subdir'],
                                                         mozpath.basename(path))
 
-            # Keep a set of already seen support file patterns, because
-            # repeatedly processing the patterns from the default section
-            # for every test is quite costly (see bug 922517).
-            extras = (('head', set()),
-                      ('tail', set()),
-                      ('support-files', set()),
-                      ('generated-files', set()))
             def process_support_files(test):
-                patterns, installs, external = convert_support_files(extras, test,
-                                                                     install_root,
-                                                                     manifest_dir,
-                                                                     out_dir)
+                install_info = self._test_files_converter.convert_support_files(
+                    test, install_root, manifest_dir, out_dir)
 
-                obj.pattern_installs.extend(patterns)
-                for source, dest in installs:
+                obj.pattern_installs.extend(install_info.pattern_installs)
+                for source, dest in install_info.installs:
                     obj.installs[source] = (dest, False)
-                obj.external_installs |= external
+                obj.external_installs |= install_info.external_installs
+                for install_path in install_info.deferred_installs:
+                    if all(['*' not in install_path,
+                            not os.path.isfile(mozpath.join(context.config.topsrcdir,
+                                                            install_path[2:])),
+                            install_path not in install_info.external_installs]):
+                        raise SandboxValidationError('Error processing test '
+                           'manifest %s: entry in support-files not present '
+                           'in the srcdir: %s' % (path, install_path), context)
+
+                obj.deferred_installs |= install_info.deferred_installs
 
             for test in filtered:
                 obj.tests.append(test)
 
                 # Some test files are compiled and should not be copied into the
                 # test package. They function as identifiers rather than files.
                 if package_tests:
                     manifest_relpath = mozpath.relpath(test['path'],
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/browser.ini
@@ -0,0 +1,6 @@
+[DEFAULT]
+support-files =
+  another-file.sjs
+  data/**
+
+[test_sub.js]
\ No newline at end of file
new file mode 100644
new file mode 100644
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/mochitest.ini
@@ -0,0 +1,8 @@
+[DEFAULT]
+support-files =
+  support-file.txt
+  !/child/test_sub.js
+  !/child/another-file.sjs
+  !/child/data/**
+
+[test_foo.js]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += ['mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['child/browser.ini']
new file mode 100644
new file mode 100644
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -534,24 +534,64 @@ class TestRecursiveMakeBackend(BackendTe
             self.assertIn('dir1/test_bar.js', o)
 
             self.assertEqual(len(o['xpcshell.js']), 1)
 
     def test_test_manifest_pattern_matches_recorded(self):
         """Pattern matches in test manifests' support-files should be recorded."""
         env = self._consume('test-manifests-written', RecursiveMakeBackend)
         m = InstallManifest(path=mozpath.join(env.topobjdir,
-            '_build_manifests', 'install', '_tests'))
+            '_build_manifests', 'install', '_test_files'))
 
         # This is not the most robust test in the world, but it gets the job
         # done.
         entries = [e for e in m._dests.keys() if '**' in e]
         self.assertEqual(len(entries), 1)
         self.assertIn('support/**', entries[0])
 
+    def test_test_manifest_deffered_installs_written(self):
+        """Shared support files are written to their own data file by the backend."""
+        env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
+        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+        self.assertTrue(os.path.exists(all_tests_path))
+        test_installs_path = mozpath.join(env.topobjdir, 'test-installs.json')
+
+        with open(test_installs_path, 'r') as fh:
+            test_installs = json.load(fh)
+
+        self.assertEqual(set(test_installs.keys()),
+                         set(['child/test_sub.js',
+                              'child/data/**',
+                              'child/another-file.sjs']))
+        for key in test_installs.keys():
+            self.assertIn(key, test_installs)
+
+        test_files_manifest = mozpath.join(env.topobjdir,
+                                           '_build_manifests',
+                                           'install',
+                                           '_test_files')
+
+        # First, read the generated for ini manifest contents.
+        m = InstallManifest(path=test_files_manifest)
+
+        # Then, synthesize one from the test-installs.json file. This should
+        # allow us to re-create a subset of the above.
+        synthesized_manifest = InstallManifest()
+        for item, installs in test_installs.items():
+            for install_info in installs:
+                if len(install_info) == 3:
+                    synthesized_manifest.add_pattern_symlink(*install_info)
+                if len(install_info) == 2:
+                    synthesized_manifest.add_symlink(*install_info)
+
+        self.assertEqual(len(synthesized_manifest), 3)
+        for item, info in synthesized_manifest._dests.items():
+            self.assertIn(item, m)
+            self.assertEqual(info, m._dests[item])
+
     def test_xpidl_generation(self):
         """Ensure xpidl files and directories are written out."""
         env = self._consume('xpidl', RecursiveMakeBackend)
 
         # Install manifests should contain entries.
         install_dir = mozpath.join(env.topobjdir, '_build_manifests',
             'install')
         self.assertTrue(os.path.isfile(mozpath.join(install_dir, 'dist_idl')))
@@ -764,17 +804,17 @@ class TestRecursiveMakeBackend(BackendTe
 
         self.assertIn('JAR_MANIFEST := %s/jar.mn' % env.topsrcdir, lines)
 
     def test_test_manifests_duplicate_support_files(self):
         """Ensure duplicate support-files in test manifests work."""
         env = self._consume('test-manifests-duplicate-support-files',
             RecursiveMakeBackend)
 
-        p = os.path.join(env.topobjdir, '_build_manifests', 'install', '_tests')
+        p = os.path.join(env.topobjdir, '_build_manifests', 'install', '_test_files')
         m = InstallManifest(p)
         self.assertIn('testing/mochitest/tests/support-file.txt', m)
 
     def test_android_eclipse(self):
         env = self._consume('android_eclipse', RecursiveMakeBackend)
 
         with open(mozpath.join(env.topobjdir, 'backend.mk'), 'rb') as fh:
             lines = fh.readlines()
@@ -819,17 +859,17 @@ class TestRecursiveMakeBackend(BackendTe
             o = json.load(fh)
 
             self.assertIn('mochitest.js', o)
             self.assertIn('not_packaged.java', o)
 
         man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
         self.assertTrue(os.path.isdir(man_dir))
 
-        full = mozpath.join(man_dir, '_tests')
+        full = mozpath.join(man_dir, '_test_files')
         self.assertTrue(os.path.exists(full))
 
         m = InstallManifest(path=full)
 
         # Only mochitest.js should be in the install manifest.
         self.assertTrue('testing/mochitest/tests/mochitest.js' in m)
 
         # The path is odd here because we do not normalize at test manifest
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/browser.ini
@@ -0,0 +1,6 @@
+[DEFAULT]
+support-files =
+  another-file.sjs
+  data/**
+
+[test_sub.js]
\ No newline at end of file
new file mode 100644
new file mode 100644
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/mochitest.ini
@@ -0,0 +1,9 @@
+[DEFAULT]
+support-files =
+  support-file.txt
+  !/child/test_sub.js
+  !/child/another-file.sjs
+  !/child/data/**
+  !/does/not/exist.sjs
+
+[test_foo.js]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += ['mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['child/browser.ini']
new file mode 100644
new file mode 100644
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/browser.ini
@@ -0,0 +1,6 @@
+[DEFAULT]
+support-files =
+  another-file.sjs
+  data/**
+
+[test_sub.js]
\ No newline at end of file
new file mode 100644
new file mode 100644
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/mochitest.ini
@@ -0,0 +1,8 @@
+[DEFAULT]
+support-files =
+  support-file.txt
+  !/child/test_sub.js
+  !/child/another-file.sjs
+  !/child/data/**
+
+[test_foo.js]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += ['mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['child/browser.ini']
new file mode 100644
new file mode 100644
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -439,16 +439,43 @@ class TestEmitterBasic(unittest.TestCase
         expected = [
             mozpath.normpath(mozpath.join(o.install_prefix, "../.well-known/foo.txt")),
             mozpath.join(o.install_prefix, "absolute-support.ini"),
             mozpath.join(o.install_prefix, "test_file.js"),
         ]
         paths = sorted([v[0] for v in o.installs.values()])
         self.assertEqual(paths, expected)
 
+    def test_test_manifest_shared_support_files(self):
+        """Support files starting with '!' are given separate treatment, so their
+        installation can be resolved when running tests.
+        """
+        reader = self.reader('test-manifest-shared-support')
+        supported, child = self.read_topsrcdir(reader)
+
+        expected_deferred_installs = {
+            '!/child/test_sub.js',
+            '!/child/another-file.sjs',
+            '!/child/data/**',
+        }
+
+        self.assertEqual(len(supported.installs), 3)
+        self.assertEqual(set(supported.deferred_installs),
+                         expected_deferred_installs)
+        self.assertEqual(len(child.installs), 3)
+        self.assertEqual(len(child.pattern_installs), 1)
+
+    def test_test_manifest_deffered_install_missing(self):
+        """A non-existent shared support file reference produces an error."""
+        reader = self.reader('test-manifest-shared-missing')
+
+        with self.assertRaisesRegexp(SandboxValidationError,
+                                     'entry in support-files not present in the srcdir'):
+            self.read_topsrcdir(reader)
+
     def test_test_manifest_install_to_subdir(self):
         """ """
         reader = self.reader('test-manifest-install-subdir')
 
         objs = self.read_topsrcdir(reader)
         self.assertEqual(len(objs), 1)
         o = objs[0]
         self.assertEqual(len(o.installs), 3)
--- a/python/mozbuild/mozbuild/test/test_testing.py
+++ b/python/mozbuild/mozbuild/test/test_testing.py
@@ -234,16 +234,20 @@ class TestTestResolver(Base):
         topobjdir = tempfile.mkdtemp()
         self._temp_dirs.append(topobjdir)
 
         with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh:
             fh.write(ALL_TESTS_JSON)
 
         o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)
 
+        # Monkey patch the test resolver to avoid tests failing to find make
+        # due to our fake topscrdir.
+        TestResolver._run_make = lambda *a, **b: None
+
         return o._spawn(TestResolver)
 
     def test_cwd_children_only(self):
         """If cwd is defined, only resolve tests under the specified cwd."""
         r = self._get_resolver()
 
         # Pretend we're under '/services' and ask for 'common'. This should
         # pick up all tests from '/services/common'
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -5,16 +5,19 @@
 from __future__ import absolute_import, unicode_literals
 
 import json
 import os
 import sys
 
 import mozpack.path as mozpath
 
+from mozpack.copier import FileCopier
+from mozpack.manifests import InstallManifest
+
 from .base import MozbuildObject
 from .util import OrderedDefaultDict
 from collections import defaultdict
 
 import manifestparser
 import reftest
 
 def rewrite_test_base(test, new_base, honor_install_to_subdir=False):
@@ -162,16 +165,22 @@ class TestMetadata(object):
 
 
 class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
+        # If installing tests is going to result in re-generating the build
+        # backend, we need to do this here, so that the updated contents of
+        # all-tests.json make it to the set of tests to run.
+        self._run_make(target='run-tests-deps', pass_thru=True,
+                       print_directory=False)
+
         self._tests = TestMetadata(filename=os.path.join(self.topobjdir,
             'all-tests.json'))
         self._test_rewrites = {
             'a11y': os.path.join(self.topobjdir, '_tests', 'testing',
                 'mochitest', 'a11y'),
             'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing',
                 'mochitest', 'browser'),
             'jetpack-package': os.path.join(self.topobjdir, '_tests', 'testing',
@@ -282,79 +291,194 @@ REFTEST_FLAVORS = ('crashtest', 'reftest
 WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',)
 
 def all_test_flavors():
     return ([v[0] for v in TEST_MANIFESTS.values()] +
             list(REFTEST_FLAVORS) +
             list(WEB_PLATFORM_TESTS_FLAVORS) +
             ['python'])
 
-def convert_support_files(extras, test, install_root, manifest_dir, out_dir):
-    # Processes a "support-files" entry from a test object and returns
-    # the installs to perform for this test object.
-    #
-    # Arguments:
-    #  extras - Tuples used for the basis of memoization (the same support-files
-    #           in the same manifest always have the same effect).
-    #  test - The test object to process.
-    #  install_root - The directory under $objdir/_tests that will contain
-    #                 the tests for this harness (examples are "testing/mochitest",
-    #                 "xpcshell").
-    #  manifest_dir - Absoulute path to the (srcdir) directory containing the
-    #                 manifest that included this test
-    #  out_dir - The path relative to $objdir/_tests used as the destination for the
-    #            test, based on the relative path to the manifest in the srcdir,
-    #            the install_root, and 'install-to-subdir', if present in the manifest.
-    pattern_installs, installs, external = [], [], set()
-    for thing, seen in extras:
-        value = test.get(thing, '')
-        # We need to memoize on the basis of both the path and the output
-        # directory for the benefit of tests specifying 'install-to-subdir'.
-        if (value, out_dir) in seen:
+class TestInstallInfo(object):
+    def __init__(self):
+        self.pattern_installs = []
+        self.installs = []
+        self.external_installs = set()
+        self.deferred_installs = set()
+
+    def __ior__(self, other):
+        self.pattern_installs.extend(other.pattern_installs)
+        self.installs.extend(other.installs)
+        self.external_installs |= other.external_installs
+        self.deferred_installs |= other.deferred_installs
+        return self
+
+class SupportFilesConverter(object):
+    """Processes a "support-files" entry from a test object, either from
+    a parsed object from a test manifests or its representation in
+    moz.build and returns the installs to perform for this test object.
+
+    Processing the same support files multiple times will not have any further
+    effect, and the structure of the parsed objects from manifests will have a
+    lot of repeated entries, so this class takes care of memoizing.
+    """
+    def __init__(self):
+        self._fields = (('head', set()),
+                        ('tail', set()),
+                        ('support-files', set()),
+                        ('generated-files', set()))
+
+    def convert_support_files(self, test, install_root, manifest_dir, out_dir):
+        # Arguments:
+        #  test - The test object to process.
+        #  install_root - The directory under $objdir/_tests that will contain
+        #                 the tests for this harness (examples are "testing/mochitest",
+        #                 "xpcshell").
+        #  manifest_dir - Absoulute path to the (srcdir) directory containing the
+        #                 manifest that included this test
+        #  out_dir - The path relative to $objdir/_tests used as the destination for the
+        #            test, based on the relative path to the manifest in the srcdir,
+        #            the install_root, and 'install-to-subdir', if present in the manifest.
+        info = TestInstallInfo()
+        for thing, seen in self._fields:
+            value = test.get(thing, '')
+            # We need to memoize on the basis of both the path and the output
+            # directory for the benefit of tests specifying 'install-to-subdir'.
+            if (value, out_dir) in seen:
+                continue
+            seen.add((value, out_dir))
+            for pattern in value.split():
+                if thing == 'generated-files':
+                    info.external_installs.add(mozpath.normpath(mozpath.join(out_dir, pattern)))
+                # '!' indicates our syntax for inter-directory support file
+                # dependencies. These receive special handling in the backend.
+                elif pattern[0] == '!':
+                    info.deferred_installs.add(pattern)
+                # We only support globbing on support-files because
+                # the harness doesn't support * for head and tail.
+                elif '*' in pattern and thing == 'support-files':
+                    info.pattern_installs.append((manifest_dir, pattern, out_dir))
+                # "absolute" paths identify files that are to be
+                # placed in the install_root directory (no globs)
+                elif pattern[0] == '/':
+                    full = mozpath.normpath(mozpath.join(manifest_dir,
+                                                         mozpath.basename(pattern)))
+                    info.installs.append((full, mozpath.join(install_root, pattern[1:])))
+                else:
+                    full = mozpath.normpath(mozpath.join(manifest_dir, pattern))
+                    dest_path = mozpath.join(out_dir, pattern)
+
+                    # If the path resolves to a different directory
+                    # tree, we take special behavior depending on the
+                    # entry type.
+                    if not full.startswith(manifest_dir):
+                        # If it's a support file, we install the file
+                        # into the current destination directory.
+                        # This implementation makes installing things
+                        # with custom prefixes impossible. If this is
+                        # needed, we can add support for that via a
+                        # special syntax later.
+                        if thing == 'support-files':
+                            dest_path = mozpath.join(out_dir,
+                                                     os.path.basename(pattern))
+                        # If it's not a support file, we ignore it.
+                        # This preserves old behavior so things like
+                        # head files doesn't get installed multiple
+                        # times.
+                        else:
+                            continue
+                    info.installs.append((full, mozpath.normpath(dest_path)))
+        return info
+
+def _resolve_installs(paths, topobjdir, manifest):
+    """Using the given paths as keys, find any unresolved installs noted
+    by the build backend corresponding to those keys, and add them
+    to the given manifest.
+    """
+    filename = os.path.join(topobjdir, 'test-installs.json')
+    with open(filename, 'r') as fh:
+        resolved_installs = json.load(fh)
+
+    for path in paths:
+        path = path[2:]
+        if path not in resolved_installs:
+            raise Exception('A cross-directory support file path noted in a '
+                'test manifest does not appear in any other manifest.\n "%s" '
+                'must appear in another test manifest to specify an install '
+                'for "!/%s".' % (path, path))
+        installs = resolved_installs[path]
+        for install_info in installs:
+            try:
+                if len(install_info) == 3:
+                    manifest.add_pattern_symlink(*install_info)
+                if len(install_info) == 2:
+                    manifest.add_symlink(*install_info)
+            except ValueError:
+                # A duplicate value here is pretty likely when running
+                # multiple directories at once, and harmless.
+                pass
+
+def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
+    """Installs the requested test files to the objdir. This is invoked by
+    test runners to avoid installing tens of thousands of test files when
+    only a few tests need to be run.
+    """
+    flavor_info = {flavor: (root, prefix, install)
+                   for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
+    objdir_dest = mozpath.join(topobjdir, tests_root)
+
+    converter = SupportFilesConverter()
+    install_info = TestInstallInfo()
+    for o in test_objs:
+        flavor = o['flavor']
+        if flavor not in flavor_info:
+            # This is a test flavor that isn't installed by the build system.
             continue
-        seen.add((value, out_dir))
-        for pattern in value.split():
-            if thing == 'generated-files':
-                external.add(mozpath.join(out_dir, pattern))
-            # We only support globbing on support-files because
-            # the harness doesn't support * for head and tail.
-            elif '*' in pattern and thing == 'support-files':
-                pattern_installs.append((manifest_dir, pattern, out_dir))
-            # "absolute" paths identify files that are to be
-            # placed in the install_root directory (no globs)
-            elif pattern[0] == '/':
-                full = mozpath.normpath(mozpath.join(manifest_dir,
-                                                     mozpath.basename(pattern)))
-                installs.append((full, mozpath.join(install_root, pattern[1:])))
-            else:
-                full = mozpath.normpath(mozpath.join(manifest_dir, pattern))
-                dest_path = mozpath.join(out_dir, pattern)
+        root, prefix, install = flavor_info[flavor]
+        if not install:
+            # This flavor isn't installed to the objdir.
+            continue
+
+        manifest_path = o['manifest']
+        manifest_dir = mozpath.dirname(manifest_path)
+
+        out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
+        file_relpath = o['file_relpath']
+        source = mozpath.join(topsrcdir, file_relpath)
+        dest = mozpath.join(root, prefix, file_relpath)
+        if 'install-to-subdir' in o:
+            out_dir = mozpath.join(out_dir, o['install-to-subdir'])
+            manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
+            dest = mozpath.join(out_dir, manifest_relpath)
+
+        install_info.installs.append((source, dest))
+        install_info |= converter.convert_support_files(o, root,
+                                                        manifest_dir,
+                                                        out_dir)
 
-                # If the path resolves to a different directory
-                # tree, we take special behavior depending on the
-                # entry type.
-                if not full.startswith(manifest_dir):
-                    # If it's a support file, we install the file
-                    # into the current destination directory.
-                    # This implementation makes installing things
-                    # with custom prefixes impossible. If this is
-                    # needed, we can add support for that via a
-                    # special syntax later.
-                    if thing == 'support-files':
-                        dest_path = mozpath.join(out_dir,
-                                                 os.path.basename(pattern))
-                    # If it's not a support file, we ignore it.
-                    # This preserves old behavior so things like
-                    # head files doesn't get installed multiple
-                    # times.
-                    else:
-                        continue
-                installs.append((full, mozpath.normpath(dest_path)))
+    manifest = InstallManifest()
+
+    for source, dest in set(install_info.installs):
+        if dest in install_info.external_installs:
+            continue
+        manifest.add_symlink(source, dest)
+    for base, pattern, dest in install_info.pattern_installs:
+        manifest.add_pattern_symlink(base, pattern, dest)
+
+    _resolve_installs(install_info.deferred_installs, topobjdir, manifest)
 
-    return pattern_installs, installs, external
+    # Harness files are treated as a monolith and installed each time we run tests.
+    # Fortunately there are not very many.
+    manifest |= InstallManifest(mozpath.join(topobjdir,
+                                             '_build_manifests',
+                                             'install', tests_root))
+    copier = FileCopier()
+    manifest.populate_registry(copier)
+    copier.copy(objdir_dest,
+                remove_unaccounted=False)
+
 
 # Convenience methods for test manifest reading.
 def read_manifestparser_manifest(context, manifest_path):
     path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
     return manifestparser.TestManifest(manifests=[path], strict=True,
                                        rootdir=context.config.topsrcdir,
                                        finder=context._finder)
 
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -607,17 +607,16 @@ class PushToTry(MachCommandBase):
 
         The command requires either its own mercurial extension ("push-to-try",
         installable from mach mercurial-setup) or a git repo using git-cinnabar
         (available at https://github.com/glandium/git-cinnabar).
 
         """
 
         from mozbuild.testing import TestResolver
-        from mozbuild.controller.building import BuildDriver
         from autotry import AutoTry
 
         print("mach try is under development, please file bugs blocking 1149670.")
 
         resolver_func = lambda: self._spawn(TestResolver)
         at = AutoTry(self.topsrcdir, resolver_func, self._mach_context)
 
         if kwargs["list"]:
@@ -640,19 +639,16 @@ class PushToTry(MachCommandBase):
             sys.exit(1)
 
         if not any(kwargs[item] for item in ("paths", "tests", "tags")):
             kwargs["paths"], kwargs["tags"] = at.find_paths_and_tags(kwargs["verbose"])
 
         builds, platforms, tests, talos, paths, tags, extra = self.validate_args(**kwargs)
 
         if paths or tags:
-            driver = self._spawn(BuildDriver)
-            driver.install_tests(remove=False)
-
             paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
                      for item in paths]
             paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags)
 
             if not paths_by_flavor and not tests:
                 print("No tests were found when attempting to resolve paths:\n\n\t%s" %
                       paths)
                 sys.exit(1)
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -380,19 +380,16 @@ class MachCommands(MachCommandBase):
                     flavors = [fname]
                     break
         else:
             flavors = [f for f, v in ALL_FLAVORS.iteritems() if buildapp in v['enabled_apps']]
 
         from mozbuild.controller.building import BuildDriver
         self._ensure_state_subdir_exists('.')
 
-        driver = self._spawn(BuildDriver)
-        driver.install_tests(remove=False)
-
         test_paths = kwargs['test_paths']
         kwargs['test_paths'] = []
 
         if test_paths and buildapp == 'b2g':
             # In B2G there is often a 'gecko' directory, though topsrcdir is actually
             # elsewhere. This little hack makes test paths like 'gecko/dom' work, even if
             # GECKO_PATH is set in the .userconfig
             gecko_path = mozpath.abspath(mozpath.join(kwargs['b2gPath'], 'gecko'))
@@ -405,16 +402,19 @@ class MachCommands(MachCommandBase):
                         new_paths.append(tp)
                 test_paths = new_paths
 
         mochitest = self._spawn(MochitestRunner)
         tests = []
         if resolve_tests:
             tests = mochitest.resolve_tests(test_paths, test_objects, cwd=self._mach_context.cwd)
 
+        driver = self._spawn(BuildDriver)
+        driver.install_tests(tests)
+
         subsuite = kwargs.get('subsuite')
         if subsuite == 'default':
             kwargs['subsuite'] = None
 
         suites = defaultdict(list)
         unsupported = set()
         for test in tests:
             # Filter out non-mochitests and unsupported flavors.
@@ -524,26 +524,25 @@ class RobocopCommands(MachCommandBase):
         if not kwargs.get('robocopApk'):
             kwargs['robocopApk'] = os.path.join(self.topobjdir, 'mobile', 'android',
                                                 'tests', 'browser', 'robocop',
                                                 'robocop-debug.apk')
 
         from mozbuild.controller.building import BuildDriver
         self._ensure_state_subdir_exists('.')
 
-        driver = self._spawn(BuildDriver)
-        driver.install_tests(remove=False)
-
         test_paths = kwargs['test_paths']
         kwargs['test_paths'] = []
 
         from mozbuild.testing import TestResolver
         resolver = self._spawn(TestResolver)
         tests = list(resolver.resolve_tests(paths=test_paths, cwd=self._mach_context.cwd,
                                             flavor='instrumentation', subsuite='robocop'))
+        driver = self._spawn(BuildDriver)
+        driver.install_tests(tests)
 
         if len(tests) < 1:
             print(ROBOCOP_TESTS_NOT_FOUND.format('\n'.join(
                 sorted(list(test_paths)))))
             return 1
 
         from mozrunner.devices.android_device import grant_runtime_permissions
         grant_runtime_permissions(self, kwargs['app'])
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -238,17 +238,19 @@ ifeq ($(MOZ_BUILD_APP),mobile/android)
 stage-all: stage-android
 stage-all: stage-instrumentation-tests
 endif
 
 ifeq ($(MOZ_WIDGET_TOOLKIT),gonk)
 stage-all: stage-b2g
 endif
 
-make-stage-dir:
+# Prepare _tests before any of the other staging/packaging steps.
+# make-stage-dir is a prerequisite to all the stage-* targets in testsuite-targets.mk.
+make-stage-dir: install-test-files
 	rm -rf $(PKG_STAGE)
 	$(NSINSTALL) -D $(PKG_STAGE)
 	$(NSINSTALL) -D $(PKG_STAGE)/bin
 	$(NSINSTALL) -D $(PKG_STAGE)/bin/components
 	$(NSINSTALL) -D $(PKG_STAGE)/certs
 	$(NSINSTALL) -D $(PKG_STAGE)/config
 	$(NSINSTALL) -D $(PKG_STAGE)/jetpack
 	$(NSINSTALL) -D $(PKG_STAGE)/modules
--- a/testing/xpcshell/mach_commands.py
+++ b/testing/xpcshell/mach_commands.py
@@ -342,23 +342,24 @@ class MachCommands(MachCommandBase):
         from mozbuild.controller.building import BuildDriver
 
         if test_objects is not None:
             from manifestparser import TestManifest
             m = TestManifest()
             m.tests.extend(test_objects)
             params['manifest'] = m
 
+        driver = self._spawn(BuildDriver)
+        driver.install_tests(test_objects)
+
         # We should probably have a utility function to ensure the tree is
         # ready to run tests. Until then, we just create the state dir (in
         # case the tree wasn't built with mach).
         self._ensure_state_subdir_exists('.')
 
-        driver = self._spawn(BuildDriver)
-        driver.install_tests(remove=False)
 
         params['log'] = structured.commandline.setup_logging("XPCShellTests",
                                                              params,
                                                              {"mach": sys.stdout},
                                                              {"verbose": True})
 
         if conditions.is_android(self):
             from mozrunner.devices.android_device import verify_android_device