Bug 1252931 - Add support for generic OBJDIR_FILES and OBJDIR_PP_FILES; r=gps
authorMike Shal <mshal@mozilla.com>
Thu, 17 Mar 2016 09:13:36 -0400
changeset 290402 b5d975e3f9824811d93b6eef54279a0875bf3efe
parent 290401 9061abb46bc777c543cb2ef4ec3c411866d25172
child 290403 491142a0995b238d872d058a2e61a9872408a0b5
push id74241
push usermshal@mozilla.com
push dateFri, 25 Mar 2016 21:27:10 +0000
treeherdermozilla-inbound@491142a0995b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1252931
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1252931 - Add support for generic OBJDIR_FILES and OBJDIR_PP_FILES; r=gps MozReview-Commit-ID: 6H435DeMHie
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -51,16 +51,18 @@ from ..frontend.data import (
     HostProgram,
     HostSimpleProgram,
     HostSources,
     InstallationTarget,
     JARManifest,
     JavaJarData,
     Library,
     LocalInclude,
+    ObjdirFiles,
+    ObjdirPreprocessedFiles,
     PerSourceFlag,
     Program,
     SharedLibrary,
     SimpleProgram,
     Sources,
     StaticLibrary,
     TestManifest,
     VariablePassthru,
@@ -577,17 +579,23 @@ class RecursiveMakeBackend(CommonBackend
         elif isinstance(obj, HostLibrary):
             self._process_host_library(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
 
         elif isinstance(obj, FinalTargetFiles):
             self._process_final_target_files(obj, obj.files, backend_file)
 
         elif isinstance(obj, FinalTargetPreprocessedFiles):
-            self._process_final_target_pp_files(obj, obj.files, backend_file)
+            self._process_final_target_pp_files(obj, obj.files, backend_file, 'DIST_FILES')
+
+        elif isinstance(obj, ObjdirFiles):
+            self._process_objdir_files(obj, obj.files, backend_file)
+
+        elif isinstance(obj, ObjdirPreprocessedFiles):
+            self._process_final_target_pp_files(obj, obj.files, backend_file, 'OBJDIR_PP_FILES')
 
         elif isinstance(obj, AndroidResDirs):
             # Order matters.
             for p in obj.paths:
                 backend_file.write('ANDROID_RES_DIRS += %s\n' % p.full_path)
 
         elif isinstance(obj, AndroidAssetsDirs):
             # Order matters.
@@ -1262,30 +1270,48 @@ class RecursiveMakeBackend(CommonBackend
                 tier = 'export' if obj.install_target == 'dist/include' else 'misc'
                 self._no_skip[tier].add(backend_file.relobjdir)
                 backend_file.write('%s_DEST := $(DEPTH)/%s\n'
                                    % (target_var,
                                       mozpath.join(target, path)))
                 backend_file.write('%s_TARGET := %s\n' % (target_var, tier))
                 backend_file.write('INSTALL_TARGETS += %s\n' % target_var)
 
-    def _process_final_target_pp_files(self, obj, files, backend_file):
+    def _process_final_target_pp_files(self, obj, files, backend_file, name):
         # Bug 1177710 - We'd like to install these via manifests as
         # preprocessed files. But they currently depend on non-standard flags
         # being added via some Makefiles, so for now we just pass them through
         # to the underlying Makefile.in.
+        #
+        # Note that if this becomes a manifest, OBJDIR_PP_FILES will likely
+        # still need to use PP_TARGETS internally because we can't have an
+        # install manifest for the root of the objdir.
+        for i, (path, files) in enumerate(files.walk()):
+            self._no_skip['misc'].add(backend_file.relobjdir)
+            var = '%s_%d' % (name, i)
+            for f in files:
+                backend_file.write('%s += %s\n' % (
+                    var, self._pretty_path(f, backend_file)))
+            backend_file.write('%s_PATH := $(DEPTH)/%s\n'
+                               % (var, mozpath.join(obj.install_target, path)))
+            backend_file.write('%s_TARGET := misc\n' % var)
+            backend_file.write('PP_TARGETS += %s\n' % var)
+
+    def _process_objdir_files(self, obj, files, backend_file):
+        # We can't use an install manifest for the root of the objdir, since it
+        # would delete all the other files that get put there by the build
+        # system.
         for i, (path, files) in enumerate(files.walk()):
             self._no_skip['misc'].add(backend_file.relobjdir)
             for f in files:
-                backend_file.write('DIST_FILES_%d += %s\n' % (
+                backend_file.write('OBJDIR_%d_FILES += %s\n' % (
                     i, self._pretty_path(f, backend_file)))
-            backend_file.write('DIST_FILES_%d_PATH := $(DEPTH)/%s\n'
-                               % (i, mozpath.join(obj.install_target, path)))
-            backend_file.write('DIST_FILES_%d_TARGET := misc\n' % i)
-            backend_file.write('PP_TARGETS += DIST_FILES_%d\n' % i)
+            backend_file.write('OBJDIR_%d_DEST := $(topobjdir)/%s\n' % (i, path))
+            backend_file.write('OBJDIR_%d_TARGET := misc\n' % i)
+            backend_file.write('INSTALL_TARGETS += OBJDIR_%d\n' % i)
 
     def _process_chrome_manifest_entry(self, obj, backend_file):
         fragment = Makefile()
         rule = fragment.create_rule(targets=['misc:'])
 
         top_level = mozpath.join(obj.install_target, 'chrome.manifest')
         if obj.path != top_level:
             args = [
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -1082,16 +1082,29 @@ VARIABLES = {
         """Disable the wrappers for STL which allow it to work with C++ exceptions
         disabled.
         """),
 
     'FINAL_TARGET_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
         """Like ``FINAL_TARGET_FILES``, with preprocessing.
         """),
 
+    'OBJDIR_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
+        """List of files to be installed anywhere in the objdir. Use sparingly.
+
+        ``OBJDIR_FILES`` is similar to FINAL_TARGET_FILES, but it allows copying
+        anywhere in the object directory. This is intended for various one-off
+        cases, not for general use. If you wish to add entries to OBJDIR_FILES,
+        please consult a build peer.
+        """),
+
+    'OBJDIR_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
+        """Like ``OBJDIR_FILES``, with preprocessing. Use sparingly.
+        """),
+
     'FINAL_LIBRARY': (unicode, unicode,
         """Library in which the objects of the current directory will be linked.
 
         This variable contains the name of a library, defined elsewhere with
         ``LIBRARY_NAME``, in which the objects of the current directory will be
         linked.
         """),
 
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -818,16 +818,46 @@ class FinalTargetPreprocessedFiles(Conte
     """
     __slots__ = ('files')
 
     def __init__(self, sandbox, files):
         ContextDerived.__init__(self, sandbox)
         self.files = files
 
 
+class ObjdirFiles(ContextDerived):
+    """Sandbox container object for OBJDIR_FILES, which is a
+    HierarchicalStringList.
+    """
+    __slots__ = ('files')
+
+    def __init__(self, sandbox, files):
+        ContextDerived.__init__(self, sandbox)
+        self.files = files
+
+    @property
+    def install_target(self):
+        return ''
+
+
+class ObjdirPreprocessedFiles(ContextDerived):
+    """Sandbox container object for OBJDIR_PP_FILES, which is a
+    HierarchicalStringList.
+    """
+    __slots__ = ('files')
+
+    def __init__(self, sandbox, files):
+        ContextDerived.__init__(self, sandbox)
+        self.files = files
+
+    @property
+    def install_target(self):
+        return ''
+
+
 class TestHarnessFiles(FinalTargetFiles):
     """Sandbox container object for TEST_HARNESS_FILES,
     which is a HierarchicalStringList.
     """
     @property
     def install_target(self):
         return '_tests'
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -48,16 +48,18 @@ from .data import (
     HostSimpleProgram,
     HostSources,
     InstallationTarget,
     IPDLFile,
     JARManifest,
     Library,
     Linkable,
     LocalInclude,
+    ObjdirFiles,
+    ObjdirPreprocessedFiles,
     PerSourceFlag,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     Program,
     SdkFiles,
     SharedLibrary,
     SimpleProgram,
     Sources,
@@ -825,16 +827,18 @@ class TreeMetadataEmitter(LoggingMixin):
         generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()])
 
         components = []
         for var, cls in (
             ('BRANDING_FILES', BrandingFiles),
             ('EXPORTS', Exports),
             ('FINAL_TARGET_FILES', FinalTargetFiles),
             ('FINAL_TARGET_PP_FILES', FinalTargetPreprocessedFiles),
+            ('OBJDIR_FILES', ObjdirFiles),
+            ('OBJDIR_PP_FILES', ObjdirPreprocessedFiles),
             ('SDK_FILES', SdkFiles),
             ('TEST_HARNESS_FILES', TestHarnessFiles),
         ):
             all_files = context.get(var)
             if not all_files:
                 continue
             if dist_install is False and var != 'TEST_HARNESS_FILES':
                 raise SandboxValidationError(
@@ -848,22 +852,23 @@ class TreeMetadataEmitter(LoggingMixin):
                         'Cannot install files to the root of TEST_HARNESS_FILES', context)
                 if base == 'components':
                     components.extend(files)
                 if base == 'defaults/pref':
                     has_prefs = True
                 if mozpath.split(base)[0] == 'res':
                     has_resources = True
                 for f in files:
-                    if (var == 'FINAL_TARGET_PP_FILES' and
+                    if ((var == 'FINAL_TARGET_PP_FILES' or
+                         var == 'OBJDIR_PP_FILES') and
                         not isinstance(f, SourcePath)):
                         raise SandboxValidationError(
                                 ('Only source directory paths allowed in ' +
-                                'FINAL_TARGET_PP_FILES: %s')
-                                % (f,), context)
+                                 '%s: %s')
+                                % (var, f,), context)
                     if not isinstance(f, ObjDirPath):
                         path = f.full_path
                         if '*' not in path and not os.path.exists(path):
                             raise SandboxValidationError(
                                 'File listed in %s does not exist: %s'
                                 % (var, path), context)
                     else:
                         # TODO: Bug 1254682 - The '/' check is to allow