Bug 1252931 - Add support for generic OBJDIR_FILES and OBJDIR_PP_FILES; r=gps
authorMike Shal <mshal@mozilla.com>
Thu, 17 Mar 2016 09:13:36 -0400
changeset 290501 b5d975e3f9824811d93b6eef54279a0875bf3efe
parent 290500 9061abb46bc777c543cb2ef4ec3c411866d25172
child 290502 491142a0995b238d872d058a2e61a9872408a0b5
push id19656
push usergwagner@mozilla.com
push dateMon, 04 Apr 2016 13:43:23 +0000
treeherderb2g-inbound@e99061fde28a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1252931
milestone48.0a1
Bug 1252931 - Add support for generic OBJDIR_FILES and OBJDIR_PP_FILES; r=gps MozReview-Commit-ID: 6H435DeMHie
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -51,16 +51,18 @@ from ..frontend.data import (
     HostProgram,
     HostSimpleProgram,
     HostSources,
     InstallationTarget,
     JARManifest,
     JavaJarData,
     Library,
     LocalInclude,
+    ObjdirFiles,
+    ObjdirPreprocessedFiles,
     PerSourceFlag,
     Program,
     SharedLibrary,
     SimpleProgram,
     Sources,
     StaticLibrary,
     TestManifest,
     VariablePassthru,
@@ -577,17 +579,23 @@ class RecursiveMakeBackend(CommonBackend
         elif isinstance(obj, HostLibrary):
             self._process_host_library(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
 
         elif isinstance(obj, FinalTargetFiles):
             self._process_final_target_files(obj, obj.files, backend_file)
 
         elif isinstance(obj, FinalTargetPreprocessedFiles):
-            self._process_final_target_pp_files(obj, obj.files, backend_file)
+            self._process_final_target_pp_files(obj, obj.files, backend_file, 'DIST_FILES')
+
+        elif isinstance(obj, ObjdirFiles):
+            self._process_objdir_files(obj, obj.files, backend_file)
+
+        elif isinstance(obj, ObjdirPreprocessedFiles):
+            self._process_final_target_pp_files(obj, obj.files, backend_file, 'OBJDIR_PP_FILES')
 
         elif isinstance(obj, AndroidResDirs):
             # Order matters.
             for p in obj.paths:
                 backend_file.write('ANDROID_RES_DIRS += %s\n' % p.full_path)
 
         elif isinstance(obj, AndroidAssetsDirs):
             # Order matters.
@@ -1262,30 +1270,48 @@ class RecursiveMakeBackend(CommonBackend
                 tier = 'export' if obj.install_target == 'dist/include' else 'misc'
                 self._no_skip[tier].add(backend_file.relobjdir)
                 backend_file.write('%s_DEST := $(DEPTH)/%s\n'
                                    % (target_var,
                                       mozpath.join(target, path)))
                 backend_file.write('%s_TARGET := %s\n' % (target_var, tier))
                 backend_file.write('INSTALL_TARGETS += %s\n' % target_var)
 
-    def _process_final_target_pp_files(self, obj, files, backend_file):
+    def _process_final_target_pp_files(self, obj, files, backend_file, name):
         # Bug 1177710 - We'd like to install these via manifests as
         # preprocessed files. But they currently depend on non-standard flags
         # being added via some Makefiles, so for now we just pass them through
         # to the underlying Makefile.in.
+        #
+        # Note that if this becomes a manifest, OBJDIR_PP_FILES will likely
+        # still need to use PP_TARGETS internally because we can't have an
+        # install manifest for the root of the objdir.
+        for i, (path, files) in enumerate(files.walk()):
+            self._no_skip['misc'].add(backend_file.relobjdir)
+            var = '%s_%d' % (name, i)
+            for f in files:
+                backend_file.write('%s += %s\n' % (
+                    var, self._pretty_path(f, backend_file)))
+            backend_file.write('%s_PATH := $(DEPTH)/%s\n'
+                               % (var, mozpath.join(obj.install_target, path)))
+            backend_file.write('%s_TARGET := misc\n' % var)
+            backend_file.write('PP_TARGETS += %s\n' % var)
+
+    def _process_objdir_files(self, obj, files, backend_file):
+        # We can't use an install manifest for the root of the objdir, since it
+        # would delete all the other files that get put there by the build
+        # system.
         for i, (path, files) in enumerate(files.walk()):
             self._no_skip['misc'].add(backend_file.relobjdir)
             for f in files:
-                backend_file.write('DIST_FILES_%d += %s\n' % (
+                backend_file.write('OBJDIR_%d_FILES += %s\n' % (
                     i, self._pretty_path(f, backend_file)))
-            backend_file.write('DIST_FILES_%d_PATH := $(DEPTH)/%s\n'
-                               % (i, mozpath.join(obj.install_target, path)))
-            backend_file.write('DIST_FILES_%d_TARGET := misc\n' % i)
-            backend_file.write('PP_TARGETS += DIST_FILES_%d\n' % i)
+            backend_file.write('OBJDIR_%d_DEST := $(topobjdir)/%s\n' % (i, path))
+            backend_file.write('OBJDIR_%d_TARGET := misc\n' % i)
+            backend_file.write('INSTALL_TARGETS += OBJDIR_%d\n' % i)
 
     def _process_chrome_manifest_entry(self, obj, backend_file):
         fragment = Makefile()
         rule = fragment.create_rule(targets=['misc:'])
 
         top_level = mozpath.join(obj.install_target, 'chrome.manifest')
         if obj.path != top_level:
             args = [
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -1082,16 +1082,29 @@ VARIABLES = {
         """Disable the wrappers for STL which allow it to work with C++ exceptions
         disabled.
         """),
 
     'FINAL_TARGET_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
         """Like ``FINAL_TARGET_FILES``, with preprocessing.
         """),
 
+    'OBJDIR_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
+        """List of files to be installed anywhere in the objdir. Use sparingly.
+
+        ``OBJDIR_FILES`` is similar to FINAL_TARGET_FILES, but it allows copying
+        anywhere in the object directory. This is intended for various one-off
+        cases, not for general use. If you wish to add entries to OBJDIR_FILES,
+        please consult a build peer.
+        """),
+
+    'OBJDIR_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
+        """Like ``OBJDIR_FILES``, with preprocessing. Use sparingly.
+        """),
+
     'FINAL_LIBRARY': (unicode, unicode,
         """Library in which the objects of the current directory will be linked.
 
         This variable contains the name of a library, defined elsewhere with
         ``LIBRARY_NAME``, in which the objects of the current directory will be
         linked.
         """),
 
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -818,16 +818,46 @@ class FinalTargetPreprocessedFiles(Conte
     """
     __slots__ = ('files')
 
     def __init__(self, sandbox, files):
         ContextDerived.__init__(self, sandbox)
         self.files = files
 
 
+class ObjdirFiles(ContextDerived):
+    """Sandbox container object for OBJDIR_FILES, which is a
+    HierarchicalStringList.
+    """
+    __slots__ = ('files')
+
+    def __init__(self, sandbox, files):
+        ContextDerived.__init__(self, sandbox)
+        self.files = files
+
+    @property
+    def install_target(self):
+        return ''
+
+
+class ObjdirPreprocessedFiles(ContextDerived):
+    """Sandbox container object for OBJDIR_PP_FILES, which is a
+    HierarchicalStringList.
+    """
+    __slots__ = ('files')
+
+    def __init__(self, sandbox, files):
+        ContextDerived.__init__(self, sandbox)
+        self.files = files
+
+    @property
+    def install_target(self):
+        return ''
+
+
 class TestHarnessFiles(FinalTargetFiles):
     """Sandbox container object for TEST_HARNESS_FILES,
     which is a HierarchicalStringList.
     """
     @property
     def install_target(self):
         return '_tests'
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -48,16 +48,18 @@ from .data import (
     HostSimpleProgram,
     HostSources,
     InstallationTarget,
     IPDLFile,
     JARManifest,
     Library,
     Linkable,
     LocalInclude,
+    ObjdirFiles,
+    ObjdirPreprocessedFiles,
     PerSourceFlag,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     Program,
     SdkFiles,
     SharedLibrary,
     SimpleProgram,
     Sources,
@@ -825,16 +827,18 @@ class TreeMetadataEmitter(LoggingMixin):
         generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()])
 
         components = []
         for var, cls in (
             ('BRANDING_FILES', BrandingFiles),
             ('EXPORTS', Exports),
             ('FINAL_TARGET_FILES', FinalTargetFiles),
             ('FINAL_TARGET_PP_FILES', FinalTargetPreprocessedFiles),
+            ('OBJDIR_FILES', ObjdirFiles),
+            ('OBJDIR_PP_FILES', ObjdirPreprocessedFiles),
             ('SDK_FILES', SdkFiles),
             ('TEST_HARNESS_FILES', TestHarnessFiles),
         ):
             all_files = context.get(var)
             if not all_files:
                 continue
             if dist_install is False and var != 'TEST_HARNESS_FILES':
                 raise SandboxValidationError(
@@ -848,22 +852,23 @@ class TreeMetadataEmitter(LoggingMixin):
                         'Cannot install files to the root of TEST_HARNESS_FILES', context)
                 if base == 'components':
                     components.extend(files)
                 if base == 'defaults/pref':
                     has_prefs = True
                 if mozpath.split(base)[0] == 'res':
                     has_resources = True
                 for f in files:
-                    if (var == 'FINAL_TARGET_PP_FILES' and
+                    if ((var == 'FINAL_TARGET_PP_FILES' or
+                         var == 'OBJDIR_PP_FILES') and
                         not isinstance(f, SourcePath)):
                         raise SandboxValidationError(
                                 ('Only source directory paths allowed in ' +
-                                'FINAL_TARGET_PP_FILES: %s')
-                                % (f,), context)
+                                 '%s: %s')
+                                % (var, f,), context)
                     if not isinstance(f, ObjDirPath):
                         path = f.full_path
                         if '*' not in path and not os.path.exists(path):
                             raise SandboxValidationError(
                                 'File listed in %s does not exist: %s'
                                 % (var, path), context)
                     else:
                         # TODO: Bug 1254682 - The '/' check is to allow