Bug 1210642 - Use install manifests for preprocessed files in the FasterMake build backend. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 01 Oct 2015 17:10:08 +0900
changeset 266143 af55d7cf20939aef04e73c56be4330dd601c2b99
parent 266142 42199c7b225a8c5318faf169a8e6e1d24f82cf32
child 266144 c44feed55e20e38935d42b358757d0037ed28290
push id66129
push usermh@glandium.org
push dateTue, 06 Oct 2015 02:15:35 +0000
treeherdermozilla-inbound@af55d7cf2093 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1210642
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1210642 - Use install manifests for preprocessed files in the FasterMake build backend. r=gps
config/faster/rules.mk
python/mozbuild/mozbuild/backend/fastermake.py
--- a/config/faster/rules.mk
+++ b/config/faster/rules.mk
@@ -26,33 +26,30 @@
 #   object directory
 # - PYTHON, the path to the python executable
 # - ACDEFINES, which contains a set of -Dvar=name to be used during
 #   preprocessing
 # - MOZ_CHROME_FILE_FORMAT, which defines whether to use file copies or
 #   symbolic links
 # - JAR_MN_TARGETS, which defines the targets to use for jar manifest
 #   processing, see further below
-# - PP_TARGETS, which defines the file paths of preprocessed files, see
-#   further below
 # - INSTALL_MANIFESTS, which defines the list of base directories handled
 #   by install manifests, see further below
 # - MANIFEST_TARGETS, which defines the file paths of chrome manifests, see
 #   further below
 #
 # A convention used between this file and the Makefile including it is that
 # global Make variables names are uppercase, while "local" Make variables
 # applied to specific targets are lowercase.
 
 # Targets to be triggered for a default build
 default: $(addprefix install-,$(INSTALL_MANIFESTS))
 default: $(addprefix jar-,$(JAR_MN_TARGETS))
 
 # Explicit files to be built for a default build
-default: $(addprefix $(TOPOBJDIR)/,$(PP_TARGETS))
 default: $(addprefix $(TOPOBJDIR)/,$(MANIFEST_TARGETS))
 default: $(TOPOBJDIR)/dist/bin/greprefs.js
 default: $(TOPOBJDIR)/dist/bin/platform.ini
 default: $(TOPOBJDIR)/dist/bin/webapprt/webapprt.ini
 
 # Targets from the recursive make backend to be built for a default build
 default: $(TOPOBJDIR)/config/makefiles/xpidl/xpidl
 
@@ -78,52 +75,30 @@ ACDEFINES += -DBUILD_FASTER
 	cp $< $@
 
 # Install files using install manifests
 #
 # The list of base directories is given in INSTALL_MANIFESTS. The
 # corresponding install manifests are named correspondingly, with forward
 # slashes replaced with underscores, and prefixed with `install_`. That is,
 # the install manifest for `dist/bin` would be `install_dist_bin`.
-$(addprefix install-,$(INSTALL_MANIFESTS)): install-%:
+$(addprefix install-,$(INSTALL_MANIFESTS)): install-%: $(TOPOBJDIR)/config/buildid
+	@# For now, force preprocessed files to be reprocessed every time.
+	@# The overhead is not that big, and this avoids waiting for proper
+	@# support for defines tracking in process_install_manifest.
+	@touch install_$(subst /,_,$*)
 	$(PYTHON) -m mozbuild.action.process_install_manifest \
 		--no-remove \
 		--no-remove-empty-directories \
 		$(TOPOBJDIR)/$* \
-		install_$(subst /,_,$*)
-
-# Preprocessed files. Ideally they would be using install manifests but
-# right now, it's not possible because of things like MOZ_APP_BUILDID or
-# nsURLFormatter.js.
-#
-# The list of preprocessed files is defined in PP_TARGETS. The list is
-# relative to TOPOBJDIR.
-# The source file for each of those preprocessed files is defined as a Make
-# dependency for the $(TOPOBJDIR)/path target. For example:
-#   PP_TARGETS = foo/bar
-#   $(TOPOBJDIR)/foo/bar: /path/to/source/for/foo/bar.in
-# The file name for the source doesn't need to be different.
-# Additionally, extra defines can be specified for a given preprocessing
-# by setting the `defines` variable specifically for the given target.
-# For example:
-#   $(TOPOBJDIR)/foo/bar: defines = -Dqux=foobar
-$(addprefix $(TOPOBJDIR)/,$(PP_TARGETS)): Makefile
-$(addprefix $(TOPOBJDIR)/,$(PP_TARGETS)): $(TOPOBJDIR)/%:
-	$(PYTHON) -m mozbuild.action.preprocessor \
-		--depend $(TOPOBJDIR)/faster/.deps/$(subst /,_,$*) \
 		-DAB_CD=en-US \
-		$(defines) \
+		-DMOZ_APP_BUILDID=$(shell cat $(TOPOBJDIR)/config/buildid) \
 		$(ACDEFINES) \
 		$(MOZ_DEBUG_DEFINES) \
-		$< \
-		-o $@
-
-# Include the dependency files from the above preprocessed files rule.
-$(foreach pp_target,$(PP_TARGETS), \
-	$(eval -include $(TOPOBJDIR)/faster/.deps/$(subst /,_,$(pp_target))))
+		install_$(subst /,_,$*)
 
 # Install files from jar manifests. Ideally, they would be using install
 # manifests, but the code to read jar manifests and emit appropriate
 # install manifests is not there yet.
 # Things missing:
 # - DEFINES from config/config.mk
 # - L10N
 # - -e when USE_EXTENSION_MANIFEST is set in moz.build
@@ -186,21 +161,16 @@ jar-%:
 
 # GENERATED_FILES are not supported yet, and even if they were, the
 # dependencies are missing information.
 $(foreach p,linux osx windows,jar-browser-themes-$(p)-jar.mn): \
 jar-browser-themes-%-jar.mn: \
 	$(TOPOBJDIR)/browser/themes/%/tab-selected-end.svg \
 	$(TOPOBJDIR)/browser/themes/%/tab-selected-start.svg
 
-# Extra dependencies and/or definitions for preprocessed files.
-$(TOPOBJDIR)/dist/bin/application.ini: $(TOPOBJDIR)/config/buildid
-$(TOPOBJDIR)/dist/bin/application.ini: defines += \
-	-DMOZ_APP_BUILDID=$(shell cat $(TOPOBJDIR)/config/buildid)
-
 # Files to build with the recursive backend and simply copy
 $(TOPOBJDIR)/dist/bin/greprefs.js: $(TOPOBJDIR)/modules/libpref/greprefs.js
 $(TOPOBJDIR)/dist/bin/platform.ini: $(TOPOBJDIR)/toolkit/xre/platform.ini
 $(TOPOBJDIR)/dist/bin/webapprt/webapprt.ini: $(TOPOBJDIR)/webapprt/webapprt.ini
 
 # The xpidl target in config/makefiles/xpidl requires the install manifest for
 # dist/idl to have been processed.
 $(TOPOBJDIR)/config/makefiles/xpidl/xpidl: $(TOPOBJDIR)/install-dist_idl
--- a/python/mozbuild/mozbuild/backend/fastermake.py
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -27,38 +27,53 @@ from itertools import chain
 class FasterMakeBackend(CommonBackend):
     def _init(self):
         super(FasterMakeBackend, self)._init()
 
         self._seen_directories = set()
         self._defines = dict()
         self._jar_manifests = OrderedDict()
 
-        self._preprocess_files = OrderedDict()
-
         self._manifest_entries = OrderedDefaultDict(list)
 
         self._install_manifests = OrderedDefaultDict(InstallManifest)
 
+    def _add_preprocess(self, obj, path, dest, **kwargs):
+        target = mozpath.basename(path)
+        # This matches what PP_TARGETS do in config/rules.
+        if target.endswith('.in'):
+            target = target[:-3]
+        depfile = mozpath.join(
+            self.environment.topobjdir, 'faster', '.deps',
+            mozpath.join(obj.install_target, dest, target).replace('/', '_'))
+        self._install_manifests[obj.install_target].add_preprocess(
+            mozpath.join(obj.srcdir, path),
+            mozpath.join(dest, target),
+            depfile,
+            **kwargs)
+
     def consume_object(self, obj):
         if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
-            defines = self._defines.get(obj.objdir, [])
+            defines = self._defines.get(obj.objdir, {})
             if defines:
-                defines = list(defines.get_defines())
+                defines = defines.defines
 
         if isinstance(obj, Defines):
             self._defines[obj.objdir] = obj
 
             # We're assuming below that Defines come first for a given objdir,
             # which is kind of set in stone from the order things are treated
             # in emitter.py.
             assert obj.objdir not in self._seen_directories
 
         elif isinstance(obj, JARManifest) and \
                 obj.install_target.startswith('dist/bin'):
+            defines = self._defines.get(obj.objdir, [])
+            if defines:
+                defines = list(defines.get_defines())
             self._jar_manifests[obj.path] = (obj.objdir,
                                              obj.install_target,
                                              defines)
 
         elif isinstance(obj, VariablePassthru) and \
                 obj.install_target.startswith('dist/bin'):
             for f in obj.variables.get('EXTRA_COMPONENTS', {}):
                 path = mozpath.join(obj.install_target, 'components',
@@ -69,38 +84,36 @@ class FasterMakeBackend(CommonBackend):
                 )
                 if f.endswith('.manifest'):
                     manifest = mozpath.join(obj.install_target,
                                             'chrome.manifest')
                     self._manifest_entries[manifest].append(
                         'manifest components/%s' % mozpath.basename(f))
 
             for f in obj.variables.get('EXTRA_PP_COMPONENTS', {}):
-                path = mozpath.join(obj.install_target, 'components',
-                                    mozpath.basename(f))
-                self._preprocess_files[path] = (obj.srcdir, f, defines)
+                self._add_preprocess(obj, f, 'components', defines=defines)
+
                 if f.endswith('.manifest'):
                     manifest = mozpath.join(obj.install_target,
                                             'chrome.manifest')
                     self._manifest_entries[manifest].append(
                         'manifest components/%s' % mozpath.basename(f))
 
         elif isinstance(obj, JavaScriptModules) and \
                 obj.install_target.startswith('dist/bin'):
             for path, strings in obj.modules.walk():
-                base = mozpath.join(obj.install_target, 'modules', path)
+                base = mozpath.join('modules', path)
                 for f in strings:
                     if obj.flavor == 'extra':
                         self._install_manifests[obj.install_target].add_symlink(
                             mozpath.join(obj.srcdir, f),
-                            mozpath.join('modules', path, mozpath.basename(f))
+                            mozpath.join(base, mozpath.basename(f))
                         )
                     elif obj.flavor == 'extra_pp':
-                        dest = mozpath.join(base, mozpath.basename(f))
-                        self._preprocess_files[dest] = (obj.srcdir, f, defines)
+                        self._add_preprocess(obj, f, base, defines=defines)
 
         elif isinstance(obj, JsPreferenceFile) and \
                 obj.install_target.startswith('dist/bin'):
             # The condition for the directory value in config/rules.mk is:
             # ifneq (,$(DIST_SUBDIR)$(XPI_NAME)$(LIBXUL_SDK))
             # - LIBXUL_SDK is not supported (it likely doesn't work in the
             # recursive backend anyways
             # - when XPI_NAME is set, obj.install_target will start with
@@ -114,55 +127,51 @@ class FasterMakeBackend(CommonBackend):
                 pref_dir = 'defaults/pref'
             else:
                 pref_dir = 'defaults/preferences'
 
             dest = mozpath.join(obj.install_target, pref_dir,
                                 mozpath.basename(obj.path))
             # We preprocess these, but they don't necessarily have preprocessor
             # directives, so tell the preprocessor to not complain about that.
-            defines.append('--silence-missing-directive-warnings')
-            self._preprocess_files[dest] = (obj.srcdir, obj.path, defines)
+            self._add_preprocess(obj, obj.path, pref_dir, defines=defines,
+                                 silence_missing_directive_warnings=True)
 
         elif isinstance(obj, Resources) and \
                 obj.install_target.startswith('dist/bin'):
             for path, strings in obj.resources.walk():
-                base = mozpath.join(obj.install_target, 'res', path)
+                base = mozpath.join('res', path)
                 for f in strings:
                     flags = strings.flags_for(f)
                     if flags and flags.preprocess:
-                        dest = mozpath.join(base, mozpath.basename(f))
-                        defines = Defines(obj._context, obj.defines)
-                        defines = list(defines.get_defines())
-                        defines.extend(['--marker', '%'])
-                        self._preprocess_files[dest] = (obj.srcdir, f, defines)
+                        self._add_preprocess(obj, f, base, marker='%',
+                                             defines=obj.defines)
                     else:
                         self._install_manifests[obj.install_target].add_symlink(
                             mozpath.join(obj.srcdir, f),
-                            mozpath.join('res', path, mozpath.basename(f))
+                            mozpath.join(base, mozpath.basename(f))
                         )
 
         elif isinstance(obj, FinalTargetFiles) and \
                 obj.install_target.startswith('dist/bin'):
             for path, strings in obj.files.walk():
                 base = mozpath.join(obj.install_target, path)
                 for f in strings:
                     self._install_manifests[obj.install_target].add_symlink(
                         mozpath.join(obj.srcdir, f),
                         mozpath.join(path, mozpath.basename(f))
                     )
 
         elif isinstance(obj, DistFiles) and \
                 obj.install_target.startswith('dist/bin'):
             # We preprocess these, but they don't necessarily have preprocessor
             # directives, so tell the preprocessor to not complain about that.
-            defines.append('--silence-missing-directive-warnings')
             for f in obj.files:
-                dest = mozpath.join(obj.install_target, mozpath.basename(f))
-                self._preprocess_files[dest] = (obj.srcdir, f, defines)
+                self._add_preprocess(obj, f, '', defines=defines,
+                                     silence_missing_directive_warnings=True)
 
         else:
             # We currently ignore a lot of object types, so just acknowledge
             # everything.
             return True
 
         self._seen_directories.add(obj.objdir)
         return True
@@ -212,35 +221,16 @@ class FasterMakeBackend(CommonBackend):
         for target, entries in self._manifest_entries.iteritems():
             manifest_targets.append(target)
             target = '$(TOPOBJDIR)/%s' % target
             mk.create_rule([target]).add_dependencies(
                 ['content = %s' % ' '.join('"%s"' % e for e in entries)])
 
         mk.add_statement('MANIFEST_TARGETS = %s' % ' '.join(manifest_targets))
 
-        # Add information for preprocessed files.
-        preprocess_targets = []
-
-        for target, (srcdir, f, defines) in self._preprocess_files.iteritems():
-            # This matches what PP_TARGETS do in config/rules.
-            if target.endswith('.in'):
-                target = target[:-3]
-                # PP_TARGETS assumes this is true, but doesn't enforce it.
-                assert target not in self._preprocess_files
-            preprocess_targets.append(target)
-            target = '$(TOPOBJDIR)/%s' % target
-            mk.create_rule([target]).add_dependencies(
-                [mozpath.join(srcdir, f)])
-            if defines:
-                mk.create_rule([target]).add_dependencies(
-                        ['defines = %s' % ' '.join(defines)])
-
-        mk.add_statement('PP_TARGETS = %s' % ' '.join(preprocess_targets))
-
         # Add information for install manifests.
         mk.add_statement('INSTALL_MANIFESTS = %s'
                          % ' '.join(self._install_manifests.keys()))
 
         mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
 
         for base, install_manifest in self._install_manifests.iteritems():
             with self._write_file(