Bug 1573560 - Have Node compilation write directly into FINAL_TARGET instead of requiring a symlink r=firefox-build-system-reviewers,jlast,mshal
authorRicky Stewart <rstewart@mozilla.com>
Mon, 04 Nov 2019 23:15:19 +0000
changeset 500735 6b66a5b1d3f0ed10b4a4cbacea41b94f245537f3
parent 500734 0fb0f9126e63df0a54fdddcdbc3817f1d38047bf
child 500736 a9b6b44f82afcddf49bc27e4178e2f625bb35b65
push id114166
push userapavel@mozilla.com
push dateThu, 07 Nov 2019 10:04:01 +0000
treeherdermozilla-inbound@d271c572a9bc [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfirefox-build-system-reviewers, jlast, mshal
bugs1573560
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1573560 - Have Node compilation write directly into FINAL_TARGET instead of requiring a symlink r=firefox-build-system-reviewers,jlast,mshal Differential Revision: https://phabricator.services.mozilla.com/D47367
config/faster/rules.mk
config/rules.mk
devtools/client/shared/build/build.js
devtools/client/shared/build/node-templates.mozbuild
python/mozbuild/mozbuild/backend/fastermake.py
python/mozbuild/mozbuild/backend/make.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/config/faster/rules.mk
+++ b/config/faster/rules.mk
@@ -93,9 +93,9 @@ endif
 		-DAB_CD=en-US \
 		$(ACDEFINES) \
 		install_$(subst /,_,$*)
 
 # ============================================================================
 # Below is a set of additional dependencies and variables used to build things
 # that are not supported by data in moz.build.
 
-$(TOPOBJDIR)/build/application.ini: $(TOPOBJDIR)/buildid.h $(TOPOBJDIR)/source-repo.h
+$(TOPOBJDIR)/build/.deps/application.ini.stub: $(TOPOBJDIR)/buildid.h $(TOPOBJDIR)/source-repo.h
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -1049,17 +1049,18 @@ endif
 ifneq (,$(filter target-objects target all default,$(MAKECMDGOALS)))
 _MDDEPEND_FILES += $(addsuffix .pp,$(notdir $(sort $(OBJS) $(PROGOBJS))))
 endif
 
 ifneq (,$(filter host-objects host all default,$(MAKECMDGOALS)))
 _MDDEPEND_FILES += $(addsuffix .pp,$(notdir $(sort $(HOST_OBJS) $(HOST_PROGOBJS))))
 endif
 
-MDDEPEND_FILES := $(strip $(wildcard $(addprefix $(MDDEPDIR)/,$(_MDDEPEND_FILES) $(EXTRA_MDDEPEND_FILES))))
+MDDEPEND_FILES := $(strip $(wildcard $(addprefix $(MDDEPDIR)/,$(_MDDEPEND_FILES))))
+MDDEPEND_FILES += $(EXTRA_MDDEPEND_FILES)
 
 ifneq (,$(MDDEPEND_FILES))
 -include $(MDDEPEND_FILES)
 endif
 
 ################################################################################
 # Install/copy rules
 #
--- a/devtools/client/shared/build/build.js
+++ b/devtools/client/shared/build/build.js
@@ -1,13 +1,18 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
 /* globals process, __filename, __dirname */
 
+/* Usage:  node build.js [LIST_OF_SOURCE_FILES...] OUTPUT_DIR
+ *    Compiles all source files and places the results of the compilation in
+ * OUTPUT_DIR.
+ */
+
 "use strict";
 
 const Babel = require("./babel");
 const fs = require("fs");
 const _path = require("path");
 
 const defaultPlugins = [
   "transform-flow-strip-types",
@@ -38,23 +43,42 @@ Stack:
 
 ========================
 `);
   }
 
   return out.code;
 }
 
-const deps = [__filename, _path.resolve(__dirname, "babel.js")];
+// fs.mkdirSync's "recursive" option appears not to work, so I'm writing a
+// simple version of the function myself.
+function mkdirs(filePath) {
+  if (fs.existsSync(filePath)) {
+    return;
+  }
+  mkdirs(_path.dirname(filePath));
+  try {
+    fs.mkdirSync(filePath);
+  } catch (err) {
+    // Ignore any errors resulting from the directory already existing.
+    if (err.code != "EEXIST") {
+      throw err;
+    }
+  }
+}
 
-for (let i = 2; i < process.argv.length; i++) {
+const deps = [__filename, _path.resolve(__dirname, "babel.js")];
+const outputDir = process.argv[process.argv.length - 1];
+mkdirs(outputDir);
+
+for (let i = 2; i < process.argv.length - 1; i++) {
   const srcPath = process.argv[i];
   const code = transform(srcPath);
-  const filePath = _path.basename(srcPath);
-  fs.writeFileSync(filePath, code);
+  const fullPath = _path.join(outputDir, _path.basename(srcPath));
+  fs.writeFileSync(fullPath, code);
   deps.push(srcPath);
 }
 
 if (false) {
   const code = transform("devtools/client/debugger/src/utils/prefs.js");
   console.log(code.slice(0, 1500));
 }
 
--- a/devtools/client/shared/build/node-templates.mozbuild
+++ b/devtools/client/shared/build/node-templates.mozbuild
@@ -1,39 +1,34 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 @template
 def CompiledModules(*modules):
-  # Compute where to put compiled files into omni.ja package
-  # All DevTools modules are used via resource://devtools/ URI
-  # See devtools/shared/jar.mn for how this resource is mapped into jar package.
-  base = FINAL_TARGET_FILES.chrome.devtools.modules
-
   compiled_directory_whitelist = (
     "devtools/client/debugger/src",
   )
 
   if not RELATIVEDIR.startswith(compiled_directory_whitelist):
     error("File in directory provided to CompiledModules not allowed: " + RELATIVEDIR)
 
-  # Now, navigate to the right sub-directory into devtools root modules folder
-  for dir in RELATIVEDIR.split('/'):
-    base = base[dir]
+  # HACK. Template export() propagation is janky so we have to re-implement the
+  # logic for computing FINAL_TARGET from scratch. Here we emulate the
+  # DIST_SUBDIR export in devtools/moz.build.
+  if CONFIG['MOZ_BUILD_APP'] == 'browser':
+    final_target = 'dist/bin/browser'
+  else:
+    final_target = 'dist/bin'
 
-  size = 0
-  for m in modules:
-    base += ["!" + m]
-    size += 1
-
-  if size == 0:
-    return
-
+  final = '/'.join([TOPOBJDIR, final_target, 'chrome/devtools/modules',
+                    RELATIVEDIR])
   # For the same reason as https://searchfox.org/mozilla-central/source/mobile/android/base/moz.build#180-184
   # we have to insert a first entry as recursivemake overrides the first entry and we end up with empty files
   # for the first file only.
   GeneratedFile(
-      "node.stub", *modules,
+      "node.stub", *[final + '/' + module for module in modules],
       script='/python/mozbuild/mozbuild/action/node.py',
       entry_point='generate',
-      inputs=['/devtools/client/shared/build/build.js'] + [module for module in modules])
+      inputs=['/devtools/client/shared/build/build.js'] +
+             [module for module in modules],
+      flags=[final])
--- a/python/mozbuild/mozbuild/backend/fastermake.py
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -1,49 +1,48 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals, print_function
 
 from mozbuild.backend.base import PartialBackend
-from mozbuild.backend.common import CommonBackend
-from mozbuild.frontend.context import (
-    ObjDirPath,
-)
+from mozbuild.backend.make import MakeBackend
+from mozbuild.frontend.context import ObjDirPath
 from mozbuild.frontend.data import (
     ChromeManifestEntry,
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     GeneratedFile,
     JARManifest,
     LocalizedFiles,
     LocalizedPreprocessedFiles,
     XPIDLModule,
 )
 from mozbuild.makeutil import Makefile
 from mozbuild.util import OrderedDefaultDict
 from mozpack.manifests import InstallManifest
 import mozpack.path as mozpath
 
 
-class FasterMakeBackend(CommonBackend, PartialBackend):
+class FasterMakeBackend(MakeBackend, PartialBackend):
     def _init(self):
         super(FasterMakeBackend, self)._init()
 
         self._manifest_entries = OrderedDefaultDict(set)
 
         self._install_manifests = OrderedDefaultDict(InstallManifest)
 
         self._dependencies = OrderedDefaultDict(list)
         self._l10n_dependencies = OrderedDefaultDict(list)
 
         self._has_xpidl = False
 
         self._generated_files_map = {}
+        self._generated_files = []
 
     def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
         if target is None:
             target = mozpath.basename(path)
         # This matches what PP_TARGETS do in config/rules.
         if target.endswith('.in'):
             target = target[:-3]
         if target.endswith('.css'):
@@ -135,19 +134,17 @@ class FasterMakeBackend(CommonBackend, P
         elif isinstance(obj, GeneratedFile):
             if obj.outputs:
                 first_output = mozpath.relpath(mozpath.join(
                     obj.objdir, obj.outputs[0]), self.environment.topobjdir)
                 for o in obj.outputs[1:]:
                     fullpath = mozpath.join(obj.objdir, o)
                     self._generated_files_map[mozpath.relpath(
                         fullpath, self.environment.topobjdir)] = first_output
-            # We don't actually handle GeneratedFiles, we just need to know if
-            # we can build multiple of them from a single make invocation in the
-            # faster backend.
+            self._generated_files.append(obj)
             return False
 
         elif isinstance(obj, XPIDLModule):
             self._has_xpidl = True
             # We're not actually handling XPIDL files.
             return False
 
         else:
@@ -156,16 +153,21 @@ class FasterMakeBackend(CommonBackend, P
         return True
 
     def consume_finished(self):
         mk = Makefile()
         # Add the default rule at the very beginning.
         mk.create_rule(['default'])
         mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
         mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
+        mk.add_statement('MDDEPDIR = .deps')
+        mk.add_statement('TOUCH ?= touch')
+        mk.add_statement('include $(TOPSRCDIR)/config/makefiles/functions.mk')
+        mk.add_statement('include $(TOPSRCDIR)/config/AB_rCD.mk')
+        mk.add_statement('AB_CD = en-US')
         if not self._has_xpidl:
             mk.add_statement('NO_XPIDL = 1')
 
         # Add a few necessary variables inherited from configure
         for var in (
             'PYTHON',
             'ACDEFINES',
             'MOZ_BUILD_APP',
@@ -240,12 +242,35 @@ class FasterMakeBackend(CommonBackend, P
                     base = base[1:]
                 unified_manifest.add_entries_from(install_manifest, base=base)
 
             with self._write_file(
                     mozpath.join(self.environment.topobjdir, 'faster',
                                  'unified_install_dist_bin')) as fh:
                 unified_manifest.write(fileobj=fh)
 
+        for obj in self._generated_files:
+            for stmt in self._format_statements_for_generated_file(obj,
+                                                                   'default'):
+                mk.add_statement(stmt)
+
         with self._write_file(
                 mozpath.join(self.environment.topobjdir, 'faster',
                              'Makefile')) as fh:
             mk.dump(fh, removal_guard=False)
+
+    def _pretty_path(self, path, obj):
+        if path.startswith(self.environment.topobjdir):
+            return mozpath.join(
+                '$(TOPOBJDIR)',
+                mozpath.relpath(path, self.environment.topobjdir))
+        elif path.startswith(self.environment.topsrcdir):
+            return mozpath.join(
+                '$(TOPSRCDIR)',
+                mozpath.relpath(path, self.environment.topsrcdir))
+        else:
+            return path
+
+    def _format_generated_file_input_name(self, path, obj):
+        return self._pretty_path(path.full_path, obj)
+
+    def _format_generated_file_output_name(self, path, obj):
+        return self._pretty_path(mozpath.join(obj.objdir, path), obj)
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/backend/make.py
@@ -0,0 +1,121 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from .common import CommonBackend
+
+import mozpack.path as mozpath
+
+from mozbuild.frontend.data import GeneratedFile
+from mozbuild.shellutil import quote as shell_quote
+
+
+class MakeBackend(CommonBackend):
+    """Class encapsulating logic for backends that use Make."""
+
+    def _init(self):
+        CommonBackend._init(self)
+
+    def _format_statements_for_generated_file(self, obj, tier,
+                                              extra_dependencies=''):
+        """Return the list of statements to write to the Makefile for this
+        GeneratedFile.
+
+        This function will invoke _format_generated_file_input_name and
+        _format_generated_file_output_name to munge the input/output filenames
+        before sending them to the output.
+        """
+        assert isinstance(obj, GeneratedFile)
+
+        # Localized generated files can use {AB_CD} and {AB_rCD} in their
+        # output paths.
+        if obj.localized:
+            substs = {'AB_CD': '$(AB_CD)', 'AB_rCD': '$(AB_rCD)'}
+        else:
+            substs = {}
+
+        outputs = []
+        needs_AB_rCD = False
+        for o in obj.outputs:
+            needs_AB_rCD = needs_AB_rCD or ('AB_rCD' in o)
+            try:
+                outputs.append(self._format_generated_file_output_name(
+                    o.format(**substs), obj))
+            except KeyError as e:
+                raise ValueError('%s not in %s is not a valid substitution in %s'
+                                 % (e.args[0], ', '.join(sorted(substs.keys())), o))
+
+        first_output = outputs[0]
+        dep_file = mozpath.join(mozpath.dirname(first_output), "$(MDDEPDIR)",
+                                "%s.pp" % mozpath.basename(first_output))
+        # The stub target file needs to go in MDDEPDIR so that it doesn't
+        # get written into generated Android resource directories, breaking
+        # Gradle tooling and/or polluting the Android packages.
+        stub_file = mozpath.join(mozpath.dirname(first_output), "$(MDDEPDIR)",
+                                 "%s.stub" % mozpath.basename(first_output))
+
+        if obj.inputs:
+            inputs = [self._format_generated_file_input_name(f, obj)
+                      for f in obj.inputs]
+        else:
+            inputs = []
+
+        force = ''
+        if obj.force:
+            force = ' FORCE'
+        elif obj.localized:
+            force = ' $(if $(IS_LANGUAGE_REPACK),FORCE)'
+
+        ret = []
+
+        if obj.script:
+            # If we are doing an artifact build, we don't run compiler, so
+            # we can skip generated files that are needed during compile,
+            # or let the rule run as the result of something depending on
+            # it.
+            if not (obj.required_before_compile or obj.required_during_compile) or \
+                    not self.environment.is_artifact_build:
+                if tier and not needs_AB_rCD:
+                    # Android localized resources have special Makefile
+                    # handling.
+                    ret.append('%s%s: %s' % (
+                        tier, ':' if tier != 'default' else '', stub_file))
+            for output in outputs:
+                ret.append('%s: %s ;' % (output, stub_file))
+                ret.append('GARBAGE += %s' % output)
+            ret.append('GARBAGE += %s' % stub_file)
+            ret.append('EXTRA_MDDEPEND_FILES += %s' % dep_file)
+
+            ret.append((
+                    """{stub}: {script}{inputs}{backend}{force}
+\t$(REPORT_BUILD)
+\t$(call py_action,file_generate,{locale}{script} """  # wrap for E501
+                    """{method} {output} {dep_file} {stub}{inputs}{flags})
+\t@$(TOUCH) $@
+""").format(
+                stub=stub_file,
+                output=first_output,
+                dep_file=dep_file,
+                inputs=' ' + ' '.join(inputs) if inputs else '',
+                flags=' ' + ' '.join(shell_quote(f) for f in obj.flags) if obj.flags else '',
+                backend=' ' + extra_dependencies if extra_dependencies else '',
+                # Locale repacks repack multiple locales from a single configured objdir,
+                # so standard mtime dependencies won't work properly when the build is re-run
+                # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set
+                # in this situation, so simply force the generation to run in that case.
+                force=force,
+                locale='--locale=$(AB_CD) ' if obj.localized else '',
+                script=obj.script,
+                method=obj.method
+                )
+            )
+
+        return ret
+
+    def _format_generated_file_input_name(self, path, obj):
+        raise NotImplementedError('Subclass must implement')
+
+    def _format_generated_file_output_name(self, path, obj):
+        raise NotImplementedError('Subclass must implement')
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -23,16 +23,17 @@ import mozpack.path as mozpath
 from mozbuild.frontend.context import (
     AbsolutePath,
     Path,
     RenamedSourcePath,
     SourcePath,
     ObjDirPath,
 )
 from .common import CommonBackend
+from .make import MakeBackend
 from ..frontend.data import (
     BaseLibrary,
     BaseProgram,
     BaseRustLibrary,
     ChromeManifestEntry,
     ComputedFlags,
     ConfigFileSubstitution,
     ContextDerived,
@@ -357,33 +358,33 @@ class RecursiveMakeTraversal(object):
                 new_result = self.SubDirectories()
                 new_result.dirs.extend(result.dirs)
                 new_result.dirs.extend(sorted(unattached))
                 new_result.tests.extend(result.tests)
                 result = new_result
         return result
 
 
-class RecursiveMakeBackend(CommonBackend):
+class RecursiveMakeBackend(MakeBackend):
     """Backend that integrates with the existing recursive make build system.
 
     This backend facilitates the transition from Makefile.in to moz.build
     files.
 
     This backend performs Makefile.in -> Makefile conversion. It also writes
     out .mk files containing content derived from moz.build files. Both are
     consumed by the recursive make builder.
 
     This backend may eventually evolve to write out non-recursive make files.
     However, as long as there are Makefile.in files in the tree, we are tied to
     recursive make and thus will need this backend.
     """
 
     def _init(self):
-        CommonBackend._init(self)
+        MakeBackend._init(self)
 
         self._backend_files = {}
         self._idl_dirs = set()
 
         self._makefile_in_count = 0
         self._makefile_out_count = 0
 
         self._test_manifests = {}
@@ -541,116 +542,21 @@ class RecursiveMakeBackend(CommonBackend
             elif obj.required_during_compile:
                 tier = None
             elif obj.localized:
                 tier = 'libs'
             else:
                 tier = 'misc'
             if tier:
                 self._no_skip[tier].add(backend_file.relobjdir)
-
-            # Localized generated files can use {AB_CD} and {AB_rCD} in their
-            # output paths.
-            if obj.localized:
-                substs = {'AB_CD': '$(AB_CD)', 'AB_rCD': '$(AB_rCD)'}
-            else:
-                substs = {}
-            outputs = []
-
-            needs_AB_rCD = False
-            for o in obj.outputs:
-                needs_AB_rCD = needs_AB_rCD or ('AB_rCD' in o)
-                try:
-                    outputs.append(o.format(**substs))
-                except KeyError as e:
-                    raise ValueError('%s not in %s is not a valid substitution in %s'
-                                     % (e.args[0], ', '.join(sorted(substs.keys())), o))
-
-            first_output = outputs[0]
-            dep_file = "%s.pp" % first_output
-            # The stub target file needs to go in MDDEPDIR so that it doesn't
-            # get written into generated Android resource directories, breaking
-            # Gradle tooling and/or polluting the Android packages.
-            stub_file = "$(MDDEPDIR)/%s.stub" % first_output
-
-            if obj.inputs:
-                if obj.localized:
-                    # Localized generated files can have locale-specific inputs, which are
-                    # indicated by paths starting with `en-US/` or containing `locales/en-US/`.
-                    def srcpath(p):
-                        if 'locales/en-US' in p:
-                            # We need an "absolute source path" relative to
-                            # topsrcdir, like "/source/path".
-                            if not p.startswith('/'):
-                                p = '/' + mozpath.relpath(p.full_path, obj.topsrcdir)
-                            e, f = p.split('locales/en-US/', 1)
-                            assert(f)
-                            return '$(call MERGE_RELATIVE_FILE,{},{}locales)'.format(
-                                f, e if not e.startswith('/') else e[len('/'):])
-                        elif p.startswith('en-US/'):
-                            e, f = p.split('en-US/', 1)
-                            assert(not e)
-                            return '$(call MERGE_FILE,%s)' % f
-                        return self._pretty_path(p, backend_file)
-                    inputs = [srcpath(f) for f in obj.inputs]
-                else:
-                    inputs = [self._pretty_path(f, backend_file) for f in obj.inputs]
-            else:
-                inputs = []
-
-            if needs_AB_rCD:
-                backend_file.write_once('include $(topsrcdir)/config/AB_rCD.mk\n')
-
-            force = ''
-            if obj.force:
-                force = ' FORCE'
-            elif obj.localized:
-                force = ' $(if $(IS_LANGUAGE_REPACK),FORCE)'
-
-            if obj.script:
-                # If we are doing an artifact build, we don't run compiler, so
-                # we can skip generated files that are needed during compile,
-                # or let the rule run as the result of something depending on
-                # it.
-                if not (obj.required_before_compile or obj.required_during_compile) or \
-                        not self.environment.is_artifact_build:
-                    if tier and not needs_AB_rCD:
-                        # Android localized resources have special Makefile
-                        # handling.
-                        backend_file.write('%s:: %s\n' % (tier, stub_file))
-                for output in outputs:
-                    backend_file.write('%s: %s ;\n' % (output, stub_file))
-                    backend_file.write('GARBAGE += %s\n' % output)
-                backend_file.write('GARBAGE += %s\n' % stub_file)
-                backend_file.write('EXTRA_MDDEPEND_FILES += %s\n' % dep_file)
-
-                backend_file.write((
-                    """{stub}: {script}{inputs}{backend}{force}
-\t$(REPORT_BUILD)
-\t$(call py_action,file_generate,{locale}{script} """  # wrap for E501
-                    """{method} {output} $(MDDEPDIR)/{dep_file} {stub}{inputs}{flags})
-\t@$(TOUCH) $@
-
-""").format(
-                    stub=stub_file,
-                    output=first_output,
-                    dep_file=dep_file,
-                    inputs=' ' + ' '.join(inputs) if inputs else '',
-                    flags=' ' + ' '.join(shell_quote(f) for f in obj.flags) if obj.flags else '',
-                    backend=' backend.mk' if obj.flags else '',
-                    # Locale repacks repack multiple locales from a single configured objdir,
-                    # so standard mtime dependencies won't work properly when the build is re-run
-                    # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set
-                    # in this situation, so simply force the generation to run in that case.
-                    force=force,
-                    locale='--locale=$(AB_CD) ' if obj.localized else '',
-                    script=obj.script,
-                    method=obj.method
-                    )
-                )
+            backend_file.write_once('include $(topsrcdir)/config/AB_rCD.mk\n')
+            for stmt in self._format_statements_for_generated_file(
+                    obj, tier,
+                    extra_dependencies='backend.mk' if obj.flags else ''):
+                backend_file.write(stmt + '\n')
 
         elif isinstance(obj, JARManifest):
             self._no_skip['libs'].add(backend_file.relobjdir)
             backend_file.write('JAR_MANIFEST := %s\n' % obj.path.full_path)
 
         elif isinstance(obj, RustProgram):
             self._process_rust_program(obj, backend_file)
             # Hook the program into the compile graph.
@@ -1828,8 +1734,33 @@ class RecursiveMakeBackend(CommonBackend
         with self._write_file(webidls_mk) as fh:
             mk.dump(fh, removal_guard=False)
 
         # Add the test directory to the compile graph.
         if self.environment.substs.get('ENABLE_TESTS'):
             self._compile_graph[mozpath.join(
                 mozpath.relpath(bindings_dir, self.environment.topobjdir),
                 'test', 'target-objects')]
+
+    def _format_generated_file_input_name(self, path, obj):
+        if obj.localized:
+            # Localized generated files can have locale-specific inputs, which
+            # are indicated by paths starting with `en-US/` or containing
+            # `locales/en-US/`.
+            if 'locales/en-US' in path:
+                # We need an "absolute source path" relative to
+                # topsrcdir, like "/source/path".
+                if not path.startswith('/'):
+                    path = '/' + mozpath.relpath(path.full_path, obj.topsrcdir)
+                e, f = path.split('locales/en-US/', 1)
+                assert(f)
+                return '$(call MERGE_RELATIVE_FILE,{},{}locales)'.format(
+                    f, e if not e.startswith('/') else e[len('/'):])
+            elif path.startswith('en-US/'):
+                e, f = path.split('en-US/', 1)
+                assert(not e)
+                return '$(call MERGE_FILE,%s)' % f
+            return self._pretty_path(path, self._get_backend_file_for(obj))
+        else:
+            return self._pretty_path(path, self._get_backend_file_for(obj))
+
+    def _format_generated_file_output_name(self, path, obj):
+        return path
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -394,30 +394,31 @@ class TestRecursiveMakeBackend(BackendTe
     def test_generated_files(self):
         """Ensure GENERATED_FILES is handled properly."""
         env = self._consume('generated-files', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
+            'include $(topsrcdir)/config/AB_rCD.mk',
             'bar.c: $(MDDEPDIR)/bar.c.stub ;',
             'GARBAGE += bar.c',
             'GARBAGE += $(MDDEPDIR)/bar.c.stub',
-            'EXTRA_MDDEPEND_FILES += bar.c.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/bar.c.pp',
             '$(MDDEPDIR)/bar.c.stub: %s/generate-bar.py' % env.topsrcdir,
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
             'export:: $(MDDEPDIR)/foo.h.stub',
             'foo.h: $(MDDEPDIR)/foo.h.stub ;',
             'GARBAGE += foo.h',
             'GARBAGE += $(MDDEPDIR)/foo.h.stub',
-            'EXTRA_MDDEPEND_FILES += foo.h.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo.h.pp',
             '$(MDDEPDIR)/foo.h.stub: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir),
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,%s/generate-foo.py main foo.h $(MDDEPDIR)/foo.h.pp $(MDDEPDIR)/foo.h.stub $(srcdir)/foo-data)' % (env.topsrcdir),  # noqa
             '@$(TOUCH) $@',
             '',
         ]
 
         self.maxDiff = None
@@ -426,29 +427,30 @@ class TestRecursiveMakeBackend(BackendTe
     def test_generated_files_force(self):
         """Ensure GENERATED_FILES with .force is handled properly."""
         env = self._consume('generated-files-force', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
+            'include $(topsrcdir)/config/AB_rCD.mk',
             'bar.c: $(MDDEPDIR)/bar.c.stub ;',
             'GARBAGE += bar.c',
             'GARBAGE += $(MDDEPDIR)/bar.c.stub',
-            'EXTRA_MDDEPEND_FILES += bar.c.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/bar.c.pp',
             '$(MDDEPDIR)/bar.c.stub: %s/generate-bar.py FORCE' % env.topsrcdir,
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
             'foo.c: $(MDDEPDIR)/foo.c.stub ;',
             'GARBAGE += foo.c',
             'GARBAGE += $(MDDEPDIR)/foo.c.stub',
-            'EXTRA_MDDEPEND_FILES += foo.c.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo.c.pp',
             '$(MDDEPDIR)/foo.c.stub: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir),
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(MDDEPDIR)/foo.c.stub $(srcdir)/foo-data)' % (env.topsrcdir),  # noqa
             '@$(TOUCH) $@',
             '',
         ]
 
         self.maxDiff = None
@@ -457,21 +459,22 @@ class TestRecursiveMakeBackend(BackendTe
     def test_localized_generated_files(self):
         """Ensure LOCALIZED_GENERATED_FILES is handled properly."""
         env = self._consume('localized-generated-files', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
+            'include $(topsrcdir)/config/AB_rCD.mk',
             'libs:: $(MDDEPDIR)/foo.xyz.stub',
             'foo.xyz: $(MDDEPDIR)/foo.xyz.stub ;',
             'GARBAGE += foo.xyz',
             'GARBAGE += $(MDDEPDIR)/foo.xyz.stub',
-            'EXTRA_MDDEPEND_FILES += foo.xyz.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo.xyz.pp',
             '$(MDDEPDIR)/foo.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(MDDEPDIR)/foo.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
             'LOCALIZED_FILES_0_FILES += foo.xyz',
             'LOCALIZED_FILES_0_DEST = $(FINAL_TARGET)/',
             'LOCALIZED_FILES_0_TARGET := libs',
@@ -484,31 +487,32 @@ class TestRecursiveMakeBackend(BackendTe
     def test_localized_generated_files_force(self):
         """Ensure LOCALIZED_GENERATED_FILES with .force is handled properly."""
         env = self._consume('localized-generated-files-force', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
+            'include $(topsrcdir)/config/AB_rCD.mk',
             'libs:: $(MDDEPDIR)/foo.xyz.stub',
             'foo.xyz: $(MDDEPDIR)/foo.xyz.stub ;',
             'GARBAGE += foo.xyz',
             'GARBAGE += $(MDDEPDIR)/foo.xyz.stub',
-            'EXTRA_MDDEPEND_FILES += foo.xyz.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo.xyz.pp',
             '$(MDDEPDIR)/foo.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(MDDEPDIR)/foo.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
             'libs:: $(MDDEPDIR)/abc.xyz.stub',
             'abc.xyz: $(MDDEPDIR)/abc.xyz.stub ;',
             'GARBAGE += abc.xyz',
             'GARBAGE += $(MDDEPDIR)/abc.xyz.stub',
-            'EXTRA_MDDEPEND_FILES += abc.xyz.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/abc.xyz.pp',
             '$(MDDEPDIR)/abc.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input FORCE' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main abc.xyz $(MDDEPDIR)/abc.xyz.pp $(MDDEPDIR)/abc.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
         ]
 
         self.maxDiff = None
@@ -518,40 +522,40 @@ class TestRecursiveMakeBackend(BackendTe
         """Ensure LOCALIZED_GENERATED_FILES is handled properly
         when {AB_CD} and {AB_rCD} are used."""
         env = self._consume('localized-generated-files-AB_CD', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
+            'include $(topsrcdir)/config/AB_rCD.mk',
             'libs:: $(MDDEPDIR)/foo$(AB_CD).xyz.stub',
             'foo$(AB_CD).xyz: $(MDDEPDIR)/foo$(AB_CD).xyz.stub ;',
             'GARBAGE += foo$(AB_CD).xyz',
             'GARBAGE += $(MDDEPDIR)/foo$(AB_CD).xyz.stub',
-            'EXTRA_MDDEPEND_FILES += foo$(AB_CD).xyz.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo$(AB_CD).xyz.pp',
             '$(MDDEPDIR)/foo$(AB_CD).xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo$(AB_CD).xyz $(MDDEPDIR)/foo$(AB_CD).xyz.pp $(MDDEPDIR)/foo$(AB_CD).xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
-            'include $(topsrcdir)/config/AB_rCD.mk',
             'bar$(AB_rCD).xyz: $(MDDEPDIR)/bar$(AB_rCD).xyz.stub ;',
             'GARBAGE += bar$(AB_rCD).xyz',
             'GARBAGE += $(MDDEPDIR)/bar$(AB_rCD).xyz.stub',
-            'EXTRA_MDDEPEND_FILES += bar$(AB_rCD).xyz.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/bar$(AB_rCD).xyz.pp',
             '$(MDDEPDIR)/bar$(AB_rCD).xyz.stub: %s/generate-foo.py $(call MERGE_RELATIVE_FILE,localized-input,inner/locales) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main bar$(AB_rCD).xyz $(MDDEPDIR)/bar$(AB_rCD).xyz.pp $(MDDEPDIR)/bar$(AB_rCD).xyz.stub $(call MERGE_RELATIVE_FILE,localized-input,inner/locales) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
             'zot$(AB_rCD).xyz: $(MDDEPDIR)/zot$(AB_rCD).xyz.stub ;',
             'GARBAGE += zot$(AB_rCD).xyz',
             'GARBAGE += $(MDDEPDIR)/zot$(AB_rCD).xyz.stub',
-            'EXTRA_MDDEPEND_FILES += zot$(AB_rCD).xyz.pp',
+            'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/zot$(AB_rCD).xyz.pp',
             '$(MDDEPDIR)/zot$(AB_rCD).xyz.stub: %s/generate-foo.py $(call MERGE_RELATIVE_FILE,localized-input,locales) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main zot$(AB_rCD).xyz $(MDDEPDIR)/zot$(AB_rCD).xyz.pp $(MDDEPDIR)/zot$(AB_rCD).xyz.stub $(call MERGE_RELATIVE_FILE,localized-input,locales) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
         ]
 
         self.maxDiff = None
@@ -574,16 +578,17 @@ class TestRecursiveMakeBackend(BackendTe
         self.assertIn('mozilla/mozilla2.h', m)
         self.assertIn('mozilla/dom/dom2.h', m)
         self.assertIn('mozilla/dom/dom3.h', m)
         # EXPORTS files that are also GENERATED_FILES should be handled as
         # INSTALL_TARGETS.
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
         expected = [
+            'include $(topsrcdir)/config/AB_rCD.mk',
             'dist_include_FILES += bar.h',
             'dist_include_DEST := $(DEPTH)/dist/include/',
             'dist_include_TARGET := export',
             'INSTALL_TARGETS += dist_include',
             'dist_include_mozilla_FILES += mozilla2.h',
             'dist_include_mozilla_DEST := $(DEPTH)/dist/include/mozilla',
             'dist_include_mozilla_TARGET := export',
             'INSTALL_TARGETS += dist_include_mozilla',