Bug 784841 - Part 16: Use moz.build files to build the tree; r=ted, glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 25 Feb 2013 12:47:11 -0800
changeset 123219 23673de2aa432551cb25c993238ccc1be2ae9b1f
parent 123218 c1b0d8399be92a0a541fddec5e580196e7b83e32
child 123220 63e2e94efd442d87f0ea474520bae573f70996ae
push id24376
push userMs2ger@gmail.com
push dateThu, 28 Feb 2013 17:05:14 +0000
treeherdermozilla-central@c65d59d33aa8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted, glandium
bugs784841
milestone22.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 784841 - Part 16: Use moz.build files to build the tree; r=ted, glandium
build/ConfigStatus.py
config/config.mk
config/rules.mk
js/src/build/ConfigStatus.py
js/src/config/config.mk
js/src/config/rules.mk
media/webrtc/trunk/tools/gyp/pylib/gyp/generator/mozmake.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/build/ConfigStatus.py
+++ b/build/ConfigStatus.py
@@ -1,25 +1,34 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Combined with build/autoconf/config.status.m4, ConfigStatus is an almost
 # drop-in replacement for autoconf 2.13's config.status, with features
 # borrowed from autoconf > 2.5, and additional features.
 
+import logging
 import os
 import sys
 
 from optparse import OptionParser
 
+from mach.logging import LoggingManager
 from mozbuild.backend.configenvironment import ConfigEnvironment
+from mozbuild.backend.recursivemake import RecursiveMakeBackend
+from mozbuild.frontend.emitter import TreeMetadataEmitter
+from mozbuild.frontend.reader import BuildReader
 
 from Preprocessor import Preprocessor
 
+
+log_manager = LoggingManager()
+
+
 # Basic logging facility
 verbose = False
 def log(string):
     if verbose:
         print >>sys.stderr, string
 
 
 def config_status(topobjdir = '.', topsrcdir = '.',
@@ -78,36 +87,52 @@ def config_status(topobjdir = '.', topsr
 
     # Without -n, the current directory is meant to be the top object directory
     if not options.not_topobjdir:
         topobjdir = '.'
 
     env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
             non_global_defines=non_global_defines, substs=substs)
 
+    reader = BuildReader(env)
+    emitter = TreeMetadataEmitter(env)
+    backend = RecursiveMakeBackend(env)
+    # This won't actually do anything because of the magic of generators.
+    definitions = emitter.emit(reader.read_topsrcdir())
+
     if options.recheck:
         # Execute configure from the top object directory
         if not os.path.isabs(topsrcdir):
             topsrcdir = relpath(topsrcdir, topobjdir)
         os.chdir(topobjdir)
         os.execlp('sh', 'sh', '-c', ' '.join([os.path.join(topsrcdir, 'configure'), env.substs['ac_configure_args'], '--no-create', '--no-recursion']))
 
     if options.files:
         files = options.files
         headers = []
     if options.headers:
         headers = options.headers
         if not options.files:
             files = []
     # Default to display messages when giving --file or --headers on the
     # command line.
+    log_level = logging.INFO
+
     if options.files or options.headers or options.verbose:
         global verbose
         verbose = True
+        log_level = logging.DEBUG
+
+    log_manager.add_terminal_logging(level=log_level)
+    log_manager.enable_unstructured()
+
     if not options.files and not options.headers:
         print >>sys.stderr, "creating config files and headers..."
+
+        backend.consume(definitions)
+
         files = [os.path.join(topobjdir, f) for f in files]
         headers = [os.path.join(topobjdir, f) for f in headers]
 
     for file in files:
         env.create_config_file(file)
     for header in headers:
         env.create_config_header(header)
--- a/config/config.mk
+++ b/config/config.mk
@@ -728,21 +728,16 @@ endif # ! OS2
 
 # Make sure any compiled classes work with at least JVM 1.4
 JAVAC_FLAGS += -source 1.4
 
 ifdef MOZ_DEBUG
 JAVAC_FLAGS += -g
 endif
 
-ifdef TIERS
-DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
-STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
-endif
-
 CREATE_PRECOMPLETE_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/createprecomplete.py)
 
 # MDDEPDIR is the subdirectory where dependency files are stored
 MDDEPDIR := .deps
 
 EXPAND_LIBS_EXEC = $(PYTHON) $(topsrcdir)/config/expandlibs_exec.py $(if $@,--depend $(MDDEPDIR)/$(@F).pp --target $@)
 EXPAND_LIBS_GEN = $(PYTHON) $(topsrcdir)/config/expandlibs_gen.py $(if $@,--depend $(MDDEPDIR)/$(@F).pp)
 EXPAND_AR = $(EXPAND_LIBS_EXEC) --extract -- $(AR)
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -5,16 +5,47 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 
 ifndef topsrcdir
 $(error topsrcdir was not set))
 endif
 
+# Integrate with mozbuild-generated make files. We first verify that no
+# variables provided by the automatically generated .mk files are
+# present. If they are, this is a violation of the separation of
+# responsibility between Makefile.in and mozbuild files.
+_MOZBUILD_EXTERNAL_VARIABLES := \
+  DIRS \
+  PARALLEL_DIRS \
+  TEST_DIRS \
+  TIERS \
+  TOOL_DIRS \
+  $(NULL)
+
+ifndef EXTERNALLY_MANAGED_MAKE_FILE
+$(foreach var,$(_MOZBUILD_EXTERNAL_VARIABLES),$(if $($(var)),\
+    $(error Variable $(var) is defined in Makefile. It should only be defined in moz.build files.),\
+    ))
+
+# Import the automatically generated backend file. If this file doesn't exist,
+# the backend hasn't been properly configured. We want this to be a fatal
+# error, hence not using "-include".
+ifndef STANDALONE_MAKEFILE
+GLOBAL_DEPS += backend.mk
+include backend.mk
+endif
+endif
+
+ifdef TIERS
+DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
+STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
+endif
+
 ifndef MOZILLA_DIR
 MOZILLA_DIR = $(topsrcdir)
 endif
 
 ifndef INCLUDED_CONFIG_MK
 include $(topsrcdir)/config/config.mk
 endif
 
@@ -1142,19 +1173,39 @@ endif
 			-sourcepath $(_JAVA_SOURCEPATH) -d $(_JAVA_DIR) $(_VPATH_SRCS)
 
 $(JAVA_LIBRARY): $(addprefix $(_JAVA_DIR)/,$(JAVA_SRCS:.java=.class)) $(GLOBAL_DEPS)
 	$(JAR) cf $@ -C $(_JAVA_DIR) .
 
 GARBAGE_DIRS += $(_JAVA_DIR)
 
 ###############################################################################
-# Update Makefiles
+# Update Files Managed by Build Backend
 ###############################################################################
 
+ifdef MOZBUILD_DERIVED
+
+# If this Makefile is derived from moz.build files, substitution for all .in
+# files is handled by SUBSTITUTE_FILES. This includes Makefile.in.
+ifneq ($(SUBSTITUTE_FILES),,)
+$(SUBSTITUTE_FILES): % : $(srcdir)/%.in $(DEPTH)/config/autoconf.mk
+	@$(PYTHON) $(DEPTH)/config.status -n --file=$@
+	@$(TOUCH) $@
+endif
+
+# Detect when the backend.mk needs rebuilt. This will cause a full scan and
+# rebuild. While relatively expensive, it should only occur once per recursion.
+ifneq ($(BACKEND_INPUT_FILES),,)
+backend.mk: $(BACKEND_INPUT_FILES)
+	@$(PYTHON) $(DEPTH)/config.status -n
+	@$(TOUCH) $@
+endif
+
+endif # MOZBUILD_DERIVED
+
 ifndef NO_MAKEFILE_RULE
 Makefile: Makefile.in
 	@$(PYTHON) $(DEPTH)/config.status -n --file=Makefile
 	@$(TOUCH) $@
 endif
 
 ifndef NO_SUBMAKEFILES_RULE
 ifdef SUBMAKEFILES
--- a/js/src/build/ConfigStatus.py
+++ b/js/src/build/ConfigStatus.py
@@ -1,25 +1,34 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Combined with build/autoconf/config.status.m4, ConfigStatus is an almost
 # drop-in replacement for autoconf 2.13's config.status, with features
 # borrowed from autoconf > 2.5, and additional features.
 
+import logging
 import os
 import sys
 
 from optparse import OptionParser
 
+from mach.logging import LoggingManager
 from mozbuild.backend.configenvironment import ConfigEnvironment
+from mozbuild.backend.recursivemake import RecursiveMakeBackend
+from mozbuild.frontend.emitter import TreeMetadataEmitter
+from mozbuild.frontend.reader import BuildReader
 
 from Preprocessor import Preprocessor
 
+
+log_manager = LoggingManager()
+
+
 # Basic logging facility
 verbose = False
 def log(string):
     if verbose:
         print >>sys.stderr, string
 
 
 def config_status(topobjdir = '.', topsrcdir = '.',
@@ -78,36 +87,52 @@ def config_status(topobjdir = '.', topsr
 
     # Without -n, the current directory is meant to be the top object directory
     if not options.not_topobjdir:
         topobjdir = '.'
 
     env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
             non_global_defines=non_global_defines, substs=substs)
 
+    reader = BuildReader(env)
+    emitter = TreeMetadataEmitter(env)
+    backend = RecursiveMakeBackend(env)
+    # This won't actually do anything because of the magic of generators.
+    definitions = emitter.emit(reader.read_topsrcdir())
+
     if options.recheck:
         # Execute configure from the top object directory
         if not os.path.isabs(topsrcdir):
             topsrcdir = relpath(topsrcdir, topobjdir)
         os.chdir(topobjdir)
         os.execlp('sh', 'sh', '-c', ' '.join([os.path.join(topsrcdir, 'configure'), env.substs['ac_configure_args'], '--no-create', '--no-recursion']))
 
     if options.files:
         files = options.files
         headers = []
     if options.headers:
         headers = options.headers
         if not options.files:
             files = []
     # Default to display messages when giving --file or --headers on the
     # command line.
+    log_level = logging.INFO
+
     if options.files or options.headers or options.verbose:
         global verbose
         verbose = True
+        log_level = logging.DEBUG
+
+    log_manager.add_terminal_logging(level=log_level)
+    log_manager.enable_unstructured()
+
     if not options.files and not options.headers:
         print >>sys.stderr, "creating config files and headers..."
+
+        backend.consume(definitions)
+
         files = [os.path.join(topobjdir, f) for f in files]
         headers = [os.path.join(topobjdir, f) for f in headers]
 
     for file in files:
         env.create_config_file(file)
     for header in headers:
         env.create_config_header(header)
--- a/js/src/config/config.mk
+++ b/js/src/config/config.mk
@@ -728,21 +728,16 @@ endif # ! OS2
 
 # Make sure any compiled classes work with at least JVM 1.4
 JAVAC_FLAGS += -source 1.4
 
 ifdef MOZ_DEBUG
 JAVAC_FLAGS += -g
 endif
 
-ifdef TIERS
-DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
-STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
-endif
-
 CREATE_PRECOMPLETE_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/createprecomplete.py)
 
 # MDDEPDIR is the subdirectory where dependency files are stored
 MDDEPDIR := .deps
 
 EXPAND_LIBS_EXEC = $(PYTHON) $(topsrcdir)/config/expandlibs_exec.py $(if $@,--depend $(MDDEPDIR)/$(@F).pp --target $@)
 EXPAND_LIBS_GEN = $(PYTHON) $(topsrcdir)/config/expandlibs_gen.py $(if $@,--depend $(MDDEPDIR)/$(@F).pp)
 EXPAND_AR = $(EXPAND_LIBS_EXEC) --extract -- $(AR)
--- a/js/src/config/rules.mk
+++ b/js/src/config/rules.mk
@@ -5,16 +5,47 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 
 ifndef topsrcdir
 $(error topsrcdir was not set))
 endif
 
+# Integrate with mozbuild-generated make files. We first verify that no
+# variables provided by the automatically generated .mk files are
+# present. If they are, this is a violation of the separation of
+# responsibility between Makefile.in and mozbuild files.
+_MOZBUILD_EXTERNAL_VARIABLES := \
+  DIRS \
+  PARALLEL_DIRS \
+  TEST_DIRS \
+  TIERS \
+  TOOL_DIRS \
+  $(NULL)
+
+ifndef EXTERNALLY_MANAGED_MAKE_FILE
+$(foreach var,$(_MOZBUILD_EXTERNAL_VARIABLES),$(if $($(var)),\
+    $(error Variable $(var) is defined in Makefile. It should only be defined in moz.build files.),\
+    ))
+
+# Import the automatically generated backend file. If this file doesn't exist,
+# the backend hasn't been properly configured. We want this to be a fatal
+# error, hence not using "-include".
+ifndef STANDALONE_MAKEFILE
+GLOBAL_DEPS += backend.mk
+include backend.mk
+endif
+endif
+
+ifdef TIERS
+DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
+STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
+endif
+
 ifndef MOZILLA_DIR
 MOZILLA_DIR = $(topsrcdir)
 endif
 
 ifndef INCLUDED_CONFIG_MK
 include $(topsrcdir)/config/config.mk
 endif
 
@@ -1142,19 +1173,39 @@ endif
 			-sourcepath $(_JAVA_SOURCEPATH) -d $(_JAVA_DIR) $(_VPATH_SRCS)
 
 $(JAVA_LIBRARY): $(addprefix $(_JAVA_DIR)/,$(JAVA_SRCS:.java=.class)) $(GLOBAL_DEPS)
 	$(JAR) cf $@ -C $(_JAVA_DIR) .
 
 GARBAGE_DIRS += $(_JAVA_DIR)
 
 ###############################################################################
-# Update Makefiles
+# Update Files Managed by Build Backend
 ###############################################################################
 
+ifdef MOZBUILD_DERIVED
+
+# If this Makefile is derived from moz.build files, substitution for all .in
+# files is handled by SUBSTITUTE_FILES. This includes Makefile.in.
+ifneq ($(SUBSTITUTE_FILES),,)
+$(SUBSTITUTE_FILES): % : $(srcdir)/%.in $(DEPTH)/config/autoconf.mk
+	@$(PYTHON) $(DEPTH)/config.status -n --file=$@
+	@$(TOUCH) $@
+endif
+
+# Detect when the backend.mk needs rebuilt. This will cause a full scan and
+# rebuild. While relatively expensive, it should only occur once per recursion.
+ifneq ($(BACKEND_INPUT_FILES),,)
+backend.mk: $(BACKEND_INPUT_FILES)
+	@$(PYTHON) $(DEPTH)/config.status -n
+	@$(TOUCH) $@
+endif
+
+endif # MOZBUILD_DERIVED
+
 ifndef NO_MAKEFILE_RULE
 Makefile: Makefile.in
 	@$(PYTHON) $(DEPTH)/config.status -n --file=Makefile
 	@$(TOUCH) $@
 endif
 
 ifndef NO_SUBMAKEFILES_RULE
 ifdef SUBMAKEFILES
--- a/media/webrtc/trunk/tools/gyp/pylib/gyp/generator/mozmake.py
+++ b/media/webrtc/trunk/tools/gyp/pylib/gyp/generator/mozmake.py
@@ -1,15 +1,12 @@
 # Copyright (c) 2012 Mozilla Foundation. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# Python 2.5 needs this for the with statement.
-from __future__ import with_statement
-
 import collections
 import gyp
 import gyp.common
 import sys
 import os
 import re
 import shlex
 
@@ -30,16 +27,18 @@ for unused in ['RULE_INPUT_PATH', 'RULE_
   generator_default_variables[unused] = ''
 
 COMMON_HEADER = """# This makefile was automatically generated from %(buildfile)s. Please do not edit it directly.
 DEPTH		= %(depth)s
 topsrcdir	= %(topsrcdir)s
 srcdir          = %(srcdir)s
 VPATH           = %(srcdir)s
 
+EXTERNALLY_MANAGED_MAKE_FILE := 1
+
 """
 
 COMMON_FOOTER = """
 # Skip rules that deal with regenerating Makefiles from Makefile.in files.
 NO_MAKEFILE_RULE = 1
 NO_SUBMAKEFILES_RULE = 1
 
 include $(topsrcdir)/config/rules.mk
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1,57 +1,117 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
+import errno
 import logging
 import os
 
 from .base import BuildBackend
 from ..frontend.data import (
     ConfigFileSubstitution,
     DirectoryTraversal,
 )
 from ..util import FileAvoidWrite
 
 
 class BackendMakeFile(object):
     """Represents a generated backend.mk file.
 
-    This is both a wrapper around FileAvoidWrite as well as a container that
+    This is both a wrapper around a file handle as well as a container that
     holds accumulated state.
+
+    It's worth taking a moment to explain the make dependencies. The
+    generated backend.mk as well as the Makefile.in (if it exists) are in the
+    GLOBAL_DEPS list. This means that if one of them changes, all targets
+    in that Makefile are invalidated. backend.mk also depends on all of its
+    input files.
+
+    It's worth considering the effect of file mtimes on build behavior.
+
+    Since we perform an "all or none" traversal of moz.build files (the whole
+    tree is scanned as opposed to individual files), if we were to blindly
+    write backend.mk files, the net effect of updating a single mozbuild file
+    in the tree is all backend.mk files have new mtimes. This would in turn
+    invalidate all make targets across the whole tree! This would effectively
+    undermine incremental builds as any mozbuild change would cause the entire
+    tree to rebuild!
+
+    The solution is to not update the mtimes of backend.mk files unless they
+    actually change. We use FileAvoidWrite to accomplish this. However, this
+    puts us in a somewhat complicated position when it comes to tree recursion.
+    As you are recursing the tree, the first time you come across a backend.mk
+    that is out of date, a full tree build will be incurred. In typical make
+    build systems, we would touch the out-of-date target (backend.mk) to ensure
+    its mtime is newer than all its dependencies - even if the contents did
+    not change. However, we can't rely on just this approach. During recursion,
+    the first trigger of backend generation will cause only that backend.mk to
+    update. If there is another backend.mk that is also out of date according
+    to mtime but whose contents were not changed, when we recurse to that
+    directory, make will trigger another full backend generation! This would
+    be completely redundant and would slow down builds! This is not acceptable.
+
+    We work around this problem by having backend generation update the mtime
+    of backend.mk if they are older than their inputs - even if the file
+    contents did not change. This is essentially a middle ground between
+    always updating backend.mk and only updating the backend.mk that was out
+    of date during recursion.
     """
 
     def __init__(self, srcdir, objdir):
         self.srcdir = srcdir
         self.objdir = objdir
+        self.path = os.path.join(objdir, 'backend.mk')
 
         # Filenames that influenced the content of this file.
         self.inputs = set()
 
-        # Filenames that are automatically generated by the build backend.
-        self.outputs = set()
+        self.fh = FileAvoidWrite(self.path)
+        self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
+        self.fh.write('\n')
+        self.fh.write('MOZBUILD_DERIVED := 1\n')
 
-        self.fh = FileAvoidWrite(os.path.join(objdir, 'backend.mk'))
-        self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
+        # SUBSTITUTE_FILES handles Makefile.in -> Makefile conversion.
+        self.fh.write('NO_MAKEFILE_RULE := 1\n')
+
 
     def write(self, buf):
         self.fh.write(buf)
 
     def close(self):
-        if len(self.inputs):
-            self.fh.write('BACKEND_INPUT_FILES += %s\n' % ' '.join(self.inputs))
-
-        if len(self.outputs):
-            self.fh.write('BACKEND_OUTPUT_FILES += %s\n' % ' '.join(self.outputs))
+        if self.inputs:
+            l = ' '.join(sorted(self.inputs))
+            self.fh.write('BACKEND_INPUT_FILES += %s\n' % l)
 
         self.fh.close()
 
+        if not self.inputs:
+            return
+
+        # Update mtime iff any of its input files are newer. See class notes
+        # for why we do this.
+        existing_mtime = os.path.getmtime(self.path)
+
+        def mtime(path):
+            try:
+                return os.path.getmtime(path)
+            except OSError as e:
+                if e.errno == errno.ENOENT:
+                    return 0
+
+                raise
+
+        input_mtime = max(mtime(path) for path in self.inputs)
+
+        if input_mtime > existing_mtime:
+            os.utime(self.path, None)
+
 
 class RecursiveMakeBackend(BuildBackend):
     """Backend that integrates with the existing recursive make build system.
 
     This backend facilitates the transition from Makefile.in to moz.build
     files.
 
     This backend performs Makefile.in -> Makefile conversion. It also writes
@@ -67,23 +127,28 @@ class RecursiveMakeBackend(BuildBackend)
         self._backend_files = {}
 
     def consume_object(self, obj):
         """Write out build files necessary to build with recursive make."""
 
         backend_file = self._backend_files.get(obj.srcdir,
             BackendMakeFile(obj.srcdir, obj.objdir))
 
+        # Define the paths that will trigger a backend rebuild. We always
+        # add autoconf.mk because that is proxy for CONFIG. We can't use
+        # config.status because there is no make target for that!
+        autoconf_path = os.path.join(obj.topobjdir, 'config', 'autoconf.mk')
+        backend_file.inputs.add(autoconf_path)
         backend_file.inputs |= obj.sandbox_all_paths
 
         if isinstance(obj, DirectoryTraversal):
             self._process_directory_traversal(obj, backend_file)
         elif isinstance(obj, ConfigFileSubstitution):
-            backend_file.inputs.add(obj.input_path)
-            backend_file.outputs.add(obj.output_path)
+            backend_file.write('SUBSTITUTE_FILES += %s\n' % obj.relpath)
+
             self.environment.create_config_file(obj.output_path)
 
         self._backend_files[obj.srcdir] = backend_file
 
     def consume_finished(self):
         for srcdir in sorted(self._backend_files.keys()):
             bf = self._backend_files[srcdir]
 
@@ -94,18 +159,18 @@ class RecursiveMakeBackend(BuildBackend)
 
             if not os.path.exists(makefile_in):
                 raise Exception('Could not find Makefile.in: %s' % makefile_in)
 
             out_path = os.path.join(bf.objdir, 'Makefile')
             self.log(logging.DEBUG, 'create_makefile', {'path': out_path},
                 'Generating makefile: {path}')
             self.environment.create_config_file(out_path)
-            bf.outputs.add(out_path)
 
+            bf.write('SUBSTITUTE_FILES += Makefile\n')
             bf.close()
 
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
@@ -113,16 +178,19 @@ class RecursiveMakeBackend(BuildBackend)
             if dirs:
                 fh.write('tier_%s_dirs += %s\n' % (tier, ' '.join(dirs)))
 
             # tier_static_dirs should have the same keys as tier_dirs.
             if obj.tier_static_dirs[tier]:
                 fh.write('tier_%s_staticdirs += %s\n' % (
                     tier, ' '.join(obj.tier_static_dirs[tier])))
 
+                static = ' '.join(obj.tier_static_dirs[tier])
+                fh.write('EXTERNAL_DIRS += %s\n' % static)
+
         if obj.dirs:
             fh.write('DIRS := %s\n' % ' '.join(obj.dirs))
 
         if obj.parallel_dirs:
             fh.write('PARALLEL_DIRS := %s\n' % ' '.join(obj.parallel_dirs))
 
         if obj.tool_dirs:
             fh.write('TOOL_DIRS := %s\n' % ' '.join(obj.tool_dirs))
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -97,16 +97,18 @@ class ConfigFileSubstitution(SandboxDeri
     """Describes a config file that will be generated using substitutions.
 
     The output_path attribute defines the relative path from topsrcdir of the
     output file to generate.
     """
     __slots__ = (
         'input_path',
         'output_path',
+        'relpath',
     )
 
     def __init__(self, sandbox):
         SandboxDerived.__init__(self, sandbox)
 
         self.input_path = None
         self.output_path = None
+        self.relpath = None
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -50,16 +50,17 @@ class TreeMetadataEmitter(object):
 
         for path in sandbox['CONFIGURE_SUBST_FILES']:
             if os.path.isabs(path):
                 path = path[1:]
 
             sub = ConfigFileSubstitution(sandbox)
             sub.input_path = os.path.join(sandbox['SRCDIR'], '%s.in' % path)
             sub.output_path = os.path.join(sandbox['OBJDIR'], path)
+            sub.relpath = path
             yield sub
 
     def _emit_directory_traversal_from_sandbox(self, sandbox):
         o = DirectoryTraversal(sandbox)
         o.dirs = sandbox.get('DIRS', [])
         o.parallel_dirs = sandbox.get('PARALLEL_DIRS', [])
         o.tool_dirs = sandbox.get('TOOL_DIRS', [])
         o.test_dirs = sandbox.get('TEST_DIRS', [])
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -97,18 +97,27 @@ class MozbuildSandbox(Sandbox):
 
         with self._globals.allow_all_writes() as d:
             d['TOPSRCDIR'] = config.topsrcdir
             d['TOPOBJDIR'] = topobjdir
             d['RELATIVEDIR'] = reldir
             d['SRCDIR'] = os.path.join(config.topsrcdir, reldir).replace(os.sep, '/').rstrip('/')
             d['OBJDIR'] = os.path.join(topobjdir, reldir).replace(os.sep, '/').rstrip('/')
 
-            d['CONFIG'] = ReadOnlyDefaultDict(config.substs,
-                global_default=None)
+            # config.status does not yet use unicode. However, mozbuild expects
+            # unicode everywhere. So, decode binary into unicode as necessary.
+            # Bug 844509 tracks a better way to do this.
+            substs = {}
+            for k, v in config.substs.items():
+                if not isinstance(v, text_type):
+                    v = v.decode('utf-8', 'strict')
+
+                substs[k] = v
+
+            d['CONFIG'] = ReadOnlyDefaultDict(substs, global_default=None)
 
             # Register functions.
             for name, func in FUNCTIONS.items():
                 d[name] = getattr(self, func[0])
 
         self._normalized_topsrcdir = os.path.normpath(config.topsrcdir)
 
     def exec_file(self, path, filesystem_absolute=False):
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
+import time
 
 from mozunit import main
 
 from mozbuild.backend.configenvironment import ConfigEnvironment
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
@@ -57,53 +58,58 @@ class TestRecursiveMakeBackend(BackendTe
         ])
 
     def test_backend_mk(self):
         """Ensure backend.mk file is written out properly."""
         env = self._consume('stub0', RecursiveMakeBackend)
 
         p = os.path.join(env.topobjdir, 'backend.mk')
 
-        lines = [l.strip() for l in open(p, 'rt').readlines()[1:-2]]
+        lines = [l.strip() for l in open(p, 'rt').readlines()[2:-1]]
         self.assertEqual(lines, [
+            'MOZBUILD_DERIVED := 1',
+            'NO_MAKEFILE_RULE := 1',
             'DIRS := dir1',
             'PARALLEL_DIRS := dir2',
             'TEST_DIRS := dir3',
+            'SUBSTITUTE_FILES += Makefile',
         ])
 
-    def test_no_mtime_bump(self):
+    def test_mtime_no_change(self):
         """Ensure mtime is not updated if file content does not change."""
 
         env = self._consume('stub0', RecursiveMakeBackend)
 
         makefile_path = os.path.join(env.topobjdir, 'Makefile')
         backend_path = os.path.join(env.topobjdir, 'backend.mk')
-
         makefile_mtime = os.path.getmtime(makefile_path)
         backend_mtime = os.path.getmtime(backend_path)
 
         reader = BuildReader(env)
         emitter = TreeMetadataEmitter(env)
         backend = RecursiveMakeBackend(env)
         backend.consume(emitter.emit(reader.read_topsrcdir()))
 
         self.assertEqual(os.path.getmtime(makefile_path), makefile_mtime)
         self.assertEqual(os.path.getmtime(backend_path), backend_mtime)
 
     def test_external_make_dirs(self):
         """Ensure we have make recursion into external make directories."""
         env = self._consume('external_make_dirs', RecursiveMakeBackend)
 
         backend_path = os.path.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[1:-2]]
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:-1]]
         self.assertEqual(lines, [
+            'MOZBUILD_DERIVED := 1',
+            'NO_MAKEFILE_RULE := 1',
             'DIRS := dir',
             'PARALLEL_DIRS := p_dir',
             'DIRS += external',
             'PARALLEL_DIRS += p_external',
+            'SUBSTITUTE_FILES += Makefile',
         ])
 
     def test_substitute_config_files(self):
         """Ensure substituted config files are produced."""
         env = self._consume('substitute_config_files', RecursiveMakeBackend)
 
         p = os.path.join(env.topobjdir, 'foo')
         self.assertTrue(os.path.exists(p))
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -99,16 +99,17 @@ class TestEmitterBasic(unittest.TestCase
         objs = list(emitter.emit(reader.read_topsrcdir()))
         self.assertEqual(len(objs), 3)
 
         self.assertIsInstance(objs[0], DirectoryTraversal)
         self.assertIsInstance(objs[1], ConfigFileSubstitution)
         self.assertIsInstance(objs[2], ConfigFileSubstitution)
 
         topobjdir = os.path.abspath(reader.config.topobjdir)
+        self.assertEqual(objs[1].relpath, 'foo')
         self.assertEqual(os.path.normpath(objs[1].output_path),
             os.path.normpath(os.path.join(topobjdir, 'foo')))
         self.assertEqual(os.path.normpath(objs[2].output_path),
             os.path.normpath(os.path.join(topobjdir, 'bar')))
 
 
 if __name__ == '__main__':
     main()