Bug 1416052 - Generate a make file to include in client.mk; r=nalexander
authorGregory Szorc <gps@mozilla.com>
Thu, 09 Nov 2017 22:27:16 -0800
changeset 443567 67c03dc70bfd5d2d3df5986d64240494f4624d55
parent 443566 4a26fbabb7d8d8080c1280f46e6eb7793fa134e3
child 443568 45d15eccf5504583d23b680c1ebbc4965ea6f314
push id8527
push userCallek@gmail.com
push dateThu, 11 Jan 2018 21:05:50 +0000
treeherdermozilla-beta@95342d212a7a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1416052
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1416052 - Generate a make file to include in client.mk; r=nalexander Currently, client.mk calls `mach environment` to obtain a make file to be evaluated in the context of client.mk. The reason it is implemented this way is because client.mk could be an entrypoint to the build system. With recent changes that require the use of mach to use client.mk, we are now guaranteed to have Python code running before client.mk is invoked. This means we don't need to invoke `mach` from client.mk. This commit ports the code for generating a client.mk suitable make file from `mach environment` to the build dispatcher. We now write out a new .mozconfig-client-mk file in the objdir. client.mk is changed to cat this file and to include it as a native make file. The OBJDIR environment variable is also set so client.mk knows where to read the auto-generated file from. This commit should be backwards compatible. Hopefully it is obvious, but this new make file is only temporary. As soon as the remaining mozconfig logic is moved out of client.mk, we should be able to simplify down to a single "include" in client.mk. MozReview-Commit-ID: BEfWo76Z1qA
client.mk
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/mach_commands.py
--- a/client.mk
+++ b/client.mk
@@ -44,18 +44,18 @@ define CR
 
 endef
 
 # As $(shell) doesn't preserve newlines, use sed to replace them with an
 # unlikely sequence (||), which is then replaced back to newlines by make
 # before evaluation. $(shell) replacing newlines with spaces, || is always
 # followed by a space (since sed doesn't remove newlines), except on the
 # last line, so replace both '|| ' and '||'.
-MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell $(TOPSRCDIR)/mach environment --format=client.mk | sed 's/$$/||/')))
-$(eval $(MOZCONFIG_CONTENT))
+MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell cat $(OBJDIR)/.mozconfig-client-mk | sed 's/$$/||/')))
+include $(OBJDIR)/.mozconfig-client-mk
 
 export FOUND_MOZCONFIG
 
 # As '||' was used as a newline separator, it means it's not occurring in
 # lines themselves. It can thus safely be used to replaces normal spaces,
 # to then replace newlines with normal spaces. This allows to get a list
 # of mozconfig output lines.
 MOZCONFIG_OUT_LINES := $(subst $(CR), ,$(subst $(NULL) $(NULL),||,$(MOZCONFIG_CONTENT)))
@@ -103,18 +103,17 @@ ifndef MACH
 $(error client.mk must be used via `mach`. Try running \
 `./mach $(firstword $(MAKECMDGOALS) $(.DEFAULT_GOAL))`)
 endif
 
 # Define mkdir
 include $(TOPSRCDIR)/config/makefiles/makeutils.mk
 include $(TOPSRCDIR)/config/makefiles/autotargets.mk
 
-# For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES
-# from mach environment --format=client.mk output.
+# For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES.
 MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES))
 $(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG) $(call mkdir_deps,$(OBJDIR)) $(OBJDIR)/CLOBBER
 	$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@
 
 # Include that makefile so that it is created. This should not actually change
 # the environment since MOZCONFIG_CONTENT, which MOZCONFIG_OUT_LINES derives
 # from, has already been eval'ed.
 include $(OBJDIR)/.mozconfig.mk
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -45,16 +45,17 @@ from ..testing import (
 from ..compilation.warnings import (
     WarningsCollector,
     WarningsDatabase,
 )
 from ..shellutil import (
     quote as shell_quote,
 )
 from ..util import (
+    FileAvoidWrite,
     mkdir,
     resolve_target_to_make,
 )
 
 
 FINDER_SLOW_MESSAGE = '''
 ===================
 PERFORMANCE WARNING
@@ -1318,16 +1319,34 @@ class BuildDriver(MozbuildObject):
 
     def _run_client_mk(self, target=None, line_handler=None, jobs=0,
                        verbose=None, keep_going=False, append_env=None):
         append_env = dict(append_env or {})
         append_env['TOPSRCDIR'] = self.topsrcdir
 
         append_env['CONFIG_GUESS'] = self.resolve_config_guess()
 
+        mozconfig = self.mozconfig
+        mozconfig_client_mk = os.path.join(self.topobjdir,
+                                           '.mozconfig-client-mk')
+        with FileAvoidWrite(mozconfig_client_mk) as fh:
+            for arg in mozconfig['make_extra'] or []:
+                fh.write(arg)
+                fh.write(b'\n')
+            if mozconfig['make_flags']:
+                fh.write(b'MOZ_MAKE_FLAGS=%s\n' % b' '.join(mozconfig['make_flags']))
+            objdir = mozpath.normsep(self.topobjdir)
+            fh.write(b'MOZ_OBJDIR=%s\n' % objdir)
+            fh.write(b'OBJDIR=%s\n' % objdir)
+            if mozconfig['path']:
+                fh.write(b'FOUND_MOZCONFIG=%s\n' %
+                         mozpath.normsep(mozconfig['path']))
+
+        append_env['OBJDIR'] = mozpath.normsep(self.topobjdir)
+
         return self._run_make(srcdir=True,
                               filename='client.mk',
                               allow_parallel=False,
                               ensure_exit_code=False,
                               print_directory=False,
                               target=target,
                               line_handler=line_handler,
                               log=False,
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1016,17 +1016,17 @@ class Makefiles(MachCommandBase):
                 if f == 'Makefile.in':
                     yield os.path.join(root, f)
 
 @CommandProvider
 class MachDebug(MachCommandBase):
     @Command('environment', category='build-dev',
         description='Show info about the mach and build environment.')
     @CommandArgument('--format', default='pretty',
-        choices=['pretty', 'client.mk', 'configure', 'json'],
+        choices=['pretty', 'configure', 'json'],
         help='Print data in the given format.')
     @CommandArgument('--output', '-o', type=str,
         help='Output to the given file.')
     @CommandArgument('--verbose', '-v', action='store_true',
         help='Print verbose output.')
     def environment(self, format, output=None, verbose=False):
         func = getattr(self, '_environment_%s' % format.replace('.', '_'))
 
@@ -1084,29 +1084,16 @@ class MachDebug(MachCommandBase):
                 print('config substitutions:', file=out)
                 for k in sorted(config.substs):
                     print('\t%s: %s' % (k, config.substs[k]), file=out)
 
                 print('config defines:', file=out)
                 for k in sorted(config.defines):
                     print('\t%s' % k, file=out)
 
-    def _environment_client_mk(self, out, verbose):
-        if self.mozconfig['make_extra']:
-            for arg in self.mozconfig['make_extra']:
-                print(arg, file=out)
-        if self.mozconfig['make_flags']:
-            print('MOZ_MAKE_FLAGS=%s' % ' '.join(self.mozconfig['make_flags']))
-        objdir = mozpath.normsep(self.topobjdir)
-        print('MOZ_OBJDIR=%s' % objdir, file=out)
-        print('OBJDIR=%s' % objdir, file=out)
-        if self.mozconfig['path']:
-            print('FOUND_MOZCONFIG=%s' % mozpath.normsep(self.mozconfig['path']),
-                file=out)
-
     def _environment_json(self, out, verbose):
         import json
         class EnvironmentEncoder(json.JSONEncoder):
             def default(self, obj):
                 if isinstance(obj, MozbuildObject):
                     result = {
                         'topsrcdir': obj.topsrcdir,
                         'topobjdir': obj.topobjdir,