Bug 1417264 - Write .mozconfig.mk file from Python; r=nalexander
authorGregory Szorc <gps@mozilla.com>
Mon, 13 Nov 2017 16:22:42 -0800
changeset 443992 308804a30cd321d2066b83151981db93afea33aa
parent 443991 7e4030a95e89972ecb0d02ed5fa168ff0ea8d300
child 443993 523f39e23b7f90c8e91bdc0c275324ac85e21c0a
push id8527
push userCallek@gmail.com
push dateThu, 11 Jan 2018 21:05:50 +0000
treeherdermozilla-beta@95342d212a7a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1417264
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1417264 - Write .mozconfig.mk file from Python; r=nalexander The file is a filtered version of the make file that we previously started generating for client.mk. Why there is special casing for UPLOAD_EXTRA_FILES, I'm not sure. This smells fishy and is something I'd like to take a look at once all code is ported out of client.mk. The removal of the logic from client.mk meant that we could remove a bunch of code from client.mk related to loading mozconfig files. We can now simply include the auto-generated make file directly and be done with it. MozReview-Commit-ID: 4M5NElQA7iR
client.mk
python/mozbuild/mozbuild/controller/building.py
--- a/client.mk
+++ b/client.mk
@@ -32,37 +32,18 @@ ifeq "$(CWD)" "/"
 CWD   := /.
 endif
 
 PYTHON ?= $(shell which python2.7 > /dev/null 2>&1 && echo python2.7 || echo python)
 
 ####################################
 # Load mozconfig Options
 
-# See build pages, http://www.mozilla.org/build/ for how to set up mozconfig.
-
-define CR
-
-
-endef
-
-# As $(shell) doesn't preserve newlines, use sed to replace them with an
-# unlikely sequence (||), which is then replaced back to newlines by make
-# before evaluation. $(shell) replacing newlines with spaces, || is always
-# followed by a space (since sed doesn't remove newlines), except on the
-# last line, so replace both '|| ' and '||'.
-MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell cat $(OBJDIR)/.mozconfig-client-mk | sed 's/$$/||/')))
 include $(OBJDIR)/.mozconfig-client-mk
 
-# As '||' was used as a newline separator, it means it's not occurring in
-# lines themselves. It can thus safely be used to replaces normal spaces,
-# to then replace newlines with normal spaces. This allows to get a list
-# of mozconfig output lines.
-MOZCONFIG_OUT_LINES := $(subst $(CR), ,$(subst $(NULL) $(NULL),||,$(MOZCONFIG_CONTENT)))
-
 ifdef MOZ_PARALLEL_BUILD
   MOZ_MAKE_FLAGS := $(filter-out -j%,$(MOZ_MAKE_FLAGS))
   MOZ_MAKE_FLAGS += -j$(MOZ_PARALLEL_BUILD)
 endif
 
 # Automatically add -jN to make flags if not defined. N defaults to number of cores.
 ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
   cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())')
@@ -87,26 +68,16 @@ CONFIGURES += $(TOPSRCDIR)/js/src/config
 # The default rule is build
 build::
 
 ifndef MACH
 $(error client.mk must be used via `mach`. Try running \
 `./mach $(firstword $(MAKECMDGOALS) $(.DEFAULT_GOAL))`)
 endif
 
-# For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES.
-MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES))
-$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG)
-	$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@
-
-# Include that makefile so that it is created. This should not actually change
-# the environment since MOZCONFIG_CONTENT, which MOZCONFIG_OUT_LINES derives
-# from, has already been eval'ed.
-include $(OBJDIR)/.mozconfig.mk
-
 # In automation, manage an sccache daemon. The starting of the server
 # needs to be in a make file so sccache inherits the jobserver.
 ifdef MOZBUILD_MANAGE_SCCACHE_DAEMON
 build::
 	# Terminate any sccache server that might still be around.
 	-$(MOZBUILD_MANAGE_SCCACHE_DAEMON) --stop-server > /dev/null 2>&1
 	# Start a new server, ensuring it gets the jobserver file descriptors
 	# from make (but don't use the + prefix when make -n is used, so that
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1348,21 +1348,34 @@ class BuildDriver(MozbuildObject):
         mozconfig_make_lines.append(b'MOZ_OBJDIR=%s' % objdir)
         mozconfig_make_lines.append(b'OBJDIR=%s' % objdir)
 
         if mozconfig['path']:
             mozconfig_make_lines.append(b'FOUND_MOZCONFIG=%s' %
                                         mozpath.normsep(mozconfig['path']))
             mozconfig_make_lines.append(b'export FOUND_MOZCONFIG')
 
+        # The .mozconfig.mk file only contains exported variables and lines with
+        # UPLOAD_EXTRA_FILES.
+        mozconfig_filtered_lines = [
+            line for line in mozconfig_make_lines
+            # Bug 1418122 investigate why UPLOAD_EXTRA_FILES is special and
+            # remove it.
+            if line.startswith(b'export ') or b'UPLOAD_EXTRA_FILES' in line
+        ]
+
         mozconfig_client_mk = os.path.join(self.topobjdir,
                                            '.mozconfig-client-mk')
         with FileAvoidWrite(mozconfig_client_mk) as fh:
             fh.write(b'\n'.join(mozconfig_make_lines))
 
+        mozconfig_mk = os.path.join(self.topobjdir, '.mozconfig.mk')
+        with FileAvoidWrite(mozconfig_mk) as fh:
+            fh.write(b'\n'.join(mozconfig_filtered_lines))
+
         if mozconfig_make_lines:
             self.log(logging.WARNING, 'mozconfig_content', {
                 'path': mozconfig['path'],
                 'content': '\n    '.join(mozconfig_make_lines),
             }, 'Adding make options from {path}\n    {content}')
 
         append_env['OBJDIR'] = mozpath.normsep(self.topobjdir)