author | Gregory Szorc <gps@mozilla.com> |
Fri, 27 Oct 2017 15:20:34 -0700 | |
changeset 389054 | 5fb427c50ca3e2adb541581c68721c845d1130ac |
parent 389053 | 5e5ee518a4b9d43e72c879e9d124aa432fb06e1a |
child 389055 | 540273e5cf22e5b7b8b85f584c6a7efd5e7a90ab |
push id | 32777 |
push user | archaeopteryx@coole-files.de |
push date | Mon, 30 Oct 2017 22:44:45 +0000 |
treeherder | mozilla-central@dd0f265a1300 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | nalexander |
bugs | 1412431 |
milestone | 58.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/build/moz.configure/init.configure +++ b/build/moz.configure/init.configure @@ -83,17 +83,16 @@ add_old_configure_assignment( 'DIST', check_build_environment.dist) option(env='MOZ_AUTOMATION', help='Enable options for automated builds') set_config('MOZ_AUTOMATION', depends_if('MOZ_AUTOMATION')(lambda x: True)) option(env='OLD_CONFIGURE', nargs=1, help='Path to the old configure script') -option(env='MOZ_CURRENT_PROJECT', nargs=1, help='Current build project') option(env='MOZCONFIG', nargs=1, help='Mozconfig location') option('--with-external-source-dir', env='EXTERNAL_SOURCE_DIR', nargs=1, help='External directory containing additional build files') @depends('--with-external-source-dir') def external_source_dir(value): @@ -106,21 +105,21 @@ add_old_configure_assignment('EXTERNAL_S # Read user mozconfig # ============================================================== # Note: the dependency on --help is only there to always read the mozconfig, # even when --help is passed. Without this dependency, the function wouldn't # be called when --help is passed, and the mozconfig wouldn't be read. -@depends('MOZ_CURRENT_PROJECT', 'MOZCONFIG', 'OLD_CONFIGURE', +@depends('MOZCONFIG', 'OLD_CONFIGURE', check_build_environment, '--with-external-source-dir', '--help') @imports(_from='mozbuild.mozconfig', _import='MozconfigLoader') -def mozconfig(current_project, mozconfig, old_configure, build_env, +def mozconfig(mozconfig, old_configure, build_env, external_source_dir, help): if not old_configure: die('The OLD_CONFIGURE environment variable must be set') # Don't read the mozconfig for the js configure (yay backwards # compatibility) # While the long term goal is that js and top-level use the same configure # and the same overall setup, including the possibility to use mozconfigs, @@ -138,20 +137,19 @@ def mozconfig(current_project, mozconfig # old-configure dies. if os.path.dirname(os.path.abspath(old_configure[0])).endswith('/js/src'): return {'path': None} topsrcdir = build_env.topsrcdir if external_source_dir: topsrcdir = external_source_dir[0] loader = MozconfigLoader(topsrcdir) - current_project = current_project[0] if current_project else None mozconfig = mozconfig[0] if mozconfig else None mozconfig = loader.find_mozconfig(env={'MOZCONFIG': mozconfig}) - mozconfig = loader.read_mozconfig(mozconfig, moz_build_app=current_project) + mozconfig = loader.read_mozconfig(mozconfig) return mozconfig set_config('MOZCONFIG', depends(mozconfig)(lambda m: m['path'])) option(env='PYTHON', nargs=1, help='Python interpreter')
--- a/client.mk +++ b/client.mk @@ -17,18 +17,16 @@ # build # clean (realclean is now the same as clean) # distclean # # See http://developer.mozilla.org/en/docs/Build_Documentation for # more information. # # Options: -# MOZ_BUILD_PROJECTS - Build multiple projects in subdirectories -# of MOZ_OBJDIR # MOZ_OBJDIR - Destination object directory # MOZ_MAKE_FLAGS - Flags to pass to $(MAKE) # MOZ_PREFLIGHT_ALL - Makefiles to run before building. # MOZ_POSTFLIGHT_ALL - Makefiles to run after building. # ####################################################################### # Defines @@ -100,18 +98,17 @@ define CR endef # As $(shell) doesn't preserve newlines, use sed to replace them with an # unlikely sequence (||), which is then replaced back to newlines by make # before evaluation. $(shell) replacing newlines with spaces, || is always # followed by a space (since sed doesn't remove newlines), except on the # last line, so replace both '|| ' and '||'. # Also, make MOZ_PGO available to mozconfig when passed on make command line. -# Likewise for MOZ_CURRENT_PROJECT. -MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell $(addprefix MOZ_CURRENT_PROJECT=,$(MOZ_CURRENT_PROJECT)) MOZ_PGO=$(MOZ_PGO) $(TOPSRCDIR)/mach environment --format=client.mk | sed 's/$$/||/'))) +MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell MOZ_PGO=$(MOZ_PGO) $(TOPSRCDIR)/mach environment --format=client.mk | sed 's/$$/||/'))) $(eval $(MOZCONFIG_CONTENT)) export FOUND_MOZCONFIG # As '||' was used as a newline separator, it means it's not occurring in # lines themselves. It can thus safely be used to replaces normal spaces, # to then replace newlines with normal spaces. This allows to get a list # of mozconfig output lines. @@ -139,26 +136,16 @@ ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS) endif ifdef MOZ_AUTOMATION ifeq (4.0,$(firstword $(sort 4.0 $(MAKE_VERSION)))) MOZ_MAKE_FLAGS += --output-sync=line endif endif -ifdef MOZ_BUILD_PROJECTS - -ifdef MOZ_CURRENT_PROJECT - BUILD_PROJECT_ARG = MOZ_BUILD_APP=$(MOZ_CURRENT_PROJECT) - export MOZ_CURRENT_PROJECT -else - MOZ_MAKE = $(error Cannot build in the OBJDIR when MOZ_CURRENT_PROJECT is not set.) -endif -endif # MOZ_BUILD_PROJECTS - MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR) # 'configure' scripts generated by autoconf. CONFIGURES := $(TOPSRCDIR)/configure CONFIGURES += $(TOPSRCDIR)/js/src/configure # Make targets that are going to be passed to the real build system OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check automation/build @@ -172,43 +159,26 @@ build:: # Include baseconfig.mk for its $(MAKE) validation. include $(TOPSRCDIR)/config/baseconfig.mk # Define mkdir include $(TOPSRCDIR)/config/makefiles/makeutils.mk include $(TOPSRCDIR)/config/makefiles/autotargets.mk -# Create a makefile containing the mk_add_options values from mozconfig, -# but only do so when OBJDIR is defined (see further above). -ifdef MOZ_BUILD_PROJECTS -ifdef MOZ_CURRENT_PROJECT -WANT_MOZCONFIG_MK = 1 -else -WANT_MOZCONFIG_MK = -endif -else -WANT_MOZCONFIG_MK = 1 -endif - -ifdef WANT_MOZCONFIG_MK # For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES # from mach environment --format=client.mk output. MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES)) $(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG) $(call mkdir_deps,$(OBJDIR)) $(OBJDIR)/CLOBBER $(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@ -ifdef MOZ_CURRENT_PROJECT - echo export MOZ_CURRENT_PROJECT=$(MOZ_CURRENT_PROJECT) >> $@ -endif # Include that makefile so that it is created. This should not actually change # the environment since MOZCONFIG_CONTENT, which MOZCONFIG_OUT_LINES derives # from, has already been eval'ed. include $(OBJDIR)/.mozconfig.mk -endif # Print out any options loaded from mozconfig. all realbuild clean distclean export libs install realclean:: ifneq (,$(strip $(MOZCONFIG_OUT_FILTERED))) $(info Adding client.mk options from $(FOUND_MOZCONFIG):) $(foreach line,$(MOZCONFIG_OUT_FILTERED),$(info $(NULL) $(NULL) $(NULL) $(NULL) $(subst ||, ,$(line)))) endif @@ -219,19 +189,18 @@ clobber clobber_all: clean # helper target for mobile build_and_deploy: build package install # Do everything from scratch everything: clean build #################################### # Profile-Guided Optimization -# This is up here, outside of the MOZ_CURRENT_PROJECT logic so that this -# is usable in multi-pass builds, where you might not have a runnable -# application until all the build passes and postflight scripts have run. +# This is up here so that this is usable in multi-pass builds, where you +# might not have a runnable application until all the build passes have run. profiledbuild:: $(call BUILDSTATUS,TIERS pgo_profile_generate pgo_package pgo_profile pgo_clobber pgo_profile_use) $(call BUILDSTATUS,TIER_START pgo_profile_generate) $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_GENERATE=1 MOZ_PGO_INSTRUMENTED=1 CREATE_MOZCONFIG_JSON= $(call BUILDSTATUS,TIER_FINISH pgo_profile_generate) $(call BUILDSTATUS,TIER_START pgo_package) $(MAKE) -C $(OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT= rm -f $(OBJDIR)/jarlog/en-US.log @@ -244,50 +213,23 @@ profiledbuild:: $(call BUILDSTATUS,TIER_FINISH pgo_clobber) $(call BUILDSTATUS,TIER_START pgo_profile_use) $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_USE=1 CREATE_MOZCONFIG_JSON= $(call BUILDSTATUS,TIER_FINISH pgo_profile_use) ##################################################### # Preflight, before building any project +ifdef MOZ_PREFLIGHT_ALL realbuild preflight_all:: -ifeq (,$(MOZ_CURRENT_PROJECT)$(if $(MOZ_PREFLIGHT_ALL),,1)) -# Don't run preflight_all for individual projects in multi-project builds -# (when MOZ_CURRENT_PROJECT is set.) -ifndef MOZ_BUILD_PROJECTS -# Building a single project, OBJDIR is usable. set -e; \ for mkfile in $(MOZ_PREFLIGHT_ALL); do \ $(MAKE) -f $(TOPSRCDIR)/$$mkfile preflight_all TOPSRCDIR=$(TOPSRCDIR) OBJDIR=$(OBJDIR) MOZ_OBJDIR=$(MOZ_OBJDIR); \ done -else -# OBJDIR refers to the project-specific OBJDIR, which is not available at -# this point when building multiple projects. Only MOZ_OBJDIR is available. - set -e; \ - for mkfile in $(MOZ_PREFLIGHT_ALL); do \ - $(MAKE) -f $(TOPSRCDIR)/$$mkfile preflight_all TOPSRCDIR=$(TOPSRCDIR) MOZ_OBJDIR=$(MOZ_OBJDIR) MOZ_BUILD_PROJECTS='$(MOZ_BUILD_PROJECTS)'; \ - done endif -endif - -# If we're building multiple projects, but haven't specified which project, -# loop through them. - -ifeq (,$(MOZ_CURRENT_PROJECT)$(if $(MOZ_BUILD_PROJECTS),,1)) -configure realbuild $(OBJDIR_TARGETS):: - set -e; \ - for app in $(MOZ_BUILD_PROJECTS); do \ - $(MAKE) -f $(TOPSRCDIR)/client.mk $@ MOZ_CURRENT_PROJECT=$$app; \ - done - -else - -# MOZ_CURRENT_PROJECT: either doing a single-project build, or building an -# individual project in a multi-project build. #################################### # Configure MAKEFILE = $(wildcard $(OBJDIR)/Makefile) CONFIG_STATUS = $(wildcard $(OBJDIR)/config.status) CONFIG_CACHE = $(wildcard $(OBJDIR)/config.cache) @@ -340,17 +282,16 @@ endif $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR) configure-files: $(CONFIGURES) configure-preqs = \ $(OBJDIR)/CLOBBER \ configure-files \ $(call mkdir_deps,$(OBJDIR)) \ - $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \ save-mozconfig \ $(OBJDIR)/.mozconfig.json \ $(NULL) CREATE_MOZCONFIG_JSON = $(shell $(TOPSRCDIR)/mach environment --format=json -o $(OBJDIR)/.mozconfig.json) # Force CREATE_MOZCONFIG_JSON above to be resolved, without side effects in # case the result is non empty, and allowing an override on the make command # line not running the command (using := $(shell) still runs the shell command). @@ -397,48 +338,30 @@ realbuild:: $(OBJDIR)/Makefile $(OBJDIR #################################### # Other targets # Pass these target onto the real build system $(OBJDIR_TARGETS):: $(OBJDIR)/Makefile $(OBJDIR)/config.status +$(MOZ_MAKE) $@ -endif # MOZ_CURRENT_PROJECT - #################################### # Postflight, after building all projects ifdef MOZ_AUTOMATION -ifndef MOZ_CURRENT_PROJECT $(if $(MOZ_PGO),profiledbuild,realbuild):: -# Only run the automation/build target for the first project. -# (i.e. first platform of universal builds) - $(MAKE) -f $(TOPSRCDIR)/client.mk automation/build $(addprefix MOZ_CURRENT_PROJECT=,$(firstword $(MOZ_BUILD_PROJECTS))) -endif + $(MAKE) -f $(TOPSRCDIR)/client.mk automation/build endif +ifdef MOZ_POSTFLIGHT_ALL realbuild postflight_all:: -ifeq (,$(MOZ_CURRENT_PROJECT)$(if $(MOZ_POSTFLIGHT_ALL),,1)) -# Don't run postflight_all for individual projects in multi-project builds -# (when MOZ_CURRENT_PROJECT is set.) -ifndef MOZ_BUILD_PROJECTS -# Building a single project, OBJDIR is usable. set -e; \ for mkfile in $(MOZ_POSTFLIGHT_ALL); do \ $(MAKE) -f $(TOPSRCDIR)/$$mkfile postflight_all TOPSRCDIR=$(TOPSRCDIR) OBJDIR=$(OBJDIR) MOZ_OBJDIR=$(MOZ_OBJDIR); \ done -else -# OBJDIR refers to the project-specific OBJDIR, which is not available at -# this point when building multiple projects. Only MOZ_OBJDIR is available. - set -e; \ - for mkfile in $(MOZ_POSTFLIGHT_ALL); do \ - $(MAKE) -f $(TOPSRCDIR)/$$mkfile postflight_all TOPSRCDIR=$(TOPSRCDIR) MOZ_OBJDIR=$(MOZ_OBJDIR) MOZ_BUILD_PROJECTS='$(MOZ_BUILD_PROJECTS)'; \ - done -endif endif echo-variable-%: @echo $($*) # This makefile doesn't support parallel execution. It does pass # MOZ_MAKE_FLAGS to sub-make processes, so they will correctly execute # in parallel.
--- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -210,18 +210,17 @@ class MozbuildObject(ProcessExecutionMix @property def mozconfig(self): """Returns information about the current mozconfig file. This a dict as returned by MozconfigLoader.read_mozconfig() """ if not isinstance(self._mozconfig, dict): loader = MozconfigLoader(self.topsrcdir) - self._mozconfig = loader.read_mozconfig(path=self._mozconfig, - moz_build_app=os.environ.get('MOZ_CURRENT_PROJECT')) + self._mozconfig = loader.read_mozconfig(path=self._mozconfig) return self._mozconfig @property def config_environment(self): """Returns the ConfigEnvironment for the current build configuration. This property is only available once configure has executed.
--- a/python/mozbuild/mozbuild/mozconfig.py +++ b/python/mozbuild/mozbuild/mozconfig.py @@ -184,17 +184,17 @@ class MozconfigLoader(object): for path in deprecated_paths: if os.path.exists(path): raise MozconfigFindException( MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir)) return None - def read_mozconfig(self, path=None, moz_build_app=None): + def read_mozconfig(self, path=None): """Read the contents of a mozconfig into a data structure. This takes the path to a mozconfig to load. If the given path is AUTODETECT, will try to find a mozconfig from the environment using find_mozconfig(). mozconfig files are shell scripts. So, we can't just parse them. Instead, we run the shell script in a wrapper which allows us to record @@ -309,20 +309,16 @@ class MozconfigLoader(object): filt = lambda x, y: {k: v for k, v in x.items() if k not in y} result['vars'] = diff_vars( filt(parsed['vars_before'], parsed['env_before']), filt(parsed['vars_after'], parsed['env_after']) ) result['configure_args'] = [self._expand(o) for o in parsed['ac']] - if moz_build_app is not None: - result['configure_args'].extend(self._expand(o) for o in - parsed['ac_app'][moz_build_app]) - if 'MOZ_OBJDIR' in parsed['env_before']: result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR'] mk = [self._expand(o) for o in parsed['mk']] for o in mk: match = self.RE_MAKE_VARIABLE.match(o) @@ -342,17 +338,16 @@ class MozconfigLoader(object): result['make_extra'].append(o) return result def _parse_loader_output(self, output): mk_options = [] ac_options = [] - ac_app_options = defaultdict(list) before_source = {} after_source = {} env_before_source = {} env_after_source = {} current = None current_type = None in_variable = None @@ -380,19 +375,16 @@ class MozconfigLoader(object): assert not in_variable section = line[len('------END_'):] assert current_type == section if current_type == 'AC_OPTION': ac_options.append('\n'.join(current)) elif current_type == 'MK_OPTION': mk_options.append('\n'.join(current)) - elif current_type == 'AC_APP_OPTION': - app = current.pop(0) - ac_app_options[app].append('\n'.join(current)) current = None current_type = None continue assert current_type is not None vars_mapping = { @@ -469,17 +461,16 @@ class MozconfigLoader(object): continue current.append(line) return { 'mk': mk_options, 'ac': ac_options, - 'ac_app': ac_app_options, 'vars_before': before_source, 'vars_after': after_source, 'env_before': env_before_source, 'env_after': env_after_source, } def _expand(self, s): return s.replace('@TOPSRCDIR@', self.topsrcdir)
--- a/python/mozbuild/mozbuild/mozconfig_loader +++ b/python/mozbuild/mozbuild/mozconfig_loader @@ -19,25 +19,16 @@ ac_add_options() { ;; esac echo "------BEGIN_AC_OPTION" echo $_mozconfig_opt echo "------END_AC_OPTION" done } -ac_add_app_options() { - _mozconfig_app=$1 - shift - echo "------BEGIN_AC_APP_OPTION" - echo $_mozconfig_app - echo "$*" - echo "------END_AC_APP_OPTION" -} - mk_add_options() { for _mozconfig_opt; do echo "------BEGIN_MK_OPTION" echo $_mozconfig_opt echo "------END_MK_OPTION" done }
--- a/python/mozbuild/mozbuild/test/test_mozconfig.py +++ b/python/mozbuild/mozbuild/test/test_mozconfig.py @@ -282,34 +282,16 @@ class TestMozconfigLoader(unittest.TestC mozconfig.write('ac_add_options --foo=@TOPSRCDIR@\n') mozconfig.flush() loader = self.get_loader() result = loader.read_mozconfig(mozconfig.name) self.assertEqual(result['configure_args'], [ '--foo=%s' % loader.topsrcdir]) - def test_read_ac_app_options(self): - with NamedTemporaryFile(mode='w') as mozconfig: - mozconfig.write('ac_add_options --foo=@TOPSRCDIR@\n') - mozconfig.write('ac_add_app_options app1 --bar=@TOPSRCDIR@\n') - mozconfig.write('ac_add_app_options app2 --bar=x\n') - mozconfig.flush() - - loader = self.get_loader() - result = loader.read_mozconfig(mozconfig.name, moz_build_app='app1') - self.assertEqual(result['configure_args'], [ - '--foo=%s' % loader.topsrcdir, - '--bar=%s' % loader.topsrcdir]) - - result = loader.read_mozconfig(mozconfig.name, moz_build_app='app2') - self.assertEqual(result['configure_args'], [ - '--foo=%s' % loader.topsrcdir, - '--bar=x']) - def test_read_capture_mk_options(self): """Ensures mk_add_options calls are captured.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('mk_add_options MOZ_OBJDIR=/foo/bar\n') mozconfig.write('mk_add_options MOZ_MAKE_FLAGS="-j8 -s"\n') mozconfig.write('mk_add_options FOO="BAR BAZ"\n') mozconfig.write('mk_add_options BIZ=1\n') mozconfig.flush()