author | Mike Hommey <mh+mozilla@glandium.org> |
Thu, 03 Jul 2014 07:15:31 +0900 | |
changeset 191955 | ce1c57e03b885eedfbe58a191e5306b17b04fc65 |
parent 191954 | 588203633ba714cc4ee0e1f03cf9844faee41e09 |
child 191956 | 73a651b7e30a7bf352adff29f48547e193103796 |
push id | 45713 |
push user | mh@glandium.org |
push date | Wed, 02 Jul 2014 22:16:23 +0000 |
treeherder | mozilla-inbound@ce1c57e03b88 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | gps |
bugs | 762358 |
milestone | 33.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/build/autoconf/altoptions.m4 +++ b/build/autoconf/altoptions.m4 @@ -77,12 +77,47 @@ AC_DEFUN([MOZ_ARG_WITH_STRING], dnl MOZ_ARG_HEADER(Comment) dnl This is used by webconfig to group options define(MOZ_ARG_HEADER, [# $1]) dnl MOZ_READ_MYCONFIG() - Read in 'myconfig.sh' file AC_DEFUN([MOZ_READ_MOZCONFIG], [AC_REQUIRE([AC_INIT_BINSH])dnl -# Read in '.mozconfig' script to set the initial options. -# See the mozconfig2configure script for more details. -_AUTOCONF_TOOLS_DIR=`dirname [$]0`/[$1]/build/autoconf -. $_AUTOCONF_TOOLS_DIR/mozconfig2configure]) +inserted= +dnl Shell is hard, so here is what the following does: +dnl - Reset $@ (command line arguments) +dnl - Add the configure options from mozconfig to $@ one by one +dnl - Add the original command line arguments after that, one by one +dnl +dnl There are several tricks involved: +dnl - It is not possible to preserve the whitespaces in $@ by assigning to +dnl another variable, so the two first steps above need to happen in the first +dnl iteration of the third step. +dnl - We always want the configure options to be added, so the loop must be +dnl iterated at least once, so we add a dummy argument first, and discard it. +dnl - something | while read line ... makes the while run in a subshell, meaning +dnl that anything it does is not propagated to the main shell, so we can't do +dnl set -- foo there. As a consequence, what the while loop reading mach +dnl environment output does is output a set of shell commands for the main shell +dnl to eval. +dnl - Extra care is due when lines from mach environment output contain special +dnl shell characters, so we use ' for quoting and ensure no ' end up in between +dnl the quoting mark unescaped. +dnl Some of the above is directly done in mach environment --format=configure. +failed_eval() { + echo "Failed eval'ing the following:" + $(dirname [$]0)/[$1]/mach environment --format=configure + exit 1 +} + +set -- dummy "[$]@" +for ac_option +do + if test -z "$inserted"; then + set -- + eval "$($(dirname [$]0)/[$1]/mach environment --format=configure)" || failed_eval + inserted=1 + else + set -- "[$]@" "$ac_option" + fi +done +])
deleted file mode 100755 --- a/build/autoconf/mozconfig-find +++ /dev/null @@ -1,76 +0,0 @@ -#! /bin/sh -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# mozconfigfind - Loads options from .mozconfig onto configure's -# command-line. The .mozconfig file is searched for in the -# order: -# If $MOZCONFIG is set, use that. -# If one of $TOPSRCDIR/.mozconfig or $TOPSRCDIR/mozconfig exists, use it. -# If both exist, or if various legacy locations contain a mozconfig, error. -# Otherwise, use the default build options. -# -topsrcdir=$1 - -abspath() { - if uname -s | grep -q MINGW; then - # We have no way to figure out whether we're in gmake or pymake right - # now. gmake gives us Unix-style paths while pymake gives us Windows-style - # paths, so attempt to handle both. - regexes='^\([A-Za-z]:\|\\\\\|\/\) ^\/' - else - regexes='^\/' - fi - - for regex in $regexes; do - if echo $1 | grep -q $regex; then - echo $1 - return - fi - done - - # If we're at this point, we have a relative path - echo `pwd`/$1 -} - -if [ -n "$MOZCONFIG" ] && ! [ -f "$MOZCONFIG" ]; then - echo "Specified MOZCONFIG \"$MOZCONFIG\" does not exist!" 1>&2 - exit 1 -fi - -if [ -n "$MOZ_MYCONFIG" ]; then - echo "Your environment currently has the MOZ_MYCONFIG variable set to \"$MOZ_MYCONFIG\". MOZ_MYCONFIG is no longer supported. Please use MOZCONFIG instead." 1>&2 - exit 1 -fi - -if [ -z "$MOZCONFIG" ] && [ -f "$topsrcdir/.mozconfig" ] && [ -f "$topsrcdir/mozconfig" ]; then - echo "Both \$topsrcdir/.mozconfig and \$topsrcdir/mozconfig are supported, but you must choose only one. Please remove the other." 1>&2 - exit 1 -fi - -for _config in "$MOZCONFIG" \ - "$topsrcdir/.mozconfig" \ - "$topsrcdir/mozconfig" -do - if test -f "$_config"; then - abspath $_config - exit 0 - fi -done - -# We used to support a number of other implicit .mozconfig locations. We now -# detect if we were about to use any of these locations and issue an error if we -# find any. -for _config in "$topsrcdir/mozconfig.sh" \ - "$topsrcdir/myconfig.sh" \ - "$HOME/.mozconfig" \ - "$HOME/.mozconfig.sh" \ - "$HOME/.mozmyconfig.sh" -do - if test -f "$_config"; then - echo "You currently have a mozconfig at \"$_config\". This implicit location is no longer supported. Please move it to $topsrcdir/.mozconfig or specify it explicitly via \$MOZCONFIG." 1>&2 - exit 1 - fi -done
deleted file mode 100755 --- a/build/autoconf/mozconfig2client-mk +++ /dev/null @@ -1,76 +0,0 @@ -#! /bin/sh -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# mozconfig2client-mk - Translates .mozconfig into options for client.mk. -# Prints defines to stdout. -# -# See mozconfig2configure for more details - -print_header() { - cat <<EOF -# gmake -# This file is automatically generated for client.mk. -# Do not edit. Edit $FOUND_MOZCONFIG instead. - -EOF -} - -ac_add_options() { - for _opt - do - case "$_opt" in - --target=*) - echo $_opt | sed s/--target/CONFIG_GUESS/ - ;; - *) - echo "# $_opt is used by configure (not client.mk)" - ;; - esac - done -} - -ac_add_app_options() { - echo "# $* is used by configure (not client.mk)" -} - -mk_add_options() { - for _opt - do - # Escape shell characters, space, tab, dollar, quote, backslash, - # and substitute '@<word>@' with '$(<word>)'. - _opt=`echo "$_opt" | sed -e 's/\([\"\\]\)/\\\\\1/g; s/@\([^@]*\)@/\$(\1)/g;'` - echo $_opt; - done -} - -# Main -#-------------------------------------------------- - -scriptdir=`dirname $0` -topsrcdir=$1 - -# If the path changes, configure should be rerun -echo "# PATH=$PATH" - -# If FOUND_MOZCONFIG isn't set, look for it and make sure the script doesn't error out -isfoundset=${FOUND_MOZCONFIG+yes} -if [ -z $isfoundset ]; then - FOUND_MOZCONFIG=`$scriptdir/mozconfig-find $topsrcdir` - if [ $? -ne 0 ]; then - echo '$(error Fix above errors before continuing.)' - else - isfoundset=yes - fi -fi - -if [ -n $isfoundset ]; then - if [ "$FOUND_MOZCONFIG" ] - then - print_header - . "$FOUND_MOZCONFIG" - echo "FOUND_MOZCONFIG := $FOUND_MOZCONFIG" - fi -fi
deleted file mode 100755 --- a/build/autoconf/mozconfig2configure +++ /dev/null @@ -1,103 +0,0 @@ -#! /bin/sh -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# mozconfig2configure - Loads options from .mozconfig onto configure's -# command-line. See mozconfig-find for how the config file is -# found -# -# The options from .mozconfig are inserted into the command-line -# before the real command-line options. This way the real options -# will override any .mozconfig options. -# -# .mozconfig is a shell script. To add an option to configure's -# command-line use the pre-defined function, ac_add_options, -# -# ac_add_options <configure-option> [<configure-option> ... ] -# -# For example, -# -# ac_add_options --with-pthreads --enable-debug -# -# ac_add_options can be called multiple times in .mozconfig. -# Each call adds more options to configure's command-line. - -# Note: $_AUTOCONF_TOOLS_DIR must be defined in the script that includes this. - -ac_add_options() { - for _opt - do - # Escape shell characters, space, tab, dollar, quote, backslash, parentheses. - _opt=`echo $_opt | sed -e 's/\([\ \ \$\"\\\(\)]\)/\\\\\1/g;s/@\([^@]*\)@/\$\1/g;'` - _opt=`echo $_opt | sed -e 's/@\([^@]*\)@/\$(\1)/g'` - - # Avoid adding duplicates - case "$ac_options" in - # Note that all options in $ac_options are enclosed in quotes, - # so there will always be a last character to match [^-A-Za-z0-9_] - *"\"$_opt[^-A-Za-z0-9_]"* ) ;; - * ) mozconfig_ac_options="$mozconfig_ac_options $_opt" ;; - esac - done -} - -ac_add_app_options() { - APP=$1 - shift; - if [ "$APP" = "$MOZ_BUILD_APP" ]; then - ac_add_options "$*"; - fi -} - -mk_add_options() { - # These options are for client.mk - # configure can safely ignore them. - : -} - -ac_echo_options() { - echo "Adding configure options from $FOUND_MOZCONFIG:" - eval "set -- $mozconfig_ac_options" - for _opt - do - echo " $_opt" - done -} - -# Main -#-------------------------------------------------- -topsrcdir=$(cd `dirname $0`; pwd -W 2>/dev/null || pwd) -ac_options= -mozconfig_ac_options= - -# Save the real command-line options -for _opt -do - # Escape shell characters, space, tab, dollar, quote, backslash. - _opt=`echo $_opt | sed -e 's/\([\ \ \$\"\\]\)/\\\\\1/g;'` - ac_options="$ac_options \"$_opt\"" -done - - -# If FOUND_MOZCONFIG isn't set, look for it and make sure the script doesn't error out -isfoundset=${FOUND_MOZCONFIG+yes} -if [ -z $isfoundset ]; then - FOUND_MOZCONFIG=`$_AUTOCONF_TOOLS_DIR/mozconfig-find $topsrcdir` - if [ $? -ne 0 ]; then - echo "Fix above errors before continuing." 1>&2 - exit 1 - fi -fi - -if [ "$FOUND_MOZCONFIG" ]; then - . "$FOUND_MOZCONFIG" -fi -export FOUND_MOZCONFIG - -if [ "$mozconfig_ac_options" ]; then - ac_echo_options 1>&2 -fi - -eval "set -- $mozconfig_ac_options $ac_options"
--- a/client.mk +++ b/client.mk @@ -45,17 +45,16 @@ endif ifeq "$(CWD)" "/" CWD := /. endif ifndef TOPSRCDIR ifeq (,$(wildcard client.mk)) TOPSRCDIR := $(patsubst %/,%,$(dir $(MAKEFILE_LIST))) -MOZ_OBJDIR = . else TOPSRCDIR := $(CWD) endif endif # try to find autoconf 2.13 - discard errors from 'which' # MacOS X 10.4 sends "no autoconf*" errors to stdout, discard those via grep AUTOCONF ?= $(shell which autoconf-2.13 autoconf2.13 autoconf213 2>/dev/null | grep -v '^no autoconf' | head -1) @@ -94,30 +93,29 @@ for a workaround of this issue.) endif endif #################################### # Load mozconfig Options # See build pages, http://www.mozilla.org/build/ for how to set up mozconfig. -MOZCONFIG_LOADER := build/autoconf/mozconfig2client-mk - define CR endef # As $(shell) doesn't preserve newlines, use sed to replace them with an # unlikely sequence (||), which is then replaced back to newlines by make # before evaluation. $(shell) replacing newlines with spaces, || is always # followed by a space (since sed doesn't remove newlines), except on the # last line, so replace both '|| ' and '||'. # Also, make MOZ_PGO available to mozconfig when passed on make command line. -MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell MOZ_PGO=$(MOZ_PGO) $(TOPSRCDIR)/$(MOZCONFIG_LOADER) $(TOPSRCDIR) | sed 's/$$/||/'))) +# Likewise for MOZ_CURRENT_PROJECT. +MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell $(addprefix MOZ_CURRENT_PROJECT=,$(MOZ_CURRENT_PROJECT)) MOZ_PGO=$(MOZ_PGO) $(TOPSRCDIR)/mach environment --format=client.mk | sed 's/$$/||/'))) $(eval $(MOZCONFIG_CONTENT)) export FOUND_MOZCONFIG # As '||' was used as a newline separator, it means it's not occurring in # lines themselves. It can thus safely be used to replaces normal spaces, # to then replace newlines with normal spaces. This allows to get a list # of mozconfig output lines. @@ -138,38 +136,28 @@ endif # Automatically add -jN to make flags if not defined. N defaults to number of cores. ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS))) cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())') MOZ_MAKE_FLAGS += -j$(cores) endif -ifndef MOZ_OBJDIR - MOZ_OBJDIR = obj-$(CONFIG_GUESS) -endif - ifdef MOZ_BUILD_PROJECTS ifdef MOZ_CURRENT_PROJECT - OBJDIR = $(MOZ_OBJDIR)/$(MOZ_CURRENT_PROJECT) - MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR) BUILD_PROJECT_ARG = MOZ_BUILD_APP=$(MOZ_CURRENT_PROJECT) + export MOZ_CURRENT_PROJECT else - OBJDIR = $(error Cannot find the OBJDIR when MOZ_CURRENT_PROJECT is not set.) MOZ_MAKE = $(error Cannot build in the OBJDIR when MOZ_CURRENT_PROJECT is not set.) endif - -else # MOZ_BUILD_PROJECTS +endif # MOZ_BUILD_PROJECTS -OBJDIR = $(MOZ_OBJDIR) MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR) -endif # MOZ_BUILD_PROJECTS - # 'configure' scripts generated by autoconf. CONFIGURES := $(TOPSRCDIR)/configure CONFIGURES += $(TOPSRCDIR)/js/src/configure # Make targets that are going to be passed to the real build system OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check automation/build ####################################################################### @@ -191,17 +179,17 @@ WANT_MOZCONFIG_MK = 1 else WANT_MOZCONFIG_MK = endif else WANT_MOZCONFIG_MK = 1 endif ifdef WANT_MOZCONFIG_MK -# For now, only output "export" lines from mozconfig2client-mk output. +# For now, only output "export" lines from mach environment --format=client.mk output. MOZCONFIG_MK_LINES := $(filter export||%,$(MOZCONFIG_OUT_LINES)) $(OBJDIR)/.mozconfig.mk: $(FOUND_MOZCONFIG) $(call mkdir_deps,$(OBJDIR)) $(OBJDIR)/CLOBBER $(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@ # Include that makefile so that it is created. This should not actually change # the environment since MOZCONFIG_CONTENT, which MOZCONFIG_OUT_LINES derives # from, has already been eval'ed. include $(OBJDIR)/.mozconfig.mk @@ -224,27 +212,21 @@ build_and_deploy: build package install # Do everything from scratch everything: clean build #################################### # Profile-Guided Optimization # This is up here, outside of the MOZ_CURRENT_PROJECT logic so that this # is usable in multi-pass builds, where you might not have a runnable # application until all the build passes and postflight scripts have run. -ifdef MOZ_OBJDIR - PGO_OBJDIR = $(MOZ_OBJDIR) -else - PGO_OBJDIR := $(TOPSRCDIR) -endif - profiledbuild:: $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_GENERATE=1 MOZ_PGO_INSTRUMENTED=1 - $(MAKE) -C $(PGO_OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT= - rm -f ${PGO_OBJDIR}/jarlog/en-US.log - MOZ_PGO_INSTRUMENTED=1 JARLOG_FILE=jarlog/en-US.log EXTRA_TEST_ARGS=10 $(MAKE) -C $(PGO_OBJDIR) pgo-profile-run + $(MAKE) -C $(OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT= + rm -f $(OBJDIR)/jarlog/en-US.log + MOZ_PGO_INSTRUMENTED=1 JARLOG_FILE=jarlog/en-US.log EXTRA_TEST_ARGS=10 $(MAKE) -C $(OBJDIR) pgo-profile-run $(MAKE) -f $(TOPSRCDIR)/client.mk maybe_clobber_profiledbuild $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_USE=1 ##################################################### # Build date unification ifdef MOZ_UNIFY_BDATE ifndef MOZ_BUILD_DATE @@ -315,16 +297,17 @@ CONFIG_STATUS_DEPS := \ $(CONFIGURES) \ $(TOPSRCDIR)/CLOBBER \ $(TOPSRCDIR)/nsprpub/configure \ $(TOPSRCDIR)/config/milestone.txt \ $(TOPSRCDIR)/browser/config/version.txt \ $(TOPSRCDIR)/build/virtualenv_packages.txt \ $(TOPSRCDIR)/python/mozbuild/mozbuild/virtualenv.py \ $(TOPSRCDIR)/testing/mozbase/packages.txt \ + $(OBJDIR)/.mozconfig.json \ $(NULL) CONFIGURE_ENV_ARGS += \ MAKE='$(MAKE)' \ $(NULL) # configure uses the program name to determine @srcdir@. Calling it without # $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full @@ -342,18 +325,23 @@ endif configure-files: $(CONFIGURES) configure-preqs = \ $(OBJDIR)/CLOBBER \ configure-files \ $(call mkdir_deps,$(OBJDIR)) \ $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \ save-mozconfig \ + $(OBJDIR)/.mozconfig.json \ $(NULL) +CREATE_MOZCONFIG_JSON := $(shell $(TOPSRCDIR)/mach environment --format=json -o $(OBJDIR)/.mozconfig.json) +$(OBJDIR)/.mozconfig.json: $(call mkdir_deps,$(OBJDIR)) + @$(TOPSRCDIR)/mach environment --format=json -o $@ + save-mozconfig: $(FOUND_MOZCONFIG) -cp $(FOUND_MOZCONFIG) $(OBJDIR)/.mozconfig configure:: $(configure-preqs) @echo cd $(OBJDIR); @echo $(CONFIGURE) $(CONFIGURE_ARGS) @cd $(OBJDIR) && $(BUILD_PROJECT_ARG) $(CONFIGURE_ENV_ARGS) $(CONFIGURE) $(CONFIGURE_ARGS) \ || ( echo '*** Fix above errors and then restart with\ @@ -362,17 +350,17 @@ configure:: $(configure-preqs) ifneq (,$(MAKEFILE)) $(OBJDIR)/Makefile: $(OBJDIR)/config.status $(OBJDIR)/config.status: $(CONFIG_STATUS_DEPS) else $(OBJDIR)/Makefile: $(CONFIG_STATUS_DEPS) endif - @$(MAKE) -f $(TOPSRCDIR)/client.mk configure + @$(MAKE) -f $(TOPSRCDIR)/client.mk configure CREATE_MOZCONFIG_JSON= ifneq (,$(CONFIG_STATUS)) $(OBJDIR)/config/autoconf.mk: $(TOPSRCDIR)/config/autoconf.mk.in $(PYTHON) $(OBJDIR)/config.status -n --file=$(OBJDIR)/config/autoconf.mk endif ####################################
--- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -156,33 +156,51 @@ class MozbuildObject(ProcessExecutionMix raise BuildEnvironmentNotFoundException( 'Could not find Mozilla source tree or build environment.') # Now we try to load the config for this environment. If mozconfig is # None, read_mozconfig() will attempt to find one in the existing # environment. If no mozconfig is present, the config will not have # much defined. loader = MozconfigLoader(topsrcdir) - config = loader.read_mozconfig(mozconfig) + current_project = os.environ.get('MOZ_CURRENT_PROJECT') + config = loader.read_mozconfig(mozconfig, moz_build_app=current_project) config_topobjdir = MozbuildObject.resolve_mozconfig_topobjdir( topsrcdir, config) # If we're inside a objdir and the found mozconfig resolves to # another objdir, we abort. The reasoning here is that if you are # inside an objdir you probably want to perform actions on that objdir, # not another one. This prevents accidental usage of the wrong objdir # when the current objdir is ambiguous. + # However, if the found mozconfig resolves to another objdir that + # doesn't exist, we may be in a subtree like when building mozilla/ + # under c-c, and the objdir was defined as a relative path. Try again + # adjusting for that. + if topobjdir and config_topobjdir: - mozilla_dir = os.path.join(config_topobjdir, 'mozilla') - if not samepath(topobjdir, config_topobjdir) \ + if not os.path.exists(config_topobjdir): + config_topobjdir = MozbuildObject.resolve_mozconfig_topobjdir( + os.path.dirname(topsrcdir), config) + if current_project: + config_topobjdir = os.path.join(config_topobjdir, + current_project) + config_topobjdir = os.path.join(config_topobjdir, + os.path.basename(topsrcdir)) + elif current_project: + config_topobjdir = os.path.join(config_topobjdir, current_project) + + _config_topobjdir = config_topobjdir + mozilla_dir = os.path.join(_config_topobjdir, 'mozilla') + if not samepath(topobjdir, _config_topobjdir) \ and (not os.path.exists(mozilla_dir) or not samepath(topobjdir, mozilla_dir)): - raise ObjdirMismatchException(topobjdir, config_topobjdir) + raise ObjdirMismatchException(topobjdir, _config_topobjdir) topobjdir = topobjdir or config_topobjdir if topobjdir: topobjdir = os.path.normpath(topobjdir) if topsrcdir == topobjdir: raise BadEnvironmentException('The object directory appears ' 'to be the same as your source directory (%s). This build ' @@ -228,17 +246,18 @@ class MozbuildObject(ProcessExecutionMix @property def mozconfig(self): """Returns information about the current mozconfig file. This a dict as returned by MozconfigLoader.read_mozconfig() """ if self._mozconfig is None: loader = MozconfigLoader(self.topsrcdir) - self._mozconfig = loader.read_mozconfig() + self._mozconfig = loader.read_mozconfig( + moz_build_app=os.environ.get('MOZ_CURRENT_PROJECT')) return self._mozconfig @property def config_environment(self): """Returns the ConfigEnvironment for the current build configuration. This property is only available once configure has executed. @@ -544,18 +563,23 @@ class MachCommandBase(MozbuildObject): without having to change everything that inherits from it. """ def __init__(self, context): # Attempt to discover topobjdir through environment detection, as it is # more reliable than mozconfig when cwd is inside an objdir. topsrcdir = context.topdir topobjdir = None + detect_virtualenv_mozinfo = True + if hasattr(context, 'detect_virtualenv_mozinfo'): + detect_virtualenv_mozinfo = getattr(context, + 'detect_virtualenv_mozinfo') try: - dummy = MozbuildObject.from_environment(cwd=context.cwd) + dummy = MozbuildObject.from_environment(cwd=context.cwd, + detect_virtualenv_mozinfo=detect_virtualenv_mozinfo) topsrcdir = dummy.topsrcdir topobjdir = dummy._topobjdir except BuildEnvironmentNotFoundException: pass except ObjdirMismatchException as e: print('Ambiguous object directory detected. We detected that ' 'both %s and %s could be object directories. This is ' 'typically caused by having a mozconfig pointing to a '
--- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -5,16 +5,18 @@ from __future__ import print_function, unicode_literals import itertools import logging import operator import os import sys +import mozpack.path as mozpath + from mach.decorators import ( CommandArgument, CommandProvider, Command, ) from mach.mixin.logging import LoggingMixin @@ -914,60 +916,132 @@ class Makefiles(MachCommandBase): for f in files: if f == 'Makefile.in': yield os.path.join(root, f) @CommandProvider class MachDebug(MachCommandBase): @Command('environment', category='build-dev', description='Show info about the mach and build environment.') + @CommandArgument('--format', default='pretty', + choices=['pretty', 'client.mk', 'configure', 'json'], + help='Print data in the given format.') + @CommandArgument('--output', '-o', type=str, + help='Output to the given file.') @CommandArgument('--verbose', '-v', action='store_true', help='Print verbose output.') - def environment(self, verbose=False): + def environment(self, format, output=None, verbose=False): + func = getattr(self, '_environment_%s' % format.replace('.', '_')) + + if output: + # We want to preserve mtimes if the output file already exists + # and the content hasn't changed. + from mozbuild.util import FileAvoidWrite + with FileAvoidWrite(output) as out: + return func(out, verbose) + return func(sys.stdout, verbose) + + def _environment_pretty(self, out, verbose): state_dir = self._mach_context.state_dir import platform - print('platform:\n\t%s' % platform.platform()) - print('python version:\n\t%s' % sys.version) - print('python prefix:\n\t%s' % sys.prefix) - print('mach cwd:\n\t%s' % self._mach_context.cwd) - print('os cwd:\n\t%s' % os.getcwd()) - print('mach directory:\n\t%s' % self._mach_context.topdir) - print('state directory:\n\t%s' % state_dir) + print('platform:\n\t%s' % platform.platform(), file=out) + print('python version:\n\t%s' % sys.version, file=out) + print('python prefix:\n\t%s' % sys.prefix, file=out) + print('mach cwd:\n\t%s' % self._mach_context.cwd, file=out) + print('os cwd:\n\t%s' % os.getcwd(), file=out) + print('mach directory:\n\t%s' % self._mach_context.topdir, file=out) + print('state directory:\n\t%s' % state_dir, file=out) - print('object directory:\n\t%s' % self.topobjdir) + print('object directory:\n\t%s' % self.topobjdir, file=out) if self.mozconfig['path']: - print('mozconfig path:\n\t%s' % self.mozconfig['path']) + print('mozconfig path:\n\t%s' % self.mozconfig['path'], file=out) if self.mozconfig['configure_args']: - print('mozconfig configure args:') + print('mozconfig configure args:', file=out) for arg in self.mozconfig['configure_args']: - print('\t%s' % arg) + print('\t%s' % arg, file=out) if self.mozconfig['make_extra']: - print('mozconfig extra make args:') + print('mozconfig extra make args:', file=out) for arg in self.mozconfig['make_extra']: - print('\t%s' % arg) + print('\t%s' % arg, file=out) if self.mozconfig['make_flags']: - print('mozconfig make flags:') + print('mozconfig make flags:', file=out) for arg in self.mozconfig['make_flags']: - print('\t%s' % arg) + print('\t%s' % arg, file=out) config = None try: config = self.config_environment except Exception: pass if config: - print('config topsrcdir:\n\t%s' % config.topsrcdir) - print('config topobjdir:\n\t%s' % config.topobjdir) + print('config topsrcdir:\n\t%s' % config.topsrcdir, file=out) + print('config topobjdir:\n\t%s' % config.topobjdir, file=out) if verbose: - print('config substitutions:') + print('config substitutions:', file=out) for k in sorted(config.substs): - print('\t%s: %s' % (k, config.substs[k])) + print('\t%s: %s' % (k, config.substs[k]), file=out) + + print('config defines:', file=out) + for k in sorted(config.defines): + print('\t%s' % k, file=out) + + def _environment_client_mk(self, out, verbose): + if self.mozconfig['make_extra']: + for arg in self.mozconfig['make_extra']: + print(arg, file=out) + objdir = mozpath.normsep(self.topobjdir) + print('MOZ_OBJDIR=%s' % objdir, file=out) + if 'MOZ_CURRENT_PROJECT' in os.environ: + objdir = mozpath.join(objdir, os.environ['MOZ_CURRENT_PROJECT']) + print('OBJDIR=%s' % objdir, file=out) + if self.mozconfig['path']: + print('FOUND_MOZCONFIG=%s' % mozpath.normsep(self.mozconfig['path']), + file=out) - print('config defines:') - for k in sorted(config.defines): - print('\t%s' % k) + def _environment_configure(self, out, verbose): + if self.mozconfig['path']: + # Replace ' with '"'"', so that shell quoting e.g. + # a'b becomes 'a'"'"'b'. + quote = lambda s: s.replace("'", """'"'"'""") + print('echo Adding configure options from %s' % + mozpath.normsep(self.mozconfig['path']), file=out) + if self.mozconfig['configure_args']: + for arg in self.mozconfig['configure_args']: + quoted_arg = quote(arg) + print("echo ' %s'" % quoted_arg, file=out) + print("""set -- "$@" '%s'""" % quoted_arg, file=out) + for key, value in self.mozconfig['env']['added'].items(): + print("export %s='%s'" % (key, quote(value)), file=out) + for key, (old, value) in self.mozconfig['env']['modified'].items(): + print("export %s='%s'" % (key, quote(value)), file=out) + for key, value in self.mozconfig['vars']['added'].items(): + print("%s='%s'" % (key, quote(value)), file=out) + for key, (old, value) in self.mozconfig['vars']['modified'].items(): + print("%s='%s'" % (key, quote(value)), file=out) + for key in self.mozconfig['env']['removed'].keys() + \ + self.mozconfig['vars']['removed'].keys(): + print("unset %s" % key, file=out) + + def _environment_json(self, out, verbose): + import json + class EnvironmentEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, MozbuildObject): + result = { + 'topsrcdir': obj.topsrcdir, + 'topobjdir': obj.topobjdir, + 'mozconfig': obj.mozconfig, + } + if verbose: + result['substs'] = obj.substs + result['defines'] = obj.defines + return result + elif isinstance(obj, set): + return list(obj) + return json.JSONEncoder.default(self, obj) + json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out)
--- a/python/mozbuild/mozbuild/mozconfig.py +++ b/python/mozbuild/mozbuild/mozconfig.py @@ -60,17 +60,21 @@ class MozconfigLoader(ProcessExecutionMi re.VERBOSE) # Default mozconfig files in the topsrcdir. DEFAULT_TOPSRCDIR_PATHS = ('.mozconfig', 'mozconfig') DEPRECATED_TOPSRCDIR_PATHS = ('mozconfig.sh', 'myconfig.sh') DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh') - IGNORE_SHELL_VARIABLES = ('_') + IGNORE_SHELL_VARIABLES = {'_'} + + ENVIRONMENT_VARIABLES = { + 'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'MOZ_OBJDIR', + } def __init__(self, topsrcdir): self.topsrcdir = topsrcdir @property def _loader_script(self): our_dir = os.path.abspath(os.path.dirname(__file__)) @@ -191,16 +195,17 @@ class MozconfigLoader(ProcessExecutionMi result = { 'path': path, 'topobjdir': None, 'configure_args': None, 'make_flags': None, 'make_extra': None, 'env': None, + 'vars': None, } if path is None: return result path = path.replace(os.sep, '/') result['configure_args'] = [] @@ -226,53 +231,69 @@ class MozconfigLoader(ProcessExecutionMi lines = lines[index + 1:] except ValueError: pass raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines) parsed = self._parse_loader_output(output) - all_variables = set(parsed['vars_before'].keys()) - all_variables |= set(parsed['vars_after'].keys()) + def diff_vars(vars_before, vars_after): + set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES + set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES + added = set2 - set1 + removed = set1 - set2 + maybe_modified = set1 & set2 + changed = { + 'added': {}, + 'removed': {}, + 'modified': {}, + 'unmodified': {}, + } - changed = { - 'added': {}, - 'removed': {}, - 'modified': {}, - 'unmodified': {}, - } + for key in added: + changed['added'][key] = vars_after[key] - for key in all_variables: - if key in self.IGNORE_SHELL_VARIABLES: - continue + for key in removed: + changed['removed'][key] = vars_before[key] - if key not in parsed['vars_before']: - changed['added'][key] = parsed['vars_after'][key] - continue - - if key not in parsed['vars_after']: - changed['removed'][key] = parsed['vars_before'][key] - continue + for key in maybe_modified: + if vars_before[key] != vars_after[key]: + changed['modified'][key] = ( + vars_before[key], vars_after[key]) + elif key in self.ENVIRONMENT_VARIABLES: + # In order for irrelevant environment variable changes not + # to incur in re-running configure, only a set of + # environment variables are stored when they are + # unmodified. Otherwise, changes such as using a different + # terminal window, or even rebooting, would trigger + # reconfigures. + changed['unmodified'][key] = vars_after[key] - if parsed['vars_before'][key] != parsed['vars_after'][key]: - changed['modified'][key] = ( - parsed['vars_before'][key], parsed['vars_after'][key]) - continue + return changed + + result['env'] = diff_vars(parsed['env_before'], parsed['env_after']) - changed['unmodified'][key] = parsed['vars_after'][key] - - result['env'] = changed + # Environment variables also appear as shell variables, but that's + # uninteresting duplication of information. Filter them out. + filt = lambda x, y: {k: v for k, v in x.items() if k not in y} + result['vars'] = diff_vars( + filt(parsed['vars_before'], parsed['env_before']), + filt(parsed['vars_after'], parsed['env_after']) + ) result['configure_args'] = [self._expand(o) for o in parsed['ac']] if moz_build_app is not None: result['configure_args'].extend(self._expand(o) for o in parsed['ac_app'][moz_build_app]) + if 'MOZ_OBJDIR' in parsed['env_before']: + result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR'] + mk = [self._expand(o) for o in parsed['mk']] for o in mk: match = self.RE_MAKE_VARIABLE.match(o) if match is None: result['make_extra'].append(o) continue @@ -292,16 +313,18 @@ class MozconfigLoader(ProcessExecutionMi return result def _parse_loader_output(self, output): mk_options = [] ac_options = [] ac_app_options = defaultdict(list) before_source = {} after_source = {} + env_before_source = {} + env_after_source = {} current = None current_type = None in_variable = None for line in output.splitlines(): # XXX This is an ugly hack. Data may be lost from things @@ -334,17 +357,24 @@ class MozconfigLoader(ProcessExecutionMi ac_app_options[app].append('\n'.join(current)) current = None current_type = None continue assert current_type is not None - if current_type in ('BEFORE_SOURCE', 'AFTER_SOURCE'): + vars_mapping = { + 'BEFORE_SOURCE': before_source, + 'AFTER_SOURCE': after_source, + 'ENV_BEFORE_SOURCE': env_before_source, + 'ENV_AFTER_SOURCE': env_after_source, + } + + if current_type in vars_mapping: # mozconfigs are sourced using the Bourne shell (or at least # in Bourne shell mode). This means |set| simply lists # variables from the current shell (not functions). (Note that # if Bash is installed in /bin/sh it acts like regular Bourne # and doesn't print functions.) So, lines should have the # form: # # key='value' @@ -395,29 +425,28 @@ class MozconfigLoader(ProcessExecutionMi in_variable = name current.append(value) continue else: value = value[:-1] if has_quote else value assert name is not None - if current_type == 'BEFORE_SOURCE': - before_source[name] = value - else: - after_source[name] = value + vars_mapping[current_type][name] = value current = [] continue current.append(line) return { 'mk': mk_options, 'ac': ac_options, 'ac_app': ac_app_options, 'vars_before': before_source, 'vars_after': after_source, + 'env_before': env_before_source, + 'env_after': env_after_source, } def _expand(self, s): return s.replace('@TOPSRCDIR@', self.topsrcdir)
--- a/python/mozbuild/mozbuild/mozconfig_loader +++ b/python/mozbuild/mozbuild/mozconfig_loader @@ -39,22 +39,30 @@ mk_add_options() { local opt for opt; do echo "------BEGIN_MK_OPTION" echo $opt echo "------END_MK_OPTION" done } +echo "------BEGIN_ENV_BEFORE_SOURCE" +env +echo "------END_ENV_BEFORE_SOURCE" + echo "------BEGIN_BEFORE_SOURCE" set echo "------END_BEFORE_SOURCE" topsrcdir=$1 . $2 unset topsrcdir echo "------BEGIN_AFTER_SOURCE" set echo "------END_AFTER_SOURCE" +echo "------BEGIN_ENV_AFTER_SOURCE" +env +echo "------END_ENV_AFTER_SOURCE" +
--- a/python/mozbuild/mozbuild/test/backend/common.py +++ b/python/mozbuild/mozbuild/test/backend/common.py @@ -78,16 +78,24 @@ CONFIGS = DefaultOnReadDict({ }, global_default={ 'defines': [], 'non_global_defines': [], 'substs': [], }) class BackendTester(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop('MOZ_OBJDIR', None) + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + def _get_environment(self, name): """Obtain a new instance of a ConfigEnvironment for a known profile. A new temporary object directory is created for the environment. The environment is cleaned up automatically when the test finishes. """ config = CONFIGS[name]
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -36,16 +36,24 @@ from mozbuild.test.common import MockCon import mozpack.path as mozpath data_path = mozpath.abspath(mozpath.dirname(__file__)) data_path = mozpath.join(data_path, 'data') class TestEmitterBasic(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop('MOZ_OBJDIR', None) + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + def reader(self, name): config = MockConfig(mozpath.join(data_path, name), extra_substs=dict( ENABLE_TESTS='1', BIN_SUFFIX='.prog', )) return BuildReader(config)
--- a/python/mozbuild/mozbuild/test/test_base.py +++ b/python/mozbuild/mozbuild/test/test_base.py @@ -16,40 +16,42 @@ from mozfile.mozfile import NamedTempora from mozunit import main from mach.logging import LoggingManager from mozbuild.base import ( BadEnvironmentException, MachCommandBase, MozbuildObject, + ObjdirMismatchException, PathArgument, ) from mozbuild.backend.configenvironment import ConfigEnvironment +from buildconfig import topsrcdir, topobjdir curdir = os.path.dirname(__file__) -topsrcdir = os.path.abspath(os.path.join(curdir, '..', '..', '..', '..')) log_manager = LoggingManager() class TestMozbuildObject(unittest.TestCase): def setUp(self): self._old_cwd = os.getcwd() self._old_env = dict(os.environ) os.environ.pop('MOZCONFIG', None) + os.environ.pop('MOZ_OBJDIR', None) def tearDown(self): os.chdir(self._old_cwd) os.environ.clear() os.environ.update(self._old_env) - def get_base(self): - return MozbuildObject(topsrcdir, None, log_manager) + def get_base(self, topobjdir=None): + return MozbuildObject(topsrcdir, None, log_manager, topobjdir=topobjdir) def test_objdir_config_guess(self): base = self.get_base() with NamedTemporaryFile() as mozconfig: os.environ[b'MOZCONFIG'] = mozconfig.name self.assertIsNotNone(base.topobjdir) @@ -66,17 +68,16 @@ class TestMozbuildObject(unittest.TestCa mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/foo/') mozconfig.flush() os.environ[b'MOZCONFIG'] = mozconfig.name self.assertEqual(base.topobjdir, os.path.join(base.topsrcdir, 'foo')) self.assertTrue(base.topobjdir.endswith('foo')) - @unittest.skip('Failing on buildbot.') def test_objdir_config_status(self): """Ensure @CONFIG_GUESS@ is handled when loading mozconfig.""" base = self.get_base() guess = base._config_guess # There may be symlinks involved, so we use real paths to ensure # path consistency. d = os.path.realpath(tempfile.mkdtemp()) @@ -97,26 +98,27 @@ class TestMozbuildObject(unittest.TestCa mozinfo = os.path.join(topobjdir, 'mozinfo.json') with open(mozinfo, 'wt') as fh: json.dump(dict( topsrcdir=d, mozconfig=mozconfig, ), fh) - os.environ[b'MOZCONFIG'] = mozconfig + os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8') os.chdir(topobjdir) - obj = MozbuildObject.from_environment() + obj = MozbuildObject.from_environment( + detect_virtualenv_mozinfo=False) self.assertEqual(obj.topobjdir, topobjdir) finally: + os.chdir(self._old_cwd) shutil.rmtree(d) - @unittest.skip('Failing on buildbot.') def test_relative_objdir(self): """Relative defined objdirs are loaded properly.""" d = os.path.realpath(tempfile.mkdtemp()) try: mozconfig = os.path.join(d, 'mozconfig') with open(mozconfig, 'wt') as fh: fh.write('mk_add_options MOZ_OBJDIR=./objdir') @@ -125,26 +127,28 @@ class TestMozbuildObject(unittest.TestCa mozinfo = os.path.join(topobjdir, 'mozinfo.json') with open(mozinfo, 'wt') as fh: json.dump(dict( topsrcdir=d, mozconfig=mozconfig, ), fh) - os.environ[b'MOZCONFIG'] = mozconfig + os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8') child = os.path.join(topobjdir, 'foo', 'bar') os.makedirs(child) os.chdir(child) - obj = MozbuildObject.from_environment() + obj = MozbuildObject.from_environment( + detect_virtualenv_mozinfo=False) self.assertEqual(obj.topobjdir, topobjdir) finally: + os.chdir(self._old_cwd) shutil.rmtree(d) @unittest.skipIf(not hasattr(os, 'symlink'), 'symlinks not available.') def test_symlink_objdir(self): """Objdir that is a symlink is loaded properly.""" d = os.path.realpath(tempfile.mkdtemp()) try: topobjdir_real = os.path.join(d, 'objdir') @@ -168,19 +172,19 @@ class TestMozbuildObject(unittest.TestCa obj = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) self.assertEqual(obj.topobjdir, topobjdir_real) os.chdir(topobjdir_real) obj = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) self.assertEqual(obj.topobjdir, topobjdir_real) finally: + os.chdir(self._old_cwd) shutil.rmtree(d) - @unittest.skip('Failed on buildbot (bug 853954)') def test_mach_command_base_inside_objdir(self): """Ensure a MachCommandBase constructed from inside the objdir works.""" d = os.path.realpath(tempfile.mkdtemp()) try: topobjdir = os.path.join(d, 'objdir') os.makedirs(topobjdir) @@ -199,26 +203,27 @@ class TestMozbuildObject(unittest.TestCa class MockMachContext(object): pass context = MockMachContext() context.cwd = topobjdir context.topdir = topsrcdir context.settings = None context.log_manager = None + context.detect_virtualenv_mozinfo=False o = MachCommandBase(context) self.assertEqual(o.topobjdir, topobjdir) self.assertEqual(o.topsrcdir, topsrcdir) finally: + os.chdir(self._old_cwd) shutil.rmtree(d) - @unittest.skip('Failing on buildbot.') def test_objdir_is_srcdir_rejected(self): """Ensure the srcdir configurations are rejected.""" d = os.path.realpath(tempfile.mkdtemp()) try: # The easiest way to do this is to create a mozinfo.json with data # that will never happen. mozinfo = os.path.join(d, 'mozinfo.json') @@ -226,49 +231,85 @@ class TestMozbuildObject(unittest.TestCa json.dump({'topsrcdir': d}, fh) os.chdir(d) with self.assertRaises(BadEnvironmentException): MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) finally: + os.chdir(self._old_cwd) + shutil.rmtree(d) + + def test_objdir_mismatch(self): + """Ensure MachCommandBase throwing on objdir mismatch.""" + d = os.path.realpath(tempfile.mkdtemp()) + + try: + real_topobjdir = os.path.join(d, 'real-objdir') + os.makedirs(real_topobjdir) + + topobjdir = os.path.join(d, 'objdir') + os.makedirs(topobjdir) + + topsrcdir = os.path.join(d, 'srcdir') + os.makedirs(topsrcdir) + + mozconfig = os.path.join(d, 'mozconfig') + with open(mozconfig, 'wt') as fh: + fh.write('mk_add_options MOZ_OBJDIR=%s' % real_topobjdir) + + mozinfo = os.path.join(topobjdir, 'mozinfo.json') + with open(mozinfo, 'wt') as fh: + json.dump(dict( + topsrcdir=topsrcdir, + mozconfig=mozconfig, + ), fh) + + os.chdir(topobjdir) + + with self.assertRaises(ObjdirMismatchException): + MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) + + finally: + os.chdir(self._old_cwd) shutil.rmtree(d) def test_config_guess(self): # It's difficult to test for exact values from the output of # config.guess because they vary depending on platform. base = self.get_base() result = base._config_guess self.assertIsNotNone(result) self.assertGreater(len(result), 0) - @unittest.skip('Failing on buildbot (bug 853954).') def test_config_environment(self): - base = self.get_base() + base = self.get_base(topobjdir=topobjdir) ce = base.config_environment self.assertIsInstance(ce, ConfigEnvironment) self.assertEqual(base.defines, ce.defines) self.assertEqual(base.substs, ce.substs) self.assertIsInstance(base.defines, dict) self.assertIsInstance(base.substs, dict) - @unittest.skip('Failing on buildbot (bug 853954).') def test_get_binary_path(self): - base = self.get_base() + base = self.get_base(topobjdir=topobjdir) platform = sys.platform # We should ideally use the config.status from the build. Let's install # a fake one. - substs = [('MOZ_APP_NAME', 'awesomeapp')] + substs = [ + ('MOZ_APP_NAME', 'awesomeapp'), + ('MOZ_BUILD_APP', 'awesomeapp'), + ] if sys.platform.startswith('darwin'): substs.append(('OS_ARCH', 'Darwin')) substs.append(('BIN_SUFFIX', '')) substs.append(('MOZ_MACBUNDLE_NAME', 'Nightly.app')) elif sys.platform.startswith(('win32', 'cygwin')): substs.append(('OS_ARCH', 'WINNT')) substs.append(('BIN_SUFFIX', '.exe')) else: @@ -293,17 +334,17 @@ class TestMozbuildObject(unittest.TestCa self.assertTrue(p.endswith('awesomeapp.exe')) else: self.assertTrue(p.endswith('dist/bin/awesomeapp')) p = base.get_binary_path(validate_exists=False, where="staged-package") if platform.startswith('darwin'): self.assertTrue(p.endswith('awesomeapp/Nightly.app/Contents/MacOS/awesomeapp')) elif platform.startswith(('win32', 'cygwin')): - self.assertTrue(p.endswith('awesomeapp/awesomeapp.exe')) + self.assertTrue(p.endswith('awesomeapp\\awesomeapp.exe')) else: self.assertTrue(p.endswith('awesomeapp/awesomeapp')) self.assertRaises(Exception, base.get_binary_path, where="somewhere") p = base.get_binary_path('foobar', validate_exists=False) if platform.startswith('win32'): self.assertTrue(p.endswith('foobar.exe'))
--- a/python/mozbuild/mozbuild/test/test_mozconfig.py +++ b/python/mozbuild/mozbuild/test/test_mozconfig.py @@ -24,16 +24,17 @@ from mozbuild.mozconfig import ( MozconfigLoader, ) class TestMozconfigLoader(unittest.TestCase): def setUp(self): self._old_env = dict(os.environ) os.environ.pop('MOZCONFIG', None) + os.environ.pop('MOZ_OBJDIR', None) os.environ.pop('CC', None) os.environ.pop('CXX', None) self._temp_dirs = set() def tearDown(self): os.environ.clear() os.environ.update(self._old_env) @@ -238,32 +239,34 @@ class TestMozconfigLoader(unittest.TestC self.assertEqual(result, { 'path': None, 'topobjdir': None, 'configure_args': None, 'make_flags': None, 'make_extra': None, 'env': None, + 'vars': None, }) def test_read_empty_mozconfig(self): with NamedTemporaryFile(mode='w') as mozconfig: result = self.get_loader().read_mozconfig(mozconfig.name) self.assertEqual(result['path'], mozconfig.name) self.assertIsNone(result['topobjdir']) self.assertEqual(result['configure_args'], []) self.assertEqual(result['make_flags'], []) self.assertEqual(result['make_extra'], []) for f in ('added', 'removed', 'modified'): + self.assertEqual(len(result['vars'][f]), 0) self.assertEqual(len(result['env'][f]), 0) - self.assertGreater(len(result['env']['unmodified']), 0) + self.assertEqual(result['env']['unmodified'], {}) def test_read_capture_ac_options(self): """Ensures ac_add_options calls are captured.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('ac_add_options --enable-debug\n') mozconfig.write('ac_add_options --disable-tests --enable-foo\n') mozconfig.write('ac_add_options --foo="bar baz"\n') mozconfig.flush() @@ -311,16 +314,32 @@ class TestMozconfigLoader(unittest.TestC mozconfig.write('mk_add_options BIZ=1\n') mozconfig.flush() result = self.get_loader().read_mozconfig(mozconfig.name) self.assertEqual(result['topobjdir'], '/foo/bar') self.assertEqual(result['make_flags'], '-j8') self.assertEqual(result['make_extra'], ['FOO=BAR BAZ', 'BIZ=1']) + def test_read_empty_mozconfig_objdir_environ(self): + os.environ[b'MOZ_OBJDIR'] = b'obj-firefox' + with NamedTemporaryFile(mode='w') as mozconfig: + result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['topobjdir'], 'obj-firefox') + + def test_read_capture_mk_options_objdir_environ(self): + """Ensures mk_add_options calls are captured and override the environ.""" + os.environ[b'MOZ_OBJDIR'] = b'obj-firefox' + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('mk_add_options MOZ_OBJDIR=/foo/bar\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['topobjdir'], '/foo/bar') + def test_read_moz_objdir_substitution(self): """Ensure @TOPSRCDIR@ substitution is recognized in MOZ_OBJDIR.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/some-objdir') mozconfig.flush() loader = self.get_loader() result = loader.read_mozconfig(mozconfig.name) @@ -332,94 +351,119 @@ class TestMozconfigLoader(unittest.TestC """New variables declared in mozconfig file are detected.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('CC=/usr/local/bin/clang\n') mozconfig.write('CXX=/usr/local/bin/clang++\n') mozconfig.flush() result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['env']['added'], { + self.assertEqual(result['vars']['added'], { 'CC': '/usr/local/bin/clang', 'CXX': '/usr/local/bin/clang++'}) + self.assertEqual(result['env']['added'], {}) def test_read_exported_variables(self): """Exported variables are caught as new variables.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('export MY_EXPORTED=woot\n') mozconfig.flush() result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['vars']['added'], {}) self.assertEqual(result['env']['added'], { 'MY_EXPORTED': 'woot'}) def test_read_modify_variables(self): """Variables modified by mozconfig are detected.""" os.environ[b'CC'] = b'/usr/bin/gcc' with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('CC=/usr/local/bin/clang\n') mozconfig.flush() result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['vars']['modified'], {}) self.assertEqual(result['env']['modified'], { 'CC': ('/usr/bin/gcc', '/usr/local/bin/clang') }) + def test_read_unmodified_variables(self): + """Variables modified by mozconfig are detected.""" + os.environ[b'CC'] = b'/usr/bin/gcc' + + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['unmodified'], {}) + self.assertEqual(result['env']['unmodified'], { + 'CC': '/usr/bin/gcc' + }) + def test_read_removed_variables(self): """Variables unset by the mozconfig are detected.""" os.environ[b'CC'] = b'/usr/bin/clang' with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('unset CC\n') mozconfig.flush() result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['vars']['removed'], {}) self.assertEqual(result['env']['removed'], { 'CC': '/usr/bin/clang'}) def test_read_multiline_variables(self): """Ensure multi-line variables are captured properly.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('multi="foo\nbar"\n') mozconfig.write('single=1\n') mozconfig.flush() result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertEqual(result['env']['added'], { + self.assertEqual(result['vars']['added'], { 'multi': 'foo\nbar', 'single': '1' }) + self.assertEqual(result['env']['added'], {}) def test_read_topsrcdir_defined(self): """Ensure $topsrcdir references work as expected.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('TEST=$topsrcdir') mozconfig.flush() loader = self.get_loader() result = loader.read_mozconfig(mozconfig.name) - self.assertEqual(result['env']['added']['TEST'], + self.assertEqual(result['vars']['added']['TEST'], loader.topsrcdir.replace(os.sep, '/')) + self.assertEqual(result['env']['added'], {}) def test_read_empty_variable_value(self): """Ensure empty variable values are parsed properly.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('EMPTY=\n') + mozconfig.write('export EXPORT_EMPTY=\n') mozconfig.flush() result = self.get_loader().read_mozconfig(mozconfig.name) - self.assertIn('EMPTY', result['env']['added']) - self.assertEqual(result['env']['added']['EMPTY'], '') + self.assertEqual(result['vars']['added'], { + 'EMPTY': '', + }) + self.assertEqual(result['env']['added'], { + 'EXPORT_EMPTY': '' + }) def test_read_load_exception(self): """Ensure non-0 exit codes in mozconfigs are handled properly.""" with NamedTemporaryFile(mode='w') as mozconfig: mozconfig.write('echo "hello world"\n') mozconfig.write('exit 1\n') mozconfig.flush()
--- a/testing/xpcshell/selftest.py +++ b/testing/xpcshell/selftest.py @@ -7,16 +7,17 @@ from __future__ import with_statement import sys, os, unittest, tempfile, shutil import mozinfo from StringIO import StringIO from xml.etree.ElementTree import ElementTree from mozbuild.base import MozbuildObject +os.environ.pop('MOZ_OBJDIR') build_obj = MozbuildObject.from_environment() from runxpcshelltests import XPCShellTests mozinfo.find_and_update_from_json() objdir = build_obj.topobjdir.encode("utf-8") xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")