Bug 1283052 - Remove some of the magic around mozconfig detection. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 08 Jul 2016 08:43:17 +0900
changeset 344218 764dfcab17e9354843c9f4668c9f3617eb8e9a96
parent 344140 be469964755162110602a8690cc9e7034fd3981b
child 344219 32de23e9cca627e3e48eb56c186a1598a7ef4c20
push id6389
push userraliiev@mozilla.com
push dateMon, 19 Sep 2016 13:38:22 +0000
treeherdermozilla-beta@01d67bfe6c81 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1283052
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1283052 - Remove some of the magic around mozconfig detection. r=gps The mozconfig detection logic has bitten us on many occasions in the past. The following changes are made to tentatively improve the situation: - The API is modified such that autodetection of the mozconfig has to be a conscious decision made by the caller, and not triggered any time there is no mozconfig given, which could be a conscious decision of the opposite. - mozinfo.json now stores the actual mozconfig (or lack thereof) used during configure.
build/moz.configure/init.configure
configure.py
python/mozbuild/mozbuild/backend/configenvironment.py
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/config_status.py
python/mozbuild/mozbuild/mozconfig.py
python/mozbuild/mozbuild/mozinfo.py
python/mozbuild/mozbuild/test/test_mozinfo.py
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -110,16 +110,18 @@ def mozconfig(current_project, mozconfig
     loader = MozconfigLoader(build_env.topsrcdir)
     current_project = current_project[0] if current_project else None
     mozconfig = mozconfig[0] if mozconfig else None
     mozconfig = loader.find_mozconfig(env={'MOZCONFIG': mozconfig})
     mozconfig = loader.read_mozconfig(mozconfig, moz_build_app=current_project)
 
     return mozconfig
 
+set_config('MOZCONFIG', depends(mozconfig)(lambda m: m['path']))
+
 
 # Hacks related to old-configure
 # ==============================
 
 @depends('--help')
 def old_configure_assignments(help):
     return []
 
--- a/configure.py
+++ b/configure.py
@@ -45,16 +45,17 @@ def config_status(config):
         if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR')
     }
     sanitized_config['defines'] = {
         k: sanitized_bools(v) for k, v in config['DEFINES'].iteritems()
     }
     sanitized_config['non_global_defines'] = config['non_global_defines']
     sanitized_config['topsrcdir'] = config['TOPSRCDIR']
     sanitized_config['topobjdir'] = config['TOPOBJDIR']
+    sanitized_config['mozconfig'] = config.get('MOZCONFIG')
 
     # Create config.status. Eventually, we'll want to just do the work it does
     # here, when we're able to skip configure tests/use cached results/not rely
     # on autoconf.
     print("Creating config.status", file=sys.stderr)
     encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
     with codecs.open('config.status', 'w', encoding) as fh:
         fh.write('#!%s\n' % config['PYTHON'])
@@ -63,17 +64,17 @@ def config_status(config):
         # for True, False and None are true, false and null, which don't exist.
         # Define them.
         fh.write('true, false, null = True, False, None\n')
         for k, v in sanitized_config.iteritems():
             fh.write('%s = ' % k)
             json.dump(v, fh, sort_keys=True, indent=4, ensure_ascii=False)
             fh.write('\n')
         fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
-                 "'non_global_defines', 'substs']")
+                 "'non_global_defines', 'substs', 'mozconfig']")
 
         if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
             fh.write('''
 if __name__ == '__main__':
     args = dict([(name, globals()[name]) for name in __all__])
     from mozbuild.config_status import config_status
     config_status(**args)
 ''')
--- a/python/mozbuild/mozbuild/backend/configenvironment.py
+++ b/python/mozbuild/mozbuild/backend/configenvironment.py
@@ -29,16 +29,17 @@ class BuildConfig(object):
 
     def __init__(self):
         self.topsrcdir = None
         self.topobjdir = None
         self.defines = {}
         self.non_global_defines = []
         self.substs = {}
         self.files = []
+        self.mozconfig = None
 
     @classmethod
     def from_config_status(cls, path):
         """Create an instance from a config.status file."""
         code_cache = cls._CODE_CACHE
         mtime = os.path.getmtime(path)
 
         # cache the compiled code as it can be reused
@@ -101,26 +102,27 @@ class ConfigEnvironment(object):
 
     ConfigEnvironment expects a "top_srcdir" subst to be set with the top
     source directory, in msys format on windows. It is used to derive a
     "srcdir" subst when treating config files. It can either be an absolute
     path or a path relative to the topobjdir.
     """
 
     def __init__(self, topsrcdir, topobjdir, defines=None,
-        non_global_defines=None, substs=None, source=None):
+        non_global_defines=None, substs=None, source=None, mozconfig=None):
 
         if not source:
             source = mozpath.join(topobjdir, 'config.status')
         self.source = source
         self.defines = ReadOnlyDict(defines or {})
         self.non_global_defines = non_global_defines or []
         self.substs = dict(substs or {})
         self.topsrcdir = mozpath.abspath(topsrcdir)
         self.topobjdir = mozpath.abspath(topobjdir)
+        self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None
         self.lib_prefix = self.substs.get('LIB_PREFIX', '')
         if 'LIB_SUFFIX' in self.substs:
             self.lib_suffix = '.%s' % self.substs['LIB_SUFFIX']
         self.dll_prefix = self.substs.get('DLL_PREFIX', '')
         self.dll_suffix = self.substs.get('DLL_SUFFIX', '')
         if self.substs.get('IMPORT_LIB_SUFFIX'):
             self.import_prefix = self.lib_prefix
             self.import_suffix = '.%s' % self.substs['IMPORT_LIB_SUFFIX']
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -68,50 +68,49 @@ class ObjdirMismatchException(BadEnviron
 class MozbuildObject(ProcessExecutionMixin):
     """Base class providing basic functionality useful to many modules.
 
     Modules in this package typically require common functionality such as
     accessing the current config, getting the location of the source directory,
     running processes, etc. This classes provides that functionality. Other
     modules can inherit from this class to obtain this functionality easily.
     """
-    def __init__(self, topsrcdir, settings, log_manager, topobjdir=None):
+    def __init__(self, topsrcdir, settings, log_manager, topobjdir=None,
+                 mozconfig=MozconfigLoader.AUTODETECT):
         """Create a new Mozbuild object instance.
 
         Instances are bound to a source directory, a ConfigSettings instance,
         and a LogManager instance. The topobjdir may be passed in as well. If
         it isn't, it will be calculated from the active mozconfig.
         """
         self.topsrcdir = mozpath.normsep(topsrcdir)
         self.settings = settings
 
         self.populate_logger()
         self.log_manager = log_manager
 
         self._make = None
         self._topobjdir = mozpath.normsep(topobjdir) if topobjdir else topobjdir
-        self._mozconfig = None
+        self._mozconfig = mozconfig
         self._config_guess_output = None
         self._config_environment = None
         self._virtualenv_manager = None
 
     @classmethod
     def from_environment(cls, cwd=None, detect_virtualenv_mozinfo=True):
         """Create a MozbuildObject by detecting the proper one from the env.
 
         This examines environment state like the current working directory and
         creates a MozbuildObject from the found source directory, mozconfig, etc.
 
         The role of this function is to identify a topsrcdir, topobjdir, and
         mozconfig file.
 
         If the current working directory is inside a known objdir, we always
-        use the topsrcdir and mozconfig associated with that objdir. If no
-        mozconfig is associated with that objdir, we fall back to looking for
-        the mozconfig in the usual places.
+        use the topsrcdir and mozconfig associated with that objdir.
 
         If the current working directory is inside a known srcdir, we use that
         topsrcdir and look for mozconfigs using the default mechanism, which
         looks inside environment variables.
 
         If the current Python interpreter is running from a virtualenv inside
         an objdir, we use that as our objdir.
 
@@ -121,17 +120,17 @@ class MozbuildObject(ProcessExecutionMix
         mozinfo.json file relative to the virtualenv directory. This was
         added to facilitate testing. Callers likely shouldn't change the
         default.
         """
 
         cwd = cwd or os.getcwd()
         topsrcdir = None
         topobjdir = None
-        mozconfig = None
+        mozconfig = MozconfigLoader.AUTODETECT
 
         def load_mozinfo(path):
             info = json.load(open(path, 'rt'))
             topsrcdir = info.get('topsrcdir')
             topobjdir = os.path.dirname(path)
             mozconfig = info.get('mozconfig')
             return topsrcdir, topobjdir, mozconfig
 
@@ -191,17 +190,18 @@ class MozbuildObject(ProcessExecutionMix
 
             if topsrcdir == topobjdir:
                 raise BadEnvironmentException('The object directory appears '
                     'to be the same as your source directory (%s). This build '
                     'configuration is not supported.' % topsrcdir)
 
         # If we can't resolve topobjdir, oh well. The constructor will figure
         # it out via config.guess.
-        return cls(topsrcdir, None, None, topobjdir=topobjdir)
+        return cls(topsrcdir, None, None, topobjdir=topobjdir,
+                   mozconfig=mozconfig)
 
     @staticmethod
     def resolve_mozconfig_topobjdir(topsrcdir, mozconfig, default=None):
         topobjdir = mozconfig['topobjdir'] or default
         if not topobjdir:
             return None
 
         if '@CONFIG_GUESS@' in topobjdir:
@@ -232,19 +232,19 @@ class MozbuildObject(ProcessExecutionMix
         return self._virtualenv_manager
 
     @property
     def mozconfig(self):
         """Returns information about the current mozconfig file.
 
         This a dict as returned by MozconfigLoader.read_mozconfig()
         """
-        if self._mozconfig is None:
+        if self._mozconfig is MozconfigLoader.AUTODETECT:
             loader = MozconfigLoader(self.topsrcdir)
-            self._mozconfig = loader.read_mozconfig(
+            self._mozconfig = loader.read_mozconfig(path=self._mozconfig,
                 moz_build_app=os.environ.get('MOZ_CURRENT_PROJECT'))
 
         return self._mozconfig
 
     @property
     def config_environment(self):
         """Returns the ConfigEnvironment for the current build configuration.
 
--- a/python/mozbuild/mozbuild/config_status.py
+++ b/python/mozbuild/mozbuild/config_status.py
@@ -57,17 +57,18 @@ files by running:
 
    mach build-backend --backend=VisualStudio
 
 ===============================
 '''.strip()
 
 
 def config_status(topobjdir='.', topsrcdir='.', defines=None,
-                  non_global_defines=None, substs=None, source=None):
+                  non_global_defines=None, substs=None, source=None,
+                  mozconfig=None):
     '''Main function, providing config.status functionality.
 
     Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS
     variables.
 
     Without the -n option, this program acts as config.status and considers
     the current directory as the top object directory, even when config.status
     is in a different directory. It will, however, treat the directory
@@ -108,17 +109,18 @@ def config_status(topobjdir='.', topsrcd
                         help='do everything except writing files out.')
     options = parser.parse_args()
 
     # Without -n, the current directory is meant to be the top object directory
     if not options.not_topobjdir:
         topobjdir = os.path.abspath('.')
 
     env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
-            non_global_defines=non_global_defines, substs=substs, source=source)
+            non_global_defines=non_global_defines, substs=substs,
+            source=source, mozconfig=mozconfig)
 
     # mozinfo.json only needs written if configure changes and configure always
     # passes this environment variable.
     if 'WRITE_MOZINFO' in os.environ:
         write_mozinfo(os.path.join(topobjdir, 'mozinfo.json'), env, os.environ)
 
     cpu_start = time.clock()
     time_start = time.time()
--- a/python/mozbuild/mozbuild/mozconfig.py
+++ b/python/mozbuild/mozbuild/mozconfig.py
@@ -75,16 +75,18 @@ class MozconfigLoader(object):
     DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh')
 
     IGNORE_SHELL_VARIABLES = {'_'}
 
     ENVIRONMENT_VARIABLES = {
         'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'MOZ_OBJDIR',
     }
 
+    AUTODETECT = object()
+
     def __init__(self, topsrcdir):
         self.topsrcdir = topsrcdir
 
     @property
     def _loader_script(self):
         our_dir = os.path.abspath(os.path.dirname(__file__))
 
         return os.path.join(our_dir, 'mozconfig_loader')
@@ -185,26 +187,26 @@ class MozconfigLoader(object):
                 raise MozconfigFindException(
                     MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir))
 
         return None
 
     def read_mozconfig(self, path=None, moz_build_app=None):
         """Read the contents of a mozconfig into a data structure.
 
-        This takes the path to a mozconfig to load. If it is not defined, we
-        will try to find a mozconfig from the environment using
+        This takes the path to a mozconfig to load. If the given path is
+        AUTODETECT, will try to find a mozconfig from the environment using
         find_mozconfig().
 
         mozconfig files are shell scripts. So, we can't just parse them.
         Instead, we run the shell script in a wrapper which allows us to record
         state from execution. Thus, the output from a mozconfig is a friendly
         static data structure.
         """
-        if path is None:
+        if path is self.AUTODETECT:
             path = self.find_mozconfig()
 
         result = {
             'path': path,
             'topobjdir': None,
             'configure_args': None,
             'make_flags': None,
             'make_extra': None,
--- a/python/mozbuild/mozbuild/mozinfo.py
+++ b/python/mozbuild/mozbuild/mozinfo.py
@@ -5,17 +5,17 @@
 # This module produces a JSON file that provides basic build info and
 # configuration metadata.
 
 from __future__ import absolute_import
 
 import os
 import re
 import json
-import mozbuild.mozconfig as mozconfig
+
 
 def build_dict(config, env=os.environ):
     """
     Build a dict containing data about the build configuration from
     the environment.
     """
     substs = config.substs
 
@@ -24,19 +24,18 @@ def build_dict(config, env=os.environ):
     missing = [r for r in required if r not in substs]
     if missing:
         raise Exception("Missing required environment variables: %s" %
                         ', '.join(missing))
 
     d = {}
     d['topsrcdir'] = config.topsrcdir
 
-    the_mozconfig = mozconfig.MozconfigLoader(config.topsrcdir).find_mozconfig(env)
-    if the_mozconfig:
-        d['mozconfig'] = the_mozconfig
+    if config.mozconfig:
+        d['mozconfig'] = config.mozconfig
 
     # os
     o = substs["OS_TARGET"]
     known_os = {"Linux": "linux",
                 "WINNT": "win",
                 "Darwin": "mac",
                 "Android": "b2g" if substs.get("MOZ_WIDGET_TOOLKIT") == "gonk" else "android"}
     if o in known_os:
--- a/python/mozbuild/mozbuild/test/test_mozinfo.py
+++ b/python/mozbuild/mozbuild/test/test_mozinfo.py
@@ -240,17 +240,18 @@ class TestWriteMozinfo(unittest.TestCase
             TARGET_CPU='i386',
             MOZ_WIDGET_TOOLKIT='windows',
         ))
         tempdir = tempfile.tempdir
         c.topsrcdir = tempdir
         with NamedTemporaryFile(dir=os.path.normpath(c.topsrcdir)) as mozconfig:
             mozconfig.write('unused contents')
             mozconfig.flush()
-            write_mozinfo(self.f, c, {'MOZCONFIG': mozconfig.name})
+            c.mozconfig = mozconfig.name
+            write_mozinfo(self.f, c)
             with open(self.f) as f:
                 d = json.load(f)
                 self.assertEqual('win', d['os'])
                 self.assertEqual('x86', d['processor'])
                 self.assertEqual('windows', d['toolkit'])
                 self.assertEqual(tempdir, d['topsrcdir'])
                 self.assertEqual(mozconfig.name, d['mozconfig'])
                 self.assertEqual(32, d['bits'])