Bug 902825 - Create config.statusd directory; r?glandium draft
authorMike Shal <mshal@mozilla.com>
Fri, 18 Aug 2017 10:41:50 -0400
changeset 655244 1bf60dc967bf7c8e92fc41649a2945eb7941fec9
parent 653766 d10c97627b51a226e19d0fa801201897fe1932f6
child 655245 a0cbfe3fe8d35a0d9d75e7627dc425d826081dd9
push id76811
push userbmo:mshal@mozilla.com
push dateTue, 29 Aug 2017 19:45:55 +0000
reviewersglandium
bugs902825
milestone57.0a1
Bug 902825 - Create config.statusd directory; r?glandium The config.statusd directory is created alongside config.status, which contains the same information but is split across many files instead of all in a single file. This allows the build system to track dependencies on individual configure values. MozReview-Commit-ID: 2DbwKCJuNSX
configure.py
python/mozbuild/mozbuild/backend/configenvironment.py
--- a/configure.py
+++ b/configure.py
@@ -12,16 +12,17 @@ import sys
 import textwrap
 
 
 base_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(base_dir, 'python', 'mozbuild'))
 from mozbuild.configure import ConfigureSandbox
 from mozbuild.makeutil import Makefile
 from mozbuild.pythonutil import iter_modules_in_path
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
 from mozbuild.util import (
     indented_repr,
     encode,
 )
 
 
 def main(argv):
     config = {}
@@ -85,16 +86,19 @@ def config_status(config):
                 if __name__ == '__main__':
                     from mozbuild.util import patch_main
                     patch_main()
                     from mozbuild.config_status import config_status
                     args = dict([(name, globals()[name]) for name in __all__])
                     config_status(**args)
             '''))
 
+    partial_config = PartialConfigEnvironment(config['TOPOBJDIR'])
+    partial_config.write_vars(sanitized_config)
+
     # Write out a depfile so Make knows to re-run configure when relevant Python
     # changes.
     mk = Makefile()
     rule = mk.create_rule()
     rule.add_targets(["$(OBJDIR)/config.status"])
     rule.add_dependencies(itertools.chain(config['ALL_CONFIGURE_PATHS'],
                                           iter_modules_in_path(config['TOPOBJDIR'],
                                                                config['TOPSRCDIR'])))
--- a/python/mozbuild/mozbuild/backend/configenvironment.py
+++ b/python/mozbuild/mozbuild/backend/configenvironment.py
@@ -2,22 +2,24 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 import os
 import sys
 
-from collections import Iterable
+from collections import Iterable, OrderedDict
 from types import StringTypes, ModuleType
 
 import mozpack.path as mozpath
 
 from mozbuild.util import (
+    FileAvoidWrite,
+    indented_repr,
     memoized_property,
     ReadOnlyDict,
 )
 from mozbuild.shellutil import quote as shell_quote
 
 
 if sys.version_info.major == 2:
     text_type = unicode
@@ -206,8 +208,122 @@ class ConfigEnvironment(object):
         return ReadOnlyDict(acdefines)
 
     @staticmethod
     def from_config_status(path):
         config = BuildConfig.from_config_status(path)
 
         return ConfigEnvironment(config.topsrcdir, config.topobjdir,
             config.defines, config.non_global_defines, config.substs, path)
+
+
+class PartialConfigEnvironment(object):
+    """Allows access to individual config.status items via config.statusd/* files.
+
+    This class is similar to the full ConfigEnvironment, which uses
+    config.status, except this allows access and tracks dependencies to
+    individual configure values. It is intended to be used during the build
+    process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and
+    anything else that may need to access specific substs or defines.
+
+    Creating a PartialConfigEnvironment requires only the topobjdir, which is
+    needed to distinguish between the top-level environment and the js/src
+    environment.
+
+    The PartialConfigEnvironment automatically defines one additional subst variable
+    from all the defines not appearing in non_global_defines and mozconfig_defines:
+      - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
+        preprocessor command lines. The order in which defines were given
+        when creating the ConfigEnvironment is preserved.
+
+    and one additional define from all the defines as a dictionary:
+      - ALLDEFINES contains all of the global defines as a dictionary. This is
+      intended to be used instead of the defines structure from config.status so
+      that scripts can depend directly on its value.
+    """
+    def __init__(self, topobjdir):
+        self._substs = {}
+        self._defines = {}
+        self.topobjdir = topobjdir
+        self._files = set()
+        self._config_track = mozpath.join(topobjdir, 'config.statusd', 'config.track')
+
+    def _get_data(self, typ, var, default_value=None):
+        data = default_value
+        try:
+            filename = mozpath.join(self.topobjdir, 'config.statusd', typ, var)
+            self._files.add(filename)
+            with open(filename) as f:
+                data = eval(f.read())
+        except IOError:
+            pass
+        return data
+
+    def get_subst(self, var, default_value=None):
+        if var not in self._substs:
+            self._substs[var] = self._get_data('substs', var, default_value)
+            if (self._substs[var] is not None) and (var not in ('CPP', 'CXXCPP', 'SHELL')) and (var in os.environ):
+                self._substs[var] = os.environ[var]
+        return self._substs[var]
+
+    def set_subst(self, var, value):
+        self._substs[var] = value
+
+    def get_define(self, var, default_value=None):
+        if var not in self._defines:
+            self._defines[var] = self._get_data('defines', var, default_value)
+        return self._defines[var]
+
+    def _load_config_track(self):
+        existing_files = set()
+        try:
+            with open(self._config_track) as fh:
+                existing_files.update(fh.read().splitlines())
+        except IOError:
+            pass
+        return existing_files;
+
+    def _write_file(self, group, key, value):
+        filename = mozpath.join('config.statusd', group, key)
+        with FileAvoidWrite(filename) as fh:
+            fh.write('%s\n' % indented_repr(value))
+        return filename
+
+    def _write_group(self, group, values):
+        new_files = set()
+        for k, v in values.iteritems():
+            new_files.add(self._write_file(group, k, v))
+        return new_files
+
+    def write_vars(self, config):
+        existing_files = self._load_config_track()
+        new_files = set()
+        new_files.update(self._write_group('substs', config['substs']))
+        new_files.update(self._write_group('defines', config['defines']))
+
+        global_defines = [name for name in config['defines']
+            if not name in config['non_global_defines']
+            ]
+        acdefines = ' '.join(['-D%s=%s' % (name,
+            shell_quote(config['defines'][name]).replace('$', '$$'))
+            for name in sorted(global_defines)])
+        new_files.update(self._write_file('substs', 'ACDEFINES', acdefines))
+
+        all_defines = OrderedDict()
+        for k in global_defines:
+            all_defines[k] = config['defines'][k]
+        new_files.update(self._write_file('defines', 'ALLDEFINES', all_defines))
+
+        remove_files = existing_files - new_files
+        for filename in remove_files:
+            # We can't actually os.remove() here, since make would not see that the
+            # file has been removed and that the target needs to be updated. Instead
+            # we just overwrite the file with a value of None, which is equivalent
+            # to a non-existing file.
+            with FileAvoidWrite(filename) as fh:
+                fh.write('None')
+
+        with open(self._config_track, 'w') as fh:
+            for f in sorted(new_files):
+                fh.write('%s\n' % f)
+
+    def get_dependencies(self):
+        return ['$(wildcard %s)' % f for f in self._files]