Bug 1402012 - Create config.statusd directory; r=glandium
authorMike Shal <mshal@mozilla.com>
Fri, 18 Aug 2017 10:41:50 -0400
changeset 437010 d1d46bec63ce86d1884c89d9e254a237bafb9780
parent 437009 776ded750b88f711f99dc2a658bb8b85f53235ff
child 437011 22ebbb5d30d470415a729d5b4e9764998023cf07
push id1618
push userCallek@gmail.com
push dateThu, 11 Jan 2018 17:45:48 +0000
treeherdermozilla-release@882ca853e05a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1402012
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1402012 - Create config.statusd directory; r=glandium The config.statusd directory is created alongside config.status, which contains the same information but is split across many files instead of all in a single file. This allows the build system to track dependencies on individual configure values. MozReview-Commit-ID: 2DbwKCJuNSX
configure.py
python/mozbuild/mozbuild/backend/configenvironment.py
python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py
python/mozbuild/mozbuild/test/python.ini
--- a/configure.py
+++ b/configure.py
@@ -12,16 +12,17 @@ import sys
 import textwrap
 
 
 base_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(base_dir, 'python', 'mozbuild'))
 from mozbuild.configure import ConfigureSandbox
 from mozbuild.makeutil import Makefile
 from mozbuild.pythonutil import iter_modules_in_path
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
 from mozbuild.util import (
     indented_repr,
     encode,
 )
 
 
 def main(argv):
     config = {}
@@ -85,16 +86,19 @@ def config_status(config):
                 if __name__ == '__main__':
                     from mozbuild.util import patch_main
                     patch_main()
                     from mozbuild.config_status import config_status
                     args = dict([(name, globals()[name]) for name in __all__])
                     config_status(**args)
             '''))
 
+    partial_config = PartialConfigEnvironment(config['TOPOBJDIR'])
+    partial_config.write_vars(sanitized_config)
+
     # Write out a depfile so Make knows to re-run configure when relevant Python
     # changes.
     mk = Makefile()
     rule = mk.create_rule()
     rule.add_targets(["$(OBJDIR)/config.status"])
     rule.add_dependencies(itertools.chain(config['ALL_CONFIGURE_PATHS'],
                                           iter_modules_in_path(config['TOPOBJDIR'],
                                                                config['TOPSRCDIR'])))
--- a/python/mozbuild/mozbuild/backend/configenvironment.py
+++ b/python/mozbuild/mozbuild/backend/configenvironment.py
@@ -1,23 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 import os
 import sys
+import json
 
-from collections import Iterable
+from collections import Iterable, OrderedDict
 from types import StringTypes, ModuleType
 
 import mozpack.path as mozpath
 
 from mozbuild.util import (
+    FileAvoidWrite,
     memoized_property,
     ReadOnlyDict,
 )
 from mozbuild.shellutil import quote as shell_quote
 
 
 if sys.version_info.major == 2:
     text_type = unicode
@@ -206,8 +208,159 @@ class ConfigEnvironment(object):
         return ReadOnlyDict(acdefines)
 
     @staticmethod
     def from_config_status(path):
         config = BuildConfig.from_config_status(path)
 
         return ConfigEnvironment(config.topsrcdir, config.topobjdir,
             config.defines, config.non_global_defines, config.substs, path)
+
+
+class PartialConfigDict(object):
+    """Facilitates mapping the config.statusd defines & substs with dict-like access.
+
+    This allows a buildconfig client to use buildconfig.defines['FOO'] (and
+    similar for substs), where the value of FOO is delay-loaded until it is
+    needed.
+    """
+    def __init__(self, config_statusd, typ, environ_override=False):
+        self._dict = {}
+        self._datadir = mozpath.join(config_statusd, typ)
+        self._config_track = mozpath.join(self._datadir, 'config.track')
+        self._files = set()
+        self._environ_override = environ_override
+
+    def _load_config_track(self):
+        existing_files = set()
+        try:
+            with open(self._config_track) as fh:
+                existing_files.update(fh.read().splitlines())
+        except IOError:
+            pass
+        return existing_files
+
+    def _write_file(self, key, value):
+        filename = mozpath.join(self._datadir, key)
+        with FileAvoidWrite(filename) as fh:
+            json.dump(value, fh, indent=4)
+        return filename
+
+    def _fill_group(self, values):
+        # Clear out any cached values. This is mostly for tests that will check
+        # the environment, write out a new set of variables, and then check the
+        # environment again. Normally only configure ends up calling this
+        # function, and other consumers create their own
+        # PartialConfigEnvironments in new python processes.
+        self._dict = {}
+
+        existing_files = self._load_config_track()
+
+        new_files = set()
+        for k, v in values.iteritems():
+            new_files.add(self._write_file(k, v))
+
+        for filename in existing_files - new_files:
+            # We can't actually os.remove() here, since make would not see that the
+            # file has been removed and that the target needs to be updated. Instead
+            # we just overwrite the file with a value of None, which is equivalent
+            # to a non-existing file.
+            with FileAvoidWrite(filename) as fh:
+                json.dump(None, fh)
+
+        with FileAvoidWrite(self._config_track) as fh:
+            for f in sorted(new_files):
+                fh.write('%s\n' % f)
+
+    def __getitem__(self, key):
+        if self._environ_override:
+            if (key not in ('CPP', 'CXXCPP', 'SHELL')) and (key in os.environ):
+                return os.environ[key]
+
+        if key not in self._dict:
+            data = None
+            try:
+                filename = mozpath.join(self._datadir, key)
+                self._files.add(filename)
+                with open(filename) as f:
+                    data = json.load(f)
+            except IOError:
+                pass
+            self._dict[key] = data
+
+        if self._dict[key] is None:
+            raise KeyError("'%s'" % key)
+        return self._dict[key]
+
+    def __setitem__(self, key, value):
+        self._dict[key] = value
+
+    def get(self, key, default=None):
+        return self[key] if key in self else default
+
+    def __contains__(self, key):
+        try:
+            return self[key] is not None
+        except KeyError:
+            return False
+
+    def iteritems(self):
+        existing_files = self._load_config_track()
+        for f in existing_files:
+            # The track file contains filenames, and the basename is the
+            # variable name.
+            var = mozpath.basename(f)
+            yield var, self[var]
+
+
+class PartialConfigEnvironment(object):
+    """Allows access to individual config.status items via config.statusd/* files.
+
+    This class is similar to the full ConfigEnvironment, which uses
+    config.status, except this allows access and tracks dependencies to
+    individual configure values. It is intended to be used during the build
+    process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and
+    anything else that may need to access specific substs or defines.
+
+    Creating a PartialConfigEnvironment requires only the topobjdir, which is
+    needed to distinguish between the top-level environment and the js/src
+    environment.
+
+    The PartialConfigEnvironment automatically defines one additional subst variable
+    from all the defines not appearing in non_global_defines:
+      - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
+        preprocessor command lines. The order in which defines were given
+        when creating the ConfigEnvironment is preserved.
+
+    and one additional define from all the defines as a dictionary:
+      - ALLDEFINES contains all of the global defines as a dictionary. This is
+      intended to be used instead of the defines structure from config.status so
+      that scripts can depend directly on its value.
+    """
+    def __init__(self, topobjdir):
+        config_statusd = mozpath.join(topobjdir, 'config.statusd')
+        self.substs = PartialConfigDict(config_statusd, 'substs', environ_override=True)
+        self.defines = PartialConfigDict(config_statusd, 'defines')
+        self.topobjdir = topobjdir
+
+    def write_vars(self, config):
+        substs = config['substs'].copy()
+        defines = config['defines'].copy()
+
+        global_defines = [
+            name for name in config['defines']
+            if name not in config['non_global_defines']
+        ]
+        acdefines = ' '.join(['-D%s=%s' % (name,
+            shell_quote(config['defines'][name]).replace('$', '$$'))
+            for name in sorted(global_defines)])
+        substs['ACDEFINES'] = acdefines
+
+        all_defines = OrderedDict()
+        for k in global_defines:
+            all_defines[k] = config['defines'][k]
+        defines['ALLDEFINES'] = all_defines
+
+        self.substs._fill_group(substs)
+        self.defines._fill_group(defines)
+
+    def get_dependencies(self):
+        return ['$(wildcard %s)' % f for f in self.substs._files | self.defines._files]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py
@@ -0,0 +1,162 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import unittest
+from mozunit import main
+from tempfile import mkdtemp
+from shutil import rmtree
+
+import mozpack.path as mozpath
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
+
+config = {
+    'defines': {
+        'MOZ_FOO': '1',
+        'MOZ_BAR': '2',
+        'MOZ_NON_GLOBAL': '3',
+    },
+    'substs': {
+        'MOZ_SUBST_1': '1',
+        'MOZ_SUBST_2': '2',
+        'CPP': 'cpp',
+    },
+    'non_global_defines': [
+        'MOZ_NON_GLOBAL',
+    ],
+}
+
+
+class TestPartial(unittest.TestCase):
+    def setUp(self):
+        self._old_env = dict(os.environ)
+
+    def tearDown(self):
+        os.environ.clear()
+        os.environ.update(self._old_env)
+
+    def _objdir(self):
+        objdir = mkdtemp()
+        self.addCleanup(rmtree, objdir)
+        return objdir
+
+    def test_auto_substs(self):
+        '''Test the automatically set values of ACDEFINES, and ALLDEFINES
+        '''
+        env = PartialConfigEnvironment(self._objdir())
+        env.write_vars(config)
+        self.assertEqual(env.substs['ACDEFINES'], '-DMOZ_BAR=2 -DMOZ_FOO=1')
+        self.assertEqual(env.defines['ALLDEFINES'], {
+            'MOZ_BAR': '2',
+            'MOZ_FOO': '1',
+        })
+
+    def test_remove_subst(self):
+        '''Test removing a subst from the config. The file should be overwritten with 'None'
+        '''
+        env = PartialConfigEnvironment(self._objdir())
+        path = mozpath.join(env.topobjdir, 'config.statusd', 'substs', 'MYSUBST')
+        myconfig = config.copy()
+        env.write_vars(myconfig)
+        with self.assertRaises(KeyError):
+            x = env.substs['MYSUBST']
+        self.assertFalse(os.path.exists(path))
+
+        myconfig['substs']['MYSUBST'] = 'new'
+        env.write_vars(myconfig)
+
+        self.assertEqual(env.substs['MYSUBST'], 'new')
+        self.assertTrue(os.path.exists(path))
+
+        del myconfig['substs']['MYSUBST']
+        env.write_vars(myconfig)
+        with self.assertRaises(KeyError):
+            x = env.substs['MYSUBST']
+        # Now that the subst is gone, the file still needs to be present so that
+        # make can update dependencies correctly. Overwriting the file with
+        # 'None' is the same as deleting it as far as the
+        # PartialConfigEnvironment is concerned, but make can't track a
+        # dependency on a file that doesn't exist.
+        self.assertTrue(os.path.exists(path))
+
+    def _assert_deps(self, env, deps):
+        deps = sorted(['$(wildcard %s)' % (mozpath.join(env.topobjdir, 'config.statusd', d)) for d in deps])
+        self.assertEqual(sorted(env.get_dependencies()), deps)
+
+    def test_dependencies(self):
+        '''Test getting dependencies on defines and substs.
+        '''
+        env = PartialConfigEnvironment(self._objdir())
+        env.write_vars(config)
+        self._assert_deps(env, [])
+
+        self.assertEqual(env.defines['MOZ_FOO'], '1')
+        self._assert_deps(env, ['defines/MOZ_FOO'])
+
+        self.assertEqual(env.defines['MOZ_BAR'], '2')
+        self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR'])
+
+        # Getting a define again shouldn't add a redundant dependency
+        self.assertEqual(env.defines['MOZ_FOO'], '1')
+        self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR'])
+
+        self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
+        self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1'])
+
+        with self.assertRaises(KeyError):
+            x = env.substs['NON_EXISTENT']
+        self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1', 'substs/NON_EXISTENT'])
+        self.assertEqual(env.substs.get('NON_EXISTENT'), None)
+
+    def test_set_subst(self):
+        '''Test setting a subst
+        '''
+        env = PartialConfigEnvironment(self._objdir())
+        env.write_vars(config)
+
+        self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
+        env.substs['MOZ_SUBST_1'] = 'updated'
+        self.assertEqual(env.substs['MOZ_SUBST_1'], 'updated')
+
+        # A new environment should pull the result from the file again.
+        newenv = PartialConfigEnvironment(env.topobjdir)
+        self.assertEqual(newenv.substs['MOZ_SUBST_1'], '1')
+
+    def test_env_override(self):
+        '''Test overriding a subst with an environment variable
+        '''
+        env = PartialConfigEnvironment(self._objdir())
+        env.write_vars(config)
+
+        self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
+        self.assertEqual(env.substs['CPP'], 'cpp')
+
+        # Reset the environment and set some environment variables.
+        env = PartialConfigEnvironment(env.topobjdir)
+        os.environ['MOZ_SUBST_1'] = 'subst 1 environ'
+        os.environ['CPP'] = 'cpp environ'
+
+        # The MOZ_SUBST_1 should be overridden by the environment, while CPP is
+        # a special variable and should not.
+        self.assertEqual(env.substs['MOZ_SUBST_1'], 'subst 1 environ')
+        self.assertEqual(env.substs['CPP'], 'cpp')
+
+    def test_update(self):
+        '''Test calling update on the substs or defines pseudo dicts
+        '''
+        env = PartialConfigEnvironment(self._objdir())
+        env.write_vars(config)
+
+        mysubsts = {'NEW': 'new'}
+        mysubsts.update(env.substs.iteritems())
+        self.assertEqual(mysubsts['NEW'], 'new')
+        self.assertEqual(mysubsts['CPP'], 'cpp')
+
+        mydefines = {'DEBUG': '1'}
+        mydefines.update(env.defines.iteritems())
+        self.assertEqual(mydefines['DEBUG'], '1')
+        self.assertEqual(mydefines['MOZ_FOO'], '1')
+
+if __name__ == "__main__":
+    main()
--- a/python/mozbuild/mozbuild/test/python.ini
+++ b/python/mozbuild/mozbuild/test/python.ini
@@ -1,13 +1,14 @@
 [action/test_buildlist.py]
 [action/test_generate_browsersearch.py]
 [action/test_package_fennec_apk.py]
 [backend/test_build.py]
 [backend/test_configenvironment.py]
+[backend/test_partialconfigenvironment.py]
 [backend/test_recursivemake.py]
 [backend/test_test_manifest.py]
 [backend/test_visualstudio.py]
 [codecoverage/test_lcov_rewrite.py]
 [compilation/test_warnings.py]
 [configure/lint.py]
 [configure/test_checks_configure.py]
 [configure/test_compile_checks.py]