Bug 1402012 - Update buildconfig.py to use PartialConfigEnvironment; r=glandium
authorMike Shal <mshal@mozilla.com>
Thu, 24 Aug 2017 22:52:01 -0400
changeset 434519 cb773c661e0ca0bf297e977343076bef34411523
parent 434518 22ebbb5d30d470415a729d5b4e9764998023cf07
child 434520 a1903b0f0ead5b2952dd4a1b6a41b20a65a07f44
push id8114
push userjlorenzo@mozilla.com
push dateThu, 02 Nov 2017 16:33:21 +0000
treeherdermozilla-beta@73e0d89a540f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1402012
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1402012 - Update buildconfig.py to use PartialConfigEnvironment; r=glandium By using the PartialConfigEnvironment, the clients of buildconfig will depend on config.statusd/ files instead of config.status directly. Clients can access substs and defines using buildconfig.substs['FOO'] or buildconfig.defines['BAR'], and then collect file-level dependencies for make using buildconfig.get_dependencies(). All GENERATED_FILES rules already make use of this because file_generate.py automatically includes these dependencies (along with all python modules loaded). As a result of this commit, re-running configure will no longer cause the world to be rebuilt. Although config.status is updated, no build steps use config.status directly and instead depend on values in config.statusd/, which are written with FileAvoidWrite. Since those files are not official targets according to the make backend, make won't try to continually rebuild the backend when those files are out of date. And since they are FileAvoidWrite, make will only re-run dependent steps if the actual configure value has changed. As a result of using JSON to load data from the config.statusd directory, substs can be unicode (instead of a bare string type). generate_certdata.py converts the subst manually to a string so the value can be exported to the environment without issue on Windows. Additionally, patching the buildconfig.substs dict no longer works, so the unit-symbolstore.py test was modified to patch the underlying buildconfig.substs._dict instead. The other files that needed to be modified make use of all the defines for the preprocessor. Those that are used during 'mach build' now use buildconfig.defines['ALLDEFINES'], which maps to a special FileAvoidWrite file generated for the PartialConfigEnvironment. MozReview-Commit-ID: 2pJ4s3TVeS8
build/buildconfig.py
js/src/builtin/embedjs.py
mobile/android/base/generate_build_config.py
python/mozbuild/mozbuild/action/file_generate.py
python/mozbuild/mozbuild/action/generate_symbols_file.py
security/generate_certdata.py
toolkit/crashreporter/tools/unit-symbolstore.py
toolkit/mozapps/installer/find-dupes.py
toolkit/mozapps/installer/packager.py
--- a/build/buildconfig.py
+++ b/build/buildconfig.py
@@ -1,20 +1,19 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import sys
 from mozbuild.base import MozbuildObject
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
 
 config = MozbuildObject.from_environment()
+partial_config = PartialConfigEnvironment(config.topobjdir)
 
-for var in ('topsrcdir', 'topobjdir', 'defines', 'non_global_defines',
-            'substs'):
+for var in ('topsrcdir', 'topobjdir'):
     value = getattr(config, var)
     setattr(sys.modules[__name__], var, value)
 
-substs = dict(substs)
-
-for var in os.environ:
-    if var not in ('CPP', 'CXXCPP', 'SHELL') and var in substs:
-        substs[var] = os.environ[var]
+for var in ('defines', 'substs', 'get_dependencies'):
+    value = getattr(partial_config, var)
+    setattr(sys.modules[__name__], var, value)
--- a/js/src/builtin/embedjs.py
+++ b/js/src/builtin/embedjs.py
@@ -127,18 +127,17 @@ def messages(jsmsg):
       defines.append("#define %s %i" % (match.group(1), len(defines)))
     else:
       # Make sure that MSG_DEF isn't preceded by whitespace
       assert not line.strip().startswith("MSG_DEF")
   return '\n'.join(defines)
 
 def get_config_defines(buildconfig):
   # Collect defines equivalent to ACDEFINES and add MOZ_DEBUG_DEFINES.
-  env = {key: value for key, value in buildconfig.defines.iteritems()
-         if key not in buildconfig.non_global_defines}
+  env = buildconfig.defines['ALLDEFINES']
   for define in buildconfig.substs['MOZ_DEBUG_DEFINES']:
     env[define] = 1
   return env
 
 def process_inputs(namespace, c_out, msg_file, inputs):
   deps = [path for path in inputs if path.endswith(".h") or path.endswith(".h.js")]
   sources = [path for path in inputs if path.endswith(".js") and not path.endswith(".h.js")]
   assert len(deps) + len(sources) == len(inputs)
--- a/mobile/android/base/generate_build_config.py
+++ b/mobile/android/base/generate_build_config.py
@@ -28,18 +28,18 @@ import sys
 import buildconfig
 
 from mozbuild import preprocessor
 from mozbuild.android_version_code import android_version_code
 
 
 def _defines():
     CONFIG = defaultdict(lambda: None)
-    CONFIG.update(buildconfig.substs)
-    DEFINES = dict(buildconfig.defines)
+    CONFIG.update(buildconfig.substs.iteritems())
+    DEFINES = dict(buildconfig.defines.iteritems())
 
     for var in ('MOZ_ANDROID_ACTIVITY_STREAM'
                 'MOZ_ANDROID_ANR_REPORTER',
                 'MOZ_ANDROID_BEAM',
                 'MOZ_ANDROID_DOWNLOADS_INTEGRATION',
                 'MOZ_ANDROID_DOWNLOAD_CONTENT_SERVICE',
                 'MOZ_ANDROID_EXCLUDE_FONTS',
                 'MOZ_ANDROID_GCM',
--- a/python/mozbuild/mozbuild/action/file_generate.py
+++ b/python/mozbuild/mozbuild/action/file_generate.py
@@ -80,16 +80,20 @@ def main(argv):
                 deps = set()
 
             # Only write out the dependencies if the script was successful
             if not ret:
                 # Add dependencies on any python modules that were imported by
                 # the script.
                 deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
                                                  buildconfig.topobjdir))
+                # Add dependencies on any buildconfig items that were accessed
+                # by the script.
+                deps |= set(buildconfig.get_dependencies())
+
                 mk = Makefile()
                 mk.create_rule([args.output_file]).add_dependencies(deps)
                 with FileAvoidWrite(args.dep_file) as dep_file:
                     mk.dump(dep_file)
         # Even when our file's contents haven't changed, we want to update
         # the file's mtime so make knows this target isn't still older than
         # whatever prerequisite caused it to be built this time around.
         try:
--- a/python/mozbuild/mozbuild/action/generate_symbols_file.py
+++ b/python/mozbuild/mozbuild/action/generate_symbols_file.py
@@ -17,17 +17,17 @@ def generate_symbols_file(output, *args)
     parser = argparse.ArgumentParser()
     parser.add_argument('input')
     parser.add_argument('-D', action=DefinesAction)
     parser.add_argument('-U', action='append', default=[])
     args = parser.parse_args(args)
     input = os.path.abspath(args.input)
 
     pp = Preprocessor()
-    pp.context.update(buildconfig.defines)
+    pp.context.update(buildconfig.defines['ALLDEFINES'])
     if args.D:
         pp.context.update(args.D)
     for undefine in args.U:
         if undefine in pp.context:
             del pp.context[undefine]
     # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
     if buildconfig.substs['MOZ_DEBUG']:
         pp.context['DEBUG'] = '1'
--- a/security/generate_certdata.py
+++ b/security/generate_certdata.py
@@ -4,12 +4,12 @@
 # and moz.build `GENERATED_FILES` semantics.
 
 import buildconfig
 import os
 import subprocess
 
 def main(output, *inputs):
     env=dict(os.environ)
-    env['PERL'] = buildconfig.substs['PERL']
+    env['PERL'] = str(buildconfig.substs['PERL'])
     output.write(subprocess.check_output([buildconfig.substs['PYTHON'],
                  inputs[0], inputs[2]], env=env))
     return None
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -145,17 +145,17 @@ class TestCopyDebug(HelperMixin, unittes
         d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
                                                   symbol_path=self.symbol_dir,
                                                   copy_debug=True,
                                                   archs="abc xyz")
         d.CopyDebug = mock_copy_debug
         d.Process(os.path.join(self.test_dir, add_extension(["foo"])[0]))
         self.assertEqual(1, len(copied))
 
-    @patch.dict('buildconfig.substs', {'MAKECAB': 'makecab'})
+    @patch.dict('buildconfig.substs._dict', {'MAKECAB': 'makecab'})
     def test_copy_debug_copies_binaries(self):
         """
         Test that CopyDebug copies binaries as well on Windows.
         """
         test_file = os.path.join(self.test_dir, 'foo.dll')
         write_dll(test_file)
         code_file = 'foo.dll'
         code_id = 'abc123'
--- a/toolkit/mozapps/installer/find-dupes.py
+++ b/toolkit/mozapps/installer/find-dupes.py
@@ -97,17 +97,17 @@ def main():
     parser.add_argument('directory',
                         help='The directory to check for duplicates in')
 
     args = parser.parse_args()
 
     allowed_dupes = []
     for filename in args.dupes_files:
         pp = Preprocessor()
-        pp.context.update(buildconfig.defines)
+        pp.context.update(buildconfig.defines['ALLDEFINES'])
         if args.D:
             pp.context.update(args.D)
         for undefine in args.U:
             if undefine in pp.context:
                 del pp.context[undefine]
         pp.out = StringIO()
         pp.do_filter('substitution')
         pp.do_include(filename)
--- a/toolkit/mozapps/installer/packager.py
+++ b/toolkit/mozapps/installer/packager.py
@@ -221,17 +221,17 @@ def main():
                         help='Manifest file name')
     parser.add_argument('source', help='Source directory')
     parser.add_argument('destination', help='Destination directory')
     parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
                         default=[],
                         help='Extra files not to be considered as resources')
     args = parser.parse_args()
 
-    defines = dict(buildconfig.defines)
+    defines = dict(buildconfig.defines['ALLDEFINES'])
     if args.ignore_errors:
         errors.ignore_errors()
 
     if args.defines:
         for name, value in [split_define(d) for d in args.defines]:
             defines[name] = value
 
     compress = {