Bug 1398897 - Move includes to computed flags. draft
authorChris Manchester <cmanchester@mozilla.com>
Fri, 15 Sep 2017 11:49:07 -0700
changeset 665609 6ff1deb6aeee3a035b45e514f9a96913d52bb1c4
parent 665608 38622720f4ffdd657869624995e8ff2aee42adb4
child 665610 28ae08fe3db5bbb20c452a33e4118ed4ea28bd67
push id80119
push userbmo:cmanchester@mozilla.com
push dateFri, 15 Sep 2017 18:49:23 +0000
bugs1398897
milestone57.0a1
Bug 1398897 - Move includes to computed flags. MozReview-Commit-ID: Ec0wZfoV63B
config/config.mk
config/rules.mk
python/mozbuild/mozbuild/compilation/database.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/frontend/data/compile-includes/moz.build
python/mozbuild/mozbuild/test/frontend/data/compile-includes/subdir/header.h
python/mozbuild/mozbuild/test/frontend/data/compile-includes/test1.c
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/config/config.mk
+++ b/config/config.mk
@@ -319,18 +319,18 @@ endif # CLANG_CL
 
 # Use warnings-as-errors if ALLOW_COMPILER_WARNINGS is not set to 1 (which
 # includes the case where it's undefined).
 ifneq (1,$(ALLOW_COMPILER_WARNINGS))
 CXXFLAGS += $(WARNINGS_AS_ERRORS)
 CFLAGS   += $(WARNINGS_AS_ERRORS)
 endif # ALLOW_COMPILER_WARNINGS
 
-COMPILE_CFLAGS	= $(COMPUTED_CFLAGS) $(INCLUDES) $(OS_INCLUDES) $(DSO_CFLAGS) $(DSO_PIC_CFLAGS) $(RTL_FLAGS) $(OS_COMPILE_CFLAGS) $(_DEPEND_CFLAGS) $(CFLAGS) $(MOZBUILD_CFLAGS) $(MK_COMPILE_DEFINES)
-COMPILE_CXXFLAGS = $(COMPUTED_CXXFLAGS) $(INCLUDES) $(OS_INCLUDES) $(DSO_CFLAGS) $(DSO_PIC_CFLAGS) $(RTL_FLAGS) $(OS_COMPILE_CXXFLAGS) $(_DEPEND_CFLAGS) $(CXXFLAGS) $(MOZBUILD_CXXFLAGS) $(MK_COMPILE_DEFINES)
+COMPILE_CFLAGS	= $(COMPUTED_CFLAGS) $(OS_INCLUDES) $(DSO_CFLAGS) $(DSO_PIC_CFLAGS) $(RTL_FLAGS) $(OS_COMPILE_CFLAGS) $(_DEPEND_CFLAGS) $(CFLAGS) $(MOZBUILD_CFLAGS) $(MK_COMPILE_DEFINES)
+COMPILE_CXXFLAGS = $(COMPUTED_CXXFLAGS) $(OS_INCLUDES) $(DSO_CFLAGS) $(DSO_PIC_CFLAGS) $(RTL_FLAGS) $(OS_COMPILE_CXXFLAGS) $(_DEPEND_CFLAGS) $(CXXFLAGS) $(MOZBUILD_CXXFLAGS) $(MK_COMPILE_DEFINES)
 COMPILE_CMFLAGS = $(OS_COMPILE_CMFLAGS) $(MOZBUILD_CMFLAGS)
 COMPILE_CMMFLAGS = $(OS_COMPILE_CMMFLAGS) $(MOZBUILD_CMMFLAGS)
 ASFLAGS += $(MOZBUILD_ASFLAGS)
 
 ifndef CROSS_COMPILE
 HOST_CFLAGS += $(RTL_FLAGS)
 endif
 
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -63,17 +63,16 @@ testxpcobjdir = $(DEPTH)/_tests/xpcshell
 ifdef ENABLE_TESTS
 ifdef CPP_UNIT_TESTS
 ifdef COMPILE_ENVIRONMENT
 
 # Compile the tests to $(DIST)/bin.  Make lots of niceties available by default
 # through TestHarness.h, by modifying the list of includes and the libs against
 # which stuff links.
 SIMPLE_PROGRAMS += $(CPP_UNIT_TESTS)
-INCLUDES += -I$(ABS_DIST)/include/testing
 
 ifndef MOZ_PROFILE_GENERATE
 CPP_UNIT_TESTS_FILES = $(CPP_UNIT_TESTS)
 CPP_UNIT_TESTS_DEST = $(DIST)/cppunittests
 CPP_UNIT_TESTS_TARGET = target
 INSTALL_TARGETS += CPP_UNIT_TESTS
 endif
 
--- a/python/mozbuild/mozbuild/compilation/database.py
+++ b/python/mozbuild/mozbuild/compilation/database.py
@@ -39,23 +39,19 @@ class CompileDBBackend(CommonBackend):
 
         # The database we're going to dump out to.
         self._db = OrderedDict()
 
         # The cache for per-directory flags
         self._flags = {}
 
         self._envs = {}
-        self._includes = defaultdict(list)
         self._local_flags = defaultdict(dict)
         self._per_source_flags = defaultdict(list)
-        self._extra_includes = defaultdict(list)
         self._gyp_dirs = set()
-        self._dist_include_testing = '-I%s' % mozpath.join(
-            self.environment.topobjdir, 'dist', 'include', 'testing')
 
     def consume_object(self, obj):
         # Those are difficult directories, that will be handled later.
         if obj.relativedir in (
                 'build/unix/elfhack',
                 'build/unix/elfhack/inject',
                 'build/clang-plugin',
                 'build/clang-plugin/tests',
@@ -75,27 +71,16 @@ class CompileDBBackend(CommonBackend):
                     self._local_flags[obj.objdir][var] = value
 
         elif isinstance(obj, (Sources, GeneratedSources)):
             # For other sources, include each source file.
             for f in obj.files:
                 self._build_db_line(obj.objdir, obj.relativedir, obj.config, f,
                                     obj.canonical_suffix)
 
-        elif isinstance(obj, LocalInclude):
-            self._includes[obj.objdir].append('-I%s' % mozpath.normpath(
-                obj.path.full_path))
-
-        elif isinstance(obj, Linkable):
-            if isinstance(obj, SimpleProgram) and obj.is_unit_test:
-                if (self._dist_include_testing not in
-                        self._extra_includes[obj.objdir]):
-                    self._extra_includes[obj.objdir].append(
-                        self._dist_include_testing)
-
         elif isinstance(obj, VariablePassthru):
             if obj.variables.get('IS_GYP_DIR'):
                 self._gyp_dirs.add(obj.objdir)
             for var in ('MOZBUILD_CFLAGS', 'MOZBUILD_CXXFLAGS',
                         'MOZBUILD_CMFLAGS', 'MOZBUILD_CMMFLAGS',
                         'RTL_FLAGS'):
                 if var in obj.variables:
                     self._local_flags[obj.objdir][var] = obj.variables[var]
@@ -119,32 +104,31 @@ class CompileDBBackend(CommonBackend):
 
         for (directory, filename, unified), cmd in self._db.iteritems():
             env = self._envs[directory]
             cmd = list(cmd)
             if unified is None:
                 cmd.append(filename)
             else:
                 cmd.append(unified)
-            local_extra = list(self._extra_includes[directory])
+            os_includes = []
             if directory not in self._gyp_dirs:
                 for var in (
                     'NSPR_CFLAGS',
                     'NSS_CFLAGS',
                     'MOZ_JPEG_CFLAGS',
                     'MOZ_PNG_CFLAGS',
                     'MOZ_ZLIB_CFLAGS',
                     'MOZ_PIXMAN_CFLAGS',
                 ):
                     f = env.substs.get(var)
                     if f:
-                        local_extra.extend(f)
+                        os_includes.extend(f)
             variables = {
-                'LOCAL_INCLUDES': self._includes[directory],
-                'EXTRA_INCLUDES': local_extra,
+                'OS_INCLUDES': os_includes,
                 'DIST': mozpath.join(env.topobjdir, 'dist'),
                 'DEPTH': env.topobjdir,
                 'MOZILLA_DIR': env.topsrcdir,
                 'topsrcdir': env.topsrcdir,
                 'topobjdir': env.topobjdir,
             }
             variables.update(self._local_flags[directory])
             c = []
@@ -226,21 +210,17 @@ class CompileDBBackend(CommonBackend):
                 return
             if isinstance(value, types.StringTypes):
                 value = value.split()
             db.extend(value)
 
         db.append('$(COMPUTED_%s)' % self.CFLAGS[canonical_suffix])
 
         db.extend((
-            '-I%s' % mozpath.join(cenv.topsrcdir, reldir),
-            '-I%s' % objdir,
-            '$(LOCAL_INCLUDES)',
-            '-I%s/dist/include' % cenv.topobjdir,
-            '$(EXTRA_INCLUDES)',
+            '$(OS_INCLUDES)',
         ))
         append_var('DSO_CFLAGS')
         append_var('DSO_PIC_CFLAGS')
         if canonical_suffix in ('.c', '.cpp'):
             db.append('$(RTL_FLAGS)')
         append_var('OS_COMPILE_%s' % self.CFLAGS[canonical_suffix])
         append_var('OS_CPPFLAGS')
         append_var('OS_%s' % self.CFLAGS[canonical_suffix])
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -294,22 +294,29 @@ class InitializedDefines(ContextDerivedV
         for define in context.config.substs.get('MOZ_DEBUG_DEFINES', ()):
             self[define] = 1
         if value:
             self.update(value)
 
 
 class CompileFlags(ContextDerivedValue, dict):
     def __init__(self, context):
+        main_src_dir = mozpath.dirname(context.main_path)
+
         self.flag_variables = (
             ('STL', context.config.substs.get('STL_FLAGS'), ('CXXFLAGS',)),
             ('VISIBILITY', context.config.substs.get('VISIBILITY_FLAGS'),
              ('CXXFLAGS', 'CFLAGS')),
             ('DEFINES', None, ('CXXFLAGS', 'CFLAGS')),
             ('LIBRARY_DEFINES', None, ('CXXFLAGS', 'CFLAGS')),
+            ('BASE_INCLUDES', ['-I%s' % main_src_dir, '-I%s' % context.objdir],
+             ('CXXFLAGS', 'CFLAGS')),
+            ('LOCAL_INCLUDES', None, ('CXXFLAGS', 'CFLAGS')),
+            ('EXTRA_INCLUDES', ['-I%s/dist/include' % context.config.topobjdir],
+             ('CXXFLAGS', 'CFLAGS')),
         )
         self._known_keys = set(k for k, v, _ in self.flag_variables)
 
         # Providing defaults here doesn't play well with multiple templates
         # modifying COMPILE_FLAGS from the same moz.build, because the merge
         # done after the template runs can't tell which values coming from
         # a template were set and which were provided as defaults.
         template_name = getattr(context, 'template', None)
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -1016,23 +1016,34 @@ class TreeMetadataEmitter(LoggingMixin):
             ('TEST_WEBIDL_FILES', TestWebIDLFile),
             ('WEBIDL_FILES', WebIDLFile),
             ('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface),
         ]
         for context_var, klass in simple_lists:
             for name in context.get(context_var, []):
                 yield klass(context, name)
 
+        local_includes = []
         for local_include in context.get('LOCAL_INCLUDES', []):
             if (not isinstance(local_include, ObjDirPath) and
                     not os.path.exists(local_include.full_path)):
                 raise SandboxValidationError('Path specified in LOCAL_INCLUDES '
                     'does not exist: %s (resolved to %s)' % (local_include,
                     local_include.full_path), context)
-            yield LocalInclude(context, local_include)
+            include_obj = LocalInclude(context, local_include)
+            local_includes.append(include_obj.path.full_path)
+            yield include_obj
+
+        computed_flags.resolve_flags('LOCAL_INCLUDES', ['-I%s' % p for p in local_includes])
+
+        if len(context['CPP_UNIT_TESTS']):
+            # TODO: Move this to the CppUnitTests template.
+            dist_include_testing = mozpath.join(self.config.topobjdir,
+                                                'dist', 'include', 'testing')
+            computed_flags.flags['EXTRA_INCLUDES'].append('-I%s' % dist_include_testing)
 
         for obj in self._handle_linkables(context, passthru, generated_files):
             yield obj
 
         generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()])
 
         components = []
         for var, cls in (
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/compile-includes/moz.build
@@ -0,0 +1,13 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+@template
+def Library(name):
+    '''Template for libraries.'''
+    LIBRARY_NAME = name
+
+Library('dummy')
+
+UNIFIED_SOURCES += ['test1.c']
+
+LOCAL_INCLUDES += ['subdir']
\ No newline at end of file
new file mode 100644
new file mode 100644
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -252,16 +252,28 @@ class TestEmitterBasic(unittest.TestCase
                          ['-DMOZ_TEST_DEFINE'])
 
     def test_resolved_flags_error(self):
         reader = self.reader('resolved-flags-error')
         with self.assertRaisesRegexp(BuildReaderError,
             "`DEFINES` may not be set in COMPILE_FLAGS from moz.build"):
             self.read_topsrcdir(reader)
 
+    def test_includes_in_flags(self):
+        reader = self.reader('compile-includes')
+        defines, sources, lib, flags = self.read_topsrcdir(reader)
+        self.assertIsInstance(flags, ComputedFlags)
+        self.assertEqual(flags.flags['BASE_INCLUDES'],
+                         ['-I%s' % reader.config.topsrcdir,
+                          '-I%s' % reader.config.topobjdir])
+        self.assertEqual(flags.flags['EXTRA_INCLUDES'],
+                         ['-I%s/dist/include' % reader.config.topobjdir])
+        self.assertEqual(flags.flags['LOCAL_INCLUDES'],
+                         ['-I%s/subdir' % reader.config.topsrcdir])
+
     def test_use_yasm(self):
         # When yasm is not available, this should raise.
         reader = self.reader('use-yasm')
         with self.assertRaisesRegexp(SandboxValidationError,
             'yasm is not available'):
             self.read_topsrcdir(reader)
 
         # When yasm is available, this should work.