Bug 1207893 - Change how we track object consumption from the build backend. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 25 Sep 2015 07:03:09 +0900
changeset 264884 75c3e053bcfa70f2cf6dd0df22458d916c41fd6b
parent 264883 af6ea1464daa5bed510b6ce8f319a389fb590a6e
child 264885 55af918cafff6bfdfaa6a8c665e3a2bd240b52f3
push id29450
push usercbook@mozilla.com
push dateTue, 29 Sep 2015 10:00:39 +0000
treeherdermozilla-central@acdb22976ff8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1207893
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1207893 - Change how we track object consumption from the build backend. r=gps Currently, we set a flag on each object to know whether it has been consumed by the backend. This doesn't work nicely when multiple backends try to consume the same objects.
python/mozbuild/mozbuild/backend/android_eclipse.py
python/mozbuild/mozbuild/backend/base.py
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/cpp_eclipse.py
python/mozbuild/mozbuild/backend/fastermake.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/backend/visualstudio.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/python/mozbuild/mozbuild/backend/android_eclipse.py
+++ b/python/mozbuild/mozbuild/backend/android_eclipse.py
@@ -50,31 +50,29 @@ class AndroidEclipseBackend(CommonBacken
             created=self._created_count,
             updated=self._updated_count,
         )
 
     def consume_object(self, obj):
         """Write out Android Eclipse project files."""
 
         if not isinstance(obj, ContextDerived):
-            return
-
-        CommonBackend.consume_object(self, obj)
+            return False
 
-        # If CommonBackend acknowledged the object, we're done with it.
-        if obj._ack:
-            return
+        if CommonBackend.consume_object(self, obj):
+            # If CommonBackend acknowledged the object, we're done with it.
+            return True
 
-        # We don't want to handle most things, so we just acknowledge all objects...
-        obj.ack()
-
-        # ... and handle the one case we care about specially.
+        # Handle the one case we care about specially.
         if isinstance(obj, ContextWrapped) and isinstance(obj.wrapped, AndroidEclipseProjectData):
             self._process_android_eclipse_project_data(obj.wrapped, obj.srcdir, obj.objdir)
 
+        # We don't want to handle most things, so we just acknowledge all objects
+        return True
+
     def consume_finished(self):
         """The common backend handles WebIDL and test files. We don't handle
         these, so we don't call our superclass.
         """
 
     def _Element_for_classpathentry(self, cpe):
         """Turn a ClassPathEntry into an XML Element, like one of:
         <classpathentry including="**/*.java" kind="src" path="preprocessed"/>
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -112,17 +112,18 @@ class BuildBackend(LoggingMixin):
         frontend output and does something with it.
 
         Child classes are not expected to implement this method. Instead, the
         base class consumes objects and calls methods (possibly) implemented by
         child classes.
         """
         for obj in objs:
             obj_start = time.time()
-            self.consume_object(obj)
+            if not self.consume_object(obj):
+                raise Exception('Unhandled object of type %s' % type(obj))
             self._execution_time += time.time() - obj_start
 
             if isinstance(obj, ContextDerived):
                 self.backend_input_files |= obj.context_all_paths
 
         # Pull in all loaded Python as dependencies so any Python changes that
         # could influence our output result in a rescan.
         self.backend_input_files |= set(iter_modules_in_path(
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -201,17 +201,17 @@ class CommonBackend(BuildBackend):
 
         elif isinstance(obj, XPIDLFile):
             self._idl_manager.register_idl(obj)
 
         elif isinstance(obj, ConfigFileSubstitution):
             # Do not handle ConfigFileSubstitution for Makefiles. Leave that
             # to other
             if mozpath.basename(obj.output_path) == 'Makefile':
-                return
+                return False
             with self._get_preprocessor(obj) as pp:
                 pp.do_include(obj.input_path)
             self.backend_input_files.add(obj.input_path)
 
         elif isinstance(obj, HeaderFileSubstitution):
             self._create_config_header(obj)
             self.backend_input_files.add(obj.input_path)
 
@@ -246,19 +246,19 @@ class CommonBackend(BuildBackend):
             self._ipdl_sources.add(mozpath.join(obj.srcdir, obj.basename))
 
         elif isinstance(obj, UnifiedSources):
             if obj.have_unified_mapping:
                 self._write_unified_files(obj.unified_source_mapping, obj.objdir)
             if hasattr(self, '_process_unified_sources'):
                 self._process_unified_sources(obj)
         else:
-            return
+            return False
 
-        obj.ack()
+        return True
 
     def consume_finished(self):
         if len(self._idl_manager.idls):
             self._handle_idl_manager(self._idl_manager)
 
         self._handle_webidl_collection(self._webidls)
 
         sorted_ipdl_sources = list(sorted(self._ipdl_sources))
--- a/python/mozbuild/mozbuild/backend/cpp_eclipse.py
+++ b/python/mozbuild/mozbuild/backend/cpp_eclipse.py
@@ -60,25 +60,25 @@ class CppEclipseBackend(CommonBackend):
         # Eclipse doesn't support having the workspace inside the srcdir.
         # Since most people have their objdir inside their srcdir it's easier
         # and more consistent to just put the workspace along side the srcdir
         srcdir_parent = os.path.dirname(topsrcdir)
         workspace_dirname = "eclipse_" + os.path.basename(topobjdir)
         return os.path.join(srcdir_parent, workspace_dirname)
 
     def consume_object(self, obj):
-        obj.ack()
-
         reldir = getattr(obj, 'relativedir', None)
 
         # Note that unlike VS, Eclipse' indexer seem to crawl the headers and
         # isn't picky about the local includes.
         if isinstance(obj, Defines):
             self._paths_to_defines.setdefault(reldir, {}).update(obj.defines)
 
+        return True
+
     def consume_finished(self):
         settings_dir = os.path.join(self._project_dir, '.settings')
         launch_dir = os.path.join(self._project_dir, 'RunConfigurations')
         workspace_settings_dir = os.path.join(self._workspace_dir, '.metadata/.plugins/org.eclipse.core.runtime/.settings')
         workspace_language_dir = os.path.join(self._workspace_dir, '.metadata/.plugins/org.eclipse.cdt.core')
 
         for dir_name in [self._project_dir, settings_dir, launch_dir, workspace_settings_dir, workspace_language_dir]:
             try:
--- a/python/mozbuild/mozbuild/backend/fastermake.py
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -34,20 +34,16 @@ class FasterMakeBackend(CommonBackend):
 
         self._preprocess_files = OrderedDict()
 
         self._manifest_entries = OrderedDefaultDict(list)
 
         self._install_manifests = OrderedDefaultDict(InstallManifest)
 
     def consume_object(self, obj):
-        # We currently ignore a lot of object types, so just acknowledge
-        # everything.
-        obj.ack()
-
         if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
             defines = self._defines.get(obj.objdir, [])
             if defines:
                 defines = list(defines.get_defines())
 
         if isinstance(obj, Defines):
             self._defines[obj.objdir] = obj
 
@@ -162,19 +158,22 @@ class FasterMakeBackend(CommonBackend):
             # We preprocess these, but they don't necessarily have preprocessor
             # directives, so tell the preprocessor to not complain about that.
             defines.append('--silence-missing-directive-warnings')
             for f in obj.files:
                 dest = mozpath.join(obj.install_target, mozpath.basename(f))
                 self._preprocess_files[dest] = (obj.srcdir, f, defines)
 
         else:
-            return
+            # We currently ignore a lot of object types, so just acknowledge
+            # everything.
+            return True
 
         self._seen_directories.add(obj.objdir)
+        return True
 
     def consume_finished(self):
         mk = Makefile()
         # Add the default rule at the very beginning.
         mk.create_rule(['default'])
         mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
         mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
 
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -423,35 +423,35 @@ class RecursiveMakeBackend(CommonBackend
                 BackendMakeFile(obj.srcdir, obj.objdir, obj.config,
                     obj.topsrcdir, self.environment.topobjdir)
         return self._backend_files[obj.objdir]
 
     def consume_object(self, obj):
         """Write out build files necessary to build with recursive make."""
 
         if not isinstance(obj, ContextDerived):
-            return
+            return False
 
         backend_file = self._get_backend_file_for(obj)
 
-        CommonBackend.consume_object(self, obj)
+        consumed = CommonBackend.consume_object(self, obj)
 
         # CommonBackend handles XPIDLFile and TestManifest, but we want to do
         # some extra things for them.
         if isinstance(obj, XPIDLFile):
             backend_file.idls.append(obj)
             backend_file.xpt_name = '%s.xpt' % obj.module
             self._idl_dirs.add(obj.relobjdir)
 
         elif isinstance(obj, TestManifest):
             self._process_test_manifest(obj, backend_file)
 
         # If CommonBackend acknowledged the object, we're done with it.
-        if obj._ack:
-            return
+        if consumed:
+            return True
 
         if isinstance(obj, DirectoryTraversal):
             self._process_directory_traversal(obj, backend_file)
         elif isinstance(obj, ConfigFileSubstitution):
             # Other ConfigFileSubstitution should have been acked by
             # CommonBackend.
             assert os.path.basename(obj.output_path) == 'Makefile'
             self._create_makefile(obj)
@@ -571,17 +571,17 @@ class RecursiveMakeBackend(CommonBackend
             # in a function named _process_camel_case_data.  At some
             # point in the future, this unwrapping process may be
             # automated.
             if isinstance(obj.wrapped, JavaJarData):
                 self._process_java_jar_data(obj.wrapped, backend_file)
             elif isinstance(obj.wrapped, AndroidEclipseProjectData):
                 self._process_android_eclipse_project_data(obj.wrapped, backend_file)
             else:
-                return
+                return False
 
         elif isinstance(obj, SharedLibrary):
             self._process_shared_library(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
 
         elif isinstance(obj, StaticLibrary):
             self._process_static_library(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
@@ -613,18 +613,19 @@ class RecursiveMakeBackend(CommonBackend
             for p in obj.paths:
                 backend_file.write('ANDROID_EXTRA_RES_DIRS += %s\n' % p.full_path)
 
         elif isinstance(obj, AndroidExtraPackages):
             for p in obj.packages:
                 backend_file.write('ANDROID_EXTRA_PACKAGES += %s\n' % p)
 
         else:
-            return
-        obj.ack()
+            return False
+
+        return True
 
     def _fill_root_mk(self):
         """
         Create two files, root.mk and root-deps.mk, the first containing
         convenience variables, and the other dependency definitions for a
         hopefully proper directory traversal.
         """
         for tier, skip in self._may_skip.items():
--- a/python/mozbuild/mozbuild/backend/visualstudio.py
+++ b/python/mozbuild/mozbuild/backend/visualstudio.py
@@ -105,19 +105,16 @@ class VisualStudioBackend(CommonBackend)
     def summary(self):
         return ExecutionSummary(
             'VisualStudio backend executed in {execution_time:.2f}s\n'
             'Generated Visual Studio solution at {path:s}',
             execution_time=self._execution_time,
             path=os.path.join(self._out_dir, 'mozilla.sln'))
 
     def consume_object(self, obj):
-        # Just acknowledge everything.
-        obj.ack()
-
         reldir = getattr(obj, 'relativedir', None)
 
         if hasattr(obj, 'config') and reldir not in self._paths_to_configs:
             self._paths_to_configs[reldir] = obj.config
 
         if isinstance(obj, Sources):
             self._add_sources(reldir, obj)
 
@@ -142,16 +139,19 @@ class VisualStudioBackend(CommonBackend)
             p = obj.path
             includes = self._paths_to_includes.setdefault(reldir, [])
 
             if p.startswith('/'):
                 includes.append(os.path.join('$(TopSrcDir)', p[1:]))
             else:
                 includes.append(os.path.join('$(TopSrcDir)', reldir, p))
 
+        # Just acknowledge everything.
+        return True
+
     def _add_sources(self, reldir, obj):
         s = self._paths_to_sources.setdefault(reldir, set())
         s.update(obj.files)
 
     def _process_unified_sources(self, obj):
         reldir = getattr(obj, 'relativedir', None)
 
         s = self._paths_to_sources.setdefault(reldir, set())
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -31,22 +31,16 @@ from ..util import (
 from ..testing import (
     all_test_flavors,
 )
 
 
 class TreeMetadata(object):
     """Base class for all data being captured."""
 
-    def __init__(self):
-        self._ack = False
-
-    def ack(self):
-        self._ack = True
-
 
 class ContextDerived(TreeMetadata):
     """Build object derived from a single Context instance.
 
     It holds fields common to all context derived classes. This class is likely
     never instantiated directly but is instead derived from.
     """
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -154,18 +154,16 @@ class TreeMetadataEmitter(LoggingMixin):
         typically fed into this function.
         """
         contexts = {}
 
         def emit_objs(objs):
             for o in objs:
                 self._object_count += 1
                 yield o
-                if not o._ack:
-                    raise Exception('Unhandled object of type %s' % type(o))
 
         for out in output:
             # Nothing in sub-contexts is currently of interest to us. Filter
             # them all out.
             if isinstance(out, SubContext):
                 continue
 
             if isinstance(out, Context):
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -67,21 +67,17 @@ class TestEmitterBasic(unittest.TestCase
             BIN_SUFFIX='.prog',
             OS_TARGET='WINNT',
         ))
 
         return BuildReader(config)
 
     def read_topsrcdir(self, reader, filter_common=True):
         emitter = TreeMetadataEmitter(reader.config)
-        def ack(obj):
-            obj.ack()
-            return obj
-
-        objs = list(ack(o) for o in emitter.emit(reader.read_topsrcdir()))
+        objs = list(emitter.emit(reader.read_topsrcdir()))
         self.assertGreater(len(objs), 0)
 
         filtered = []
         for obj in objs:
             if filter_common and isinstance(obj, DirectoryTraversal):
                 continue
 
             filtered.append(obj)