bug 1384568 - Write a generated-sources.json. r=gps
authorTed Mielczarek <ted@mielczarek.org>
Wed, 26 Jul 2017 07:34:38 -0400
changeset 420291 1b1cbbf483f6a05e4525e532a253259003b7e006
parent 420290 0d580486af34eb697ae8de1610c6e92cddb361fb
child 420292 132d97901319d76dc8e56c68b954fc86542c572c
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1384568
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1384568 - Write a generated-sources.json. r=gps Capture the list of generated source files derived from moz.build data and save it in a generated-sources.json in the objdir so that we can upload generated source files for use in crash reports and when debugging release builds. MozReview-Commit-ID: FrHcyFo0rBF
python/mozbuild/mozbuild/backend/common.py
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -9,29 +9,32 @@ import json
 import os
 
 import mozpack.path as mozpath
 
 from mozbuild.backend.base import BuildBackend
 
 from mozbuild.frontend.context import (
     Context,
+    ObjDirPath,
     Path,
     RenamedSourcePath,
     VARIABLES,
 )
 from mozbuild.frontend.data import (
     BaseProgram,
     ChromeManifestEntry,
     ConfigFileSubstitution,
     ExampleWebIDLInterface,
+    Exports,
     IPDLFile,
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     GeneratedEventWebIDLFile,
+    GeneratedSources,
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     SharedLibrary,
     TestWebIDLFile,
     UnifiedSources,
     XPIDLFile,
     WebIDLFile,
@@ -174,16 +177,17 @@ class CommonBackend(BuildBackend):
     """Holds logic common to all build backends."""
 
     def _init(self):
         self._idl_manager = XPIDLManager(self.environment)
         self._webidls = WebIDLCollection()
         self._binaries = BinariesCollection()
         self._configs = set()
         self._ipdl_sources = set()
+        self._generated_sources = set()
 
     def consume_object(self, obj):
         self._configs.add(obj.config)
 
         if isinstance(obj, XPIDLFile):
             # TODO bug 1240134 tracks not processing XPIDL files during
             # artifact builds.
             self._idl_manager.register_idl(obj)
@@ -272,24 +276,35 @@ class CommonBackend(BuildBackend):
         elif isinstance(obj, BaseProgram):
             self._binaries.programs.append(obj)
             return False
 
         elif isinstance(obj, SharedLibrary):
             self._binaries.shared_libraries.append(obj)
             return False
 
+        elif isinstance(obj, GeneratedSources):
+            self._handle_generated_sources(obj.files)
+            return False
+
+        elif isinstance(obj, Exports):
+            objdir_files = [f.full_path for path, files in obj.files.walk() for f in files if isinstance(f, ObjDirPath)]
+            if objdir_files:
+                self._handle_generated_sources(objdir_files)
+            return False
+
         else:
             return False
 
         return True
 
     def consume_finished(self):
         if len(self._idl_manager.idls):
             self._handle_idl_manager(self._idl_manager)
+            self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % idl['root']) for idl in self._idl_manager.idls.values())
 
         self._handle_webidl_collection(self._webidls)
 
         sorted_ipdl_sources = list(sorted(self._ipdl_sources))
 
         def files_from(ipdl):
             base = mozpath.basename(ipdl)
             root, ext = mozpath.splitext(base)
@@ -300,16 +315,17 @@ class CommonBackend(BuildBackend):
                 # .ipdl also becomes Child/Parent.cpp files
                 files.extend(['%sChild.cpp' % root,
                               '%sParent.cpp' % root])
             return files
 
         ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl')
 
         ipdl_cppsrcs = list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources]))
+        self._handle_generated_sources(mozpath.join(ipdl_dir, f) for f in ipdl_cppsrcs)
         unified_source_mapping = list(group_unified_files(ipdl_cppsrcs,
                                                           unified_prefix='UnifiedProtocols',
                                                           unified_suffix='cpp',
                                                           files_per_unified_file=16))
 
         self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
         self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
 
@@ -320,16 +336,26 @@ class CommonBackend(BuildBackend):
         topobjdir = self.environment.topobjdir
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
             d = {
                 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
                 'programs': [p.to_dict() for p in self._binaries.programs],
             }
             json.dump(d, fh, sort_keys=True, indent=4)
 
+        # Write out a file listing generated sources.
+        with self._write_file(mozpath.join(topobjdir, 'generated-sources.json')) as fh:
+            d = {
+                'sources': sorted(self._generated_sources),
+            }
+            json.dump(d, fh, sort_keys=True, indent=4)
+
+    def _handle_generated_sources(self, files):
+        self._generated_sources.update(mozpath.relpath(f, self.environment.topobjdir) for f in files)
+
     def _handle_webidl_collection(self, webidls):
         if not webidls.all_stems():
             return
 
         bindings_dir = mozpath.join(self.environment.topobjdir, 'dom', 'bindings')
 
         all_inputs = set(webidls.all_static_sources())
         for s in webidls.all_non_static_basenames():
@@ -353,17 +379,17 @@ class CommonBackend(BuildBackend):
 
         import mozwebidlcodegen
 
         manager = mozwebidlcodegen.create_build_system_manager(
             self.environment.topsrcdir,
             self.environment.topobjdir,
             mozpath.join(self.environment.topobjdir, 'dist')
         )
-
+        self._handle_generated_sources(manager.expected_build_output_files())
         # Bindings are compiled in unified mode to speed up compilation and
         # to reduce linker memory size. Note that test bindings are separated
         # from regular ones so tests bindings aren't shipped.
         unified_source_mapping = list(group_unified_files(webidls.all_regular_cpp_basenames(),
                                                           unified_prefix='UnifiedBindings',
                                                           unified_suffix='cpp',
                                                           files_per_unified_file=32))
         self._write_unified_files(unified_source_mapping, bindings_dir,