Bug 1253436 - Write out a machine readable file with binaries metadata; r=glandium
☠☠ backed out by 04a9056213bd ☠ ☠
authorGregory Szorc <gps@mozilla.com>
Mon, 14 Mar 2016 19:31:35 -0700
changeset 288690 4167dfdf10457360c9c94ee6e55b03ef1b92c16d
parent 288689 84849ad026c9ba1bbf71c93172b0a03440e51bec
child 288691 8e35ad3b586f7aabdfce8050893a45a8c47f518c
push id18174
push usercbook@mozilla.com
push dateTue, 15 Mar 2016 09:44:58 +0000
treeherderfx-team@dd0baa33759d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1253436
milestone48.0a1
Bug 1253436 - Write out a machine readable file with binaries metadata; r=glandium This will make it easier for binaries only archive generation to consult the list of binaries that are relevant to packaging. This does overlap somewhat with compile databases. Perhaps someday the logic could converge. While I was here, I cleaned up writing of the all-tests.json file to avoid an intermediate string variable. This patch adds to_dict methods on some frontend data types. There is room to improve the serialization of these types. For example, we could use __slots__ to drive the default formatter. We could also leverage a custom JSONEncoder class that knows how to call to_dict() on instances. In the spirit of perfect is the enemy of done, I think this should be deferred to a follow-up bug. MozReview-Commit-ID: XVnKB1MNqu
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -15,26 +15,28 @@ from mozbuild.backend.base import BuildB
 
 from mozbuild.frontend.context import (
     Context,
     Path,
     RenamedSourcePath,
     VARIABLES,
 )
 from mozbuild.frontend.data import (
+    BaseProgram,
     ChromeManifestEntry,
     ConfigFileSubstitution,
     ExampleWebIDLInterface,
     IPDLFile,
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     GeneratedEventWebIDLFile,
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
+    SharedLibrary,
     TestManifest,
     TestWebIDLFile,
     UnifiedSources,
     XPIDLFile,
     WebIDLFile,
 )
 from mozbuild.jar import (
     DeprecatedJarManifest,
@@ -187,23 +189,32 @@ class TestManager(object):
 
         key = path[len(topsrcdir)+1:]
         t['file_relpath'] = key
         t['dir_relpath'] = mozpath.dirname(key)
 
         self.tests_by_path[key].append(t)
 
 
+class BinariesCollection(object):
+    """Tracks state of binaries produced by the build."""
+
+    def __init__(self):
+        self.shared_libraries = []
+        self.programs = []
+
+
 class CommonBackend(BuildBackend):
     """Holds logic common to all build backends."""
 
     def _init(self):
         self._idl_manager = XPIDLManager(self.environment)
         self._test_manager = TestManager(self.environment)
         self._webidls = WebIDLCollection()
+        self._binaries = BinariesCollection()
         self._configs = set()
         self._ipdl_sources = set()
 
     def consume_object(self, obj):
         self._configs.add(obj.config)
 
         if isinstance(obj, TestManifest):
             for test in obj.tests:
@@ -290,16 +301,25 @@ class CommonBackend(BuildBackend):
             # Unified sources aren't relevant to artifact builds.
             if self.environment.is_artifact_build:
                 return True
 
             if obj.have_unified_mapping:
                 self._write_unified_files(obj.unified_source_mapping, obj.objdir)
             if hasattr(self, '_process_unified_sources'):
                 self._process_unified_sources(obj)
+
+        elif isinstance(obj, BaseProgram):
+            self._binaries.programs.append(obj)
+            return False
+
+        elif isinstance(obj, SharedLibrary):
+            self._binaries.shared_libraries.append(obj)
+            return False
+
         else:
             return False
 
         return True
 
     def consume_finished(self):
         if len(self._idl_manager.idls):
             self._handle_idl_manager(self._idl_manager)
@@ -330,20 +350,27 @@ class CommonBackend(BuildBackend):
 
         self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
         self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
 
         for config in self._configs:
             self.backend_input_files.add(config.source)
 
         # Write out a machine-readable file describing every test.
-        path = mozpath.join(self.environment.topobjdir, 'all-tests.json')
-        with self._write_file(path) as fh:
-            s = json.dumps(self._test_manager.tests_by_path)
-            fh.write(s)
+        topobjdir = self.environment.topobjdir
+        with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
+            json.dump(self._test_manager.tests_by_path, fh)
+
+        # Write out a machine-readable file describing binaries.
+        with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
+            d = {
+                'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
+                'programs': [p.to_dict() for p in self._binaries.programs],
+            }
+            json.dump(d, fh, sort_keys=True, indent=4)
 
     def _handle_webidl_collection(self, webidls):
         if not webidls.all_stems():
             return
 
         bindings_dir = mozpath.join(self.environment.topobjdir, 'dom', 'bindings')
 
         all_inputs = set(webidls.all_static_sources())
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -31,16 +31,19 @@ from ..testing import (
     all_test_flavors,
 )
 
 
 class TreeMetadata(object):
     """Base class for all data being captured."""
     __slots__ = ()
 
+    def to_dict(self):
+        return {k.lower(): getattr(self, k) for k in self.DICT_ATTRS}
+
 
 class ContextDerived(TreeMetadata):
     """Build object derived from a single Context instance.
 
     It holds fields common to all context derived classes. This class is likely
     never instantiated directly but is instead derived from.
     """
 
@@ -352,16 +355,23 @@ class BaseProgram(Linkable):
     This class handles automatically appending a binary suffix to the program
     name.
     If the suffix is not defined, the program name is unchanged.
     Otherwise, if the program name ends with the given suffix, it is unchanged
     Otherwise, the suffix is appended to the program name.
     """
     __slots__ = ('program')
 
+    DICT_ATTRS = {
+        'install_target',
+        'kind',
+        'program',
+        'relobjdir',
+    }
+
     def __init__(self, context, program, is_unit_test=False):
         Linkable.__init__(self, context)
 
         bin_suffix = context.config.substs.get(self.SUFFIX_VAR, '')
         if not program.endswith(bin_suffix):
             program += bin_suffix
         self.program = program
         self.is_unit_test = is_unit_test
@@ -452,16 +462,25 @@ class StaticLibrary(Library):
 class SharedLibrary(Library):
     """Context derived container object for a shared library"""
     __slots__ = (
         'soname',
         'variant',
         'symbols_file',
     )
 
+    DICT_ATTRS = {
+        'basename',
+        'import_name',
+        'install_target',
+        'lib_name',
+        'relobjdir',
+        'soname',
+    }
+
     FRAMEWORK = 1
     COMPONENT = 2
     MAX_VARIANT = 3
 
     def __init__(self, context, basename, real_name=None, is_sdk=False,
             soname=None, variant=None, symbols_file=False):
         assert(variant in range(1, self.MAX_VARIANT) or variant is None)
         Library.__init__(self, context, basename, real_name, is_sdk)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -866,11 +866,37 @@ class TestRecursiveMakeBackend(BackendTe
             'LIBRARY_NAME := bar\n',
             'FORCE_SHARED_LIB := 1\n',
             'IMPORT_LIBRARY := bar\n',
             'SHARED_LIBRARY := bar\n',
             'IS_COMPONENT := 1\n',
             'DSO_SONAME := bar\n',
         ])
 
+        self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, 'binaries.json')))
+        with open(mozpath.join(env.topobjdir, 'binaries.json'), 'rb') as fh:
+            binaries = json.load(fh)
+
+        self.assertEqual(binaries, {
+            'programs': [],
+            'shared_libraries': [
+                {
+                    'basename': 'foo',
+                    'import_name': 'foo',
+                    'install_target': 'dist/bin',
+                    'lib_name': 'foo',
+                    'relobjdir': 'foo',
+                    'soname': 'foo',
+                },
+                {
+                    'basename': 'bar',
+                    'import_name': 'bar',
+                    'install_target': 'dist/bin',
+                    'lib_name': 'bar',
+                    'relobjdir': 'bar',
+                    'soname': 'bar',
+                }
+            ],
+        })
+
 
 if __name__ == '__main__':
     main()