Bug 1384400 - Add input to output mappings to mozpack.FileRegistry. r=gps
authorNick Alexander <nalexander@mozilla.com>
Tue, 25 Jul 2017 12:28:31 -0700
changeset 419944 40af30efbf6c10cab888c712377af8fe6f4ed3fc
parent 419943 8315c95af78c24ff42133c5d3cb4f95088e0e00a
child 419945 d2513137446a16ed71bf264439371b573d17bd89
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1384400
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1384400 - Add input to output mappings to mozpack.FileRegistry. r=gps This is necessary because the existing manifests don't expose full dependency information. I needed to avoid the existing dependency files because those code paths need to know the output destination of the manifest in order to parse the Make dependency files; trying to adapt this system is more complicated than just preprocessing each file to extract dependency information directly. MozReview-Commit-ID: 5m0SEqmhJMM
python/mozbuild/mozpack/copier.py
python/mozbuild/mozpack/files.py
--- a/python/mozbuild/mozpack/copier.py
+++ b/python/mozbuild/mozpack/copier.py
@@ -11,16 +11,17 @@ import sys
 from mozpack.errors import errors
 from mozpack.files import (
     BaseFile,
     Dest,
 )
 import mozpack.path as mozpath
 import errno
 from collections import (
+    defaultdict,
     Counter,
     OrderedDict,
 )
 import concurrent.futures as futures
 
 
 class FileRegistry(object):
     '''
@@ -148,16 +149,44 @@ class FileRegistry(object):
         '''
         Return the set of directories required by the paths in the container,
         in no particular order.  The returned directories are relative to an
         unspecified (virtual) root directory (and do not include said root
         directory).
         '''
         return set(k for k, v in self._required_directories.items() if v > 0)
 
+    def output_to_inputs_tree(self):
+        '''
+        Return a dictionary mapping each output path to the set of its
+        required input paths.
+
+        All paths are normalized.
+        '''
+        tree = {}
+        for output, file in self:
+            output = mozpath.normpath(output)
+            tree[output] = set(mozpath.normpath(f) for f in file.inputs())
+        return tree
+
+    def input_to_outputs_tree(self):
+        '''
+        Return a dictionary mapping each input path to the set of
+        impacted output paths.
+
+        All paths are normalized.
+        '''
+        tree = defaultdict(set)
+        for output, file in self:
+            output = mozpath.normpath(output)
+            for input in file.inputs():
+                input = mozpath.normpath(input)
+                tree[input].add(output)
+        return dict(tree)
+
 
 class FileRegistrySubtree(object):
     '''A proxy class to give access to a subtree of an existing FileRegistry.
 
     Note this doesn't implement the whole FileRegistry interface.'''
     def __new__(cls, base, registry):
         if not base:
             return registry
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -229,16 +229,22 @@ class BaseFile(object):
 
     @property
     def mode(self):
         '''
         Return the file's unix mode, or None if it has no meaning.
         '''
         return None
 
+    def inputs(self):
+        '''
+        Return an iterable of the input file paths that impact this output file.
+        '''
+        raise NotImplementedError('BaseFile.inputs() not implemented.')
+
 
 class File(BaseFile):
     '''
     File class for plain files.
     '''
     def __init__(self, path):
         self.path = path
 
@@ -256,16 +262,19 @@ class File(BaseFile):
     def read(self):
         '''Return the contents of the file.'''
         with open(self.path, 'rb') as fh:
             return fh.read()
 
     def size(self):
         return os.stat(self.path).st_size
 
+    def inputs(self):
+        return (self.path,)
+
 
 class ExecutableFile(File):
     '''
     File class for executable and library files on OS/2, OS/X and ELF systems.
     (see mozpack.executables.is_executable documentation).
     '''
     def copy(self, dest, skip_if_older=True):
         real_dest = dest
@@ -469,32 +478,46 @@ class ExistingFile(BaseFile):
 
         if not self.required:
             return
 
         if not dest.exists():
             errors.fatal("Required existing file doesn't exist: %s" %
                 dest.path)
 
+    def inputs(self):
+        return ()
+
 
 class PreprocessedFile(BaseFile):
     '''
     File class for a file that is preprocessed. PreprocessedFile.copy() runs
     the preprocessor on the file to create the output.
     '''
     def __init__(self, path, depfile_path, marker, defines, extra_depends=None,
                  silence_missing_directive_warnings=False):
         self.path = path
         self.depfile = depfile_path
         self.marker = marker
         self.defines = defines
         self.extra_depends = list(extra_depends or [])
         self.silence_missing_directive_warnings = \
             silence_missing_directive_warnings
 
+    def inputs(self):
+        pp = Preprocessor(defines=self.defines, marker=self.marker)
+        pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings)
+
+        with open(self.path, 'rU') as input:
+            with open(os.devnull, 'w') as output:
+                pp.processFile(input=input, output=output)
+
+        # This always yields at least self.path.
+        return pp.includes
+
     def copy(self, dest, skip_if_older=True):
         '''
         Invokes the preprocessor to create the destination file.
         '''
         if isinstance(dest, basestring):
             dest = Dest(dest)
         else:
             assert isinstance(dest, Dest)
@@ -560,16 +583,19 @@ class GeneratedFile(BaseFile):
         return BytesIO(self.content)
 
     def read(self):
         return self.content
 
     def size(self):
         return len(self.content)
 
+    def inputs(self):
+        return ()
+
 
 class DeflatedFile(BaseFile):
     '''
     File class for members of a jar archive. DeflatedFile.copy() effectively
     extracts the file from the jar archive.
     '''
     def __init__(self, file):
         from mozpack.mozjar import JarFileReader