Bug 1459721 - part 8 - pass full paths for IDL files to xpidl-process.py; r=chmanchester
authorNathan Froyd <froydnj@mozilla.com>
Tue, 15 May 2018 10:05:23 -0400
changeset 418383 d52f0d61dd570ae9414bfad4dd3df88051466fdd
parent 418382 7f618d13bd1cc90cf9cf36ab29d912e73dd315d4
child 418384 73dd77918ce5997d13cefc2cae32c7ae4644ce82
push id33999
push userdluca@mozilla.com
push dateTue, 15 May 2018 21:54:51 +0000
treeherdermozilla-central@380cf87c1ee3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1459721
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1459721 - part 8 - pass full paths for IDL files to xpidl-process.py; r=chmanchester The build system knows at build-backend time where to find each IDL file; making xpidl-process.py rediscover this by requiring xpidl-process.py to search through directories to find input IDL files is silly. To rememdy this, we're going to modify things so full paths are passed into the script. Those paths can then be used directly, with no searching.
config/makefiles/xpidl/Makefile.in
python/mozbuild/mozbuild/action/xpidl-process.py
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/backend/tup.py
--- a/config/makefiles/xpidl/Makefile.in
+++ b/config/makefiles/xpidl/Makefile.in
@@ -36,17 +36,16 @@ code_gen_deps := $(topsrcdir)/xpcom/refl
 
 # TODO we should use py_action, but that would require extra directories to be
 # in the virtualenv.
 %.xpt:
 	$(REPORT_BUILD)
 	$(PYTHON_PATH) $(PLY_INCLUDE) -I$(topsrcdir)/xpcom/idl-parser -I$(DEPTH)/xpcom/idl-parser/xpidl \
 		$(process_py) --cache-dir $(DEPTH)/xpcom/idl-parser/xpidl --depsdir $(idl_deps_dir) \
 		--bindings-conf $(topsrcdir)/dom/bindings/Bindings.conf \
-		$(foreach dir,$($(basename $(notdir $@))_dirs),--input-dir $(dir)) \
 		$(foreach dir,$(all_idl_dirs),-I $(dir)) \
 		$(dist_include_dir) $(dist_xpcrs_dir) $(@D) \
 		$(basename $(notdir $@)) $($(basename $(notdir $@))_deps)
 # When some IDL is added or removed, if the actual IDL file was already, or
 # still is, in the tree, simple dependencies can't detect that the XPT needs
 # to be rebuilt.
 # Add the current value of $($(xpidl_module)_deps) in the depend file, such that
 # we can later check if the value has changed since last build, which will
--- a/python/mozbuild/mozbuild/action/xpidl-process.py
+++ b/python/mozbuild/mozbuild/action/xpidl-process.py
@@ -23,42 +23,35 @@ from xpidl.rust_macros import print_rust
 from xpidl.xpidl import IDLParser
 
 from mozbuild.makeutil import Makefile
 from mozbuild.pythonutil import iter_modules_in_path
 from mozbuild.util import FileAvoidWrite
 
 
 def process(input_dirs, inc_paths, bindings_conf, cache_dir, header_dir,
-            xpcrs_dir, xpt_dir, deps_dir, module, stems):
+            xpcrs_dir, xpt_dir, deps_dir, module, idl_files):
     p = IDLParser(outputdir=cache_dir)
 
     xpts = []
     mk = Makefile()
     rule = mk.create_rule()
 
     glbl = {}
     execfile(bindings_conf, glbl)
     webidlconfig = glbl['DOMInterfaces']
 
     # Write out dependencies for Python modules we import. If this list isn't
     # up to date, we will not re-process XPIDL files if the processor changes.
     rule.add_dependencies(iter_modules_in_path(topsrcdir))
 
-    def read_stem(stem):
-        idl = '%s.idl' % stem
-        for p in input_dirs:
-            idl_file = os.path.join(topsrcdir, p, idl)
-            if os.path.exists(idl_file):
-                return idl_file, open(idl_file).read()
-
-        raise Exception('Could not find stem file for %s' % idl)
-
-    for stem in stems:
-        path, idl_data = read_stem(stem)
+    for path in idl_files:
+        basename = os.path.basename(path)
+        stem, _ = os.path.splitext(basename)
+        idl_data = open(path).read()
 
         idl = p.parse(idl_data, filename=path)
         idl.resolve(inc_paths, p, webidlconfig)
 
         header_path = os.path.join(header_dir, '%s.h' % stem)
         rs_rt_path = os.path.join(xpcrs_dir, 'rt', '%s.rs' % stem)
         rs_bt_path = os.path.join(xpcrs_dir, 'bt', '%s.rs' % stem)
 
@@ -106,17 +99,17 @@ def main(argv):
         help='Directory in which to write header files.')
     parser.add_argument('xpcrsdir',
         help='Directory in which to write rust xpcom binding files.')
     parser.add_argument('xptdir',
         help='Directory in which to write xpt file.')
     parser.add_argument('module',
         help='Final module name to use for linked output xpt file.')
     parser.add_argument('idls', nargs='+',
-        help='Source .idl file(s). Specified as stems only.')
+        help='Source .idl file(s).')
     parser.add_argument('-I', dest='incpath', action='append', default=[],
         help='Extra directories where to look for included .idl files.')
 
     args = parser.parse_args(argv)
     incpath = [os.path.join(topsrcdir, p) for p in args.incpath]
     process(args.input_dirs, incpath, args.bindings_conf, args.cache_dir,
         args.headerdir, args.xpcrsdir, args.xptdir, args.depsdir, args.module,
         args.idls)
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -81,17 +81,17 @@ class XPIDLManager(object):
 
         self.idls[entry['basename']] = entry
         # First element is a set of interface file basenames (no extension).
         #
         # Second element is a set of directory names where module IDLs
         # can be found.  Yes, we have XPIDL modules with files from
         # multiple directories.
         t = self.modules.setdefault(entry['module'], (set(), set()))
-        t[0].add(entry['root'])
+        t[0].add(entry['source'])
         t[1].add(dirname)
 
 class BinariesCollection(object):
     """Tracks state of binaries produced by the build."""
 
     def __init__(self):
         self.shared_libraries = []
         self.programs = []
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1051,32 +1051,29 @@ class RecursiveMakeBackend(CommonBackend
 
         mk = Makefile()
         all_directories = set()
 
         for module in xpt_modules:
             sources, directories = modules[module]
             all_directories |= directories
             deps = sorted(sources)
-            directories = sorted(directories)
 
             # It may seem strange to have the .idl files listed as
             # prerequisites both here and in the auto-generated .pp files.
             # It is necessary to list them here to handle the case where a
             # new .idl is added to an xpt. If we add a new .idl and nothing
             # else has changed, the new .idl won't be referenced anywhere
             # except in the command invocation. Therefore, the .xpt won't
             # be rebuilt because the dependencies say it is up to date. By
             # listing the .idls here, we ensure the make file has a
             # reference to the new .idl. Since the new .idl presumably has
             # an mtime newer than the .xpt, it will trigger xpt generation.
             mk.add_statement('%s_deps = %s' % (module, ' '.join(deps)))
 
-            mk.add_statement('%s_dirs = %s' % (module, ' '.join(directories)))
-
             build_files.add_optional_exists('%s.xpt' % module)
 
         mk.add_statement('all_idl_dirs = %s' % ' '.join(sorted(all_directories)))
 
         rules = StringIO()
         mk.dump(rules, removal_guard=False)
 
         # Create dependency for output header so we force regeneration if the
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -785,45 +785,44 @@ class TupBackend(CommonBackend):
 
         backend_file = self._get_backend_file('xpcom/xpidl')
         backend_file.export_shell()
 
         all_idl_directories = set()
         all_idl_directories.update(*map(lambda x: x[1], manager.modules.itervalues()))
 
         all_xpts = []
-        for module, (idls, directories) in sorted(manager.modules.iteritems()):
+        for module, (idls, _) in sorted(manager.modules.iteritems()):
             cmd = [
                 '$(PYTHON_PATH)',
                 '$(PLY_INCLUDE)',
                 '-I$(IDL_PARSER_DIR)',
                 '-I$(IDL_PARSER_CACHE_DIR)',
                 '$(topsrcdir)/python/mozbuild/mozbuild/action/xpidl-process.py',
                 '--cache-dir', '$(IDL_PARSER_CACHE_DIR)',
                 '--bindings-conf', '$(topsrcdir)/dom/bindings/Bindings.conf',
             ]
 
-            for d in directories:
-                cmd.extend(['--input-dir', d])
             for d in all_idl_directories:
                 cmd.extend(['-I', d])
 
             cmd.extend([
                 '$(DIST)/include',
                 '$(DIST)/xpcrs',
                 '.',
                 module,
             ])
             cmd.extend(sorted(idls))
 
             all_xpts.append('$(MOZ_OBJ_ROOT)/%s/%s.xpt' % (backend_file.relobjdir, module))
             outputs = ['%s.xpt' % module]
-            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/include/%s.h' % f for f in sorted(idls)])
-            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/xpcrs/rt/%s.rs' % f for f in sorted(idls)])
-            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/xpcrs/bt/%s.rs' % f for f in sorted(idls)])
+            stems = sorted(mozpath.splitext(mozpath.basename(idl))[0] for idl in idls)
+            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/include/%s.h' % f for f in stems])
+            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/xpcrs/rt/%s.rs' % f for f in stems])
+            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/xpcrs/bt/%s.rs' % f for f in stems])
             backend_file.rule(
                 inputs=[
                     '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidllex.py',
                     '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidlyacc.py',
                     self._installed_idls,
                 ],
                 display='XPIDL %s' % module,
                 cmd=cmd,