Bug 644608 - Implement full dependencies for expandlibs. r=ted
☠☠ backed out by d151eaf1985c ☠ ☠
authorMike Hommey <mh+mozilla@glandium.org>
Sat, 31 Mar 2012 09:24:39 +0200
changeset 94088 82c315c0bd1446798b74edc9a7be7b695a14de1e
parent 94087 c2a9fb6260384b17046911ddc5cafef4a23b2256
child 94089 369ad04efa1fe15d006ebc41dce6503c81850ad2
push id886
push userlsblakk@mozilla.com
push dateMon, 04 Jun 2012 19:57:52 +0000
treeherdermozilla-beta@bbd8d5efd6d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs644608
milestone14.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 644608 - Implement full dependencies for expandlibs. r=ted
config/config.mk
config/expandlibs_deps.py
config/rules.mk
config/tests/unit-expandlibs.py
js/src/config/config.mk
js/src/config/expandlibs_deps.py
js/src/config/rules.mk
--- a/config/config.mk
+++ b/config/config.mk
@@ -778,17 +778,17 @@ ifdef TIERS
 DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
 STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
 endif
 
 OPTIMIZE_JARS_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/optimizejars.py)
 
 CREATE_PRECOMPLETE_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/createprecomplete.py)
 
-EXPAND_LIBS = $(PYTHON) -I$(DEPTH)/config $(topsrcdir)/config/expandlibs.py
+EXPAND_LIBS_DEPS = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_deps.py
 EXPAND_LIBS_EXEC = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_exec.py
 EXPAND_LIBS_GEN = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_gen.py
 EXPAND_AR = $(EXPAND_LIBS_EXEC) --extract -- $(AR)
 EXPAND_CC = $(EXPAND_LIBS_EXEC) --uselist -- $(CC)
 EXPAND_CCC = $(EXPAND_LIBS_EXEC) --uselist -- $(CCC)
 EXPAND_LD = $(EXPAND_LIBS_EXEC) --uselist -- $(LD)
 EXPAND_MKSHLIB_ARGS = --uselist
 ifdef SYMBOL_ORDER
new file mode 100644
--- /dev/null
+++ b/config/expandlibs_deps.py
@@ -0,0 +1,83 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is a build helper for libraries
+#
+# The Initial Developer of the Original Code is
+# the Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Mike Hommey <mh@glandium.org>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+'''expandlibs_deps.py takes a list of key/value pairs and prints, for each
+key, a list of all dependencies corresponding to the corresponding value.
+Libraries are thus expanded, and their descriptor is also made part of this
+list.
+The list of key/value is passed on the command line in the form
+"var1 = value , var2 = value ..."
+'''
+import sys
+import os
+from expandlibs import ExpandArgs, relativize
+import expandlibs_config as conf
+
+class ExpandLibsDeps(ExpandArgs):
+    def _expand_desc(self, arg):
+        objs = super(ExpandLibsDeps, self)._expand_desc(arg)
+        if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
+            objs += [relativize(arg + conf.LIBS_DESC_SUFFIX)]
+        return objs
+
+def split_args(args):
+    '''Transforms a list in the form
+    ['var1', '=', 'value', ',', 'var2', '=', 'other', 'value', ',' ...]
+    into a corresponding dict 
+    { 'var1': ['value'],
+      'var2': ['other', 'value'], ... }'''
+
+    result = {}
+    while args:
+        try:
+            i = args.index(',')
+            l = args[:i]
+            args[:i + 1] = []
+        except:
+            l = args
+            args = []
+        if l[1] != '=':
+            raise RuntimeError('Expected "var = value" format')
+        result[l[0]] = l[2:]
+    return result
+
+if __name__ == '__main__':
+    for key, value in split_args(sys.argv[1:]).iteritems():
+        expanded = ExpandLibsDeps(value)
+        if os.sep == '\\':
+            expanded = [o.replace(os.sep, '/') for o in expanded]
+        print "%s = %s" % (key, " ".join(expanded))
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -773,21 +773,33 @@ endif # IS_COMPONENT
 endif # EXPORT_LIBRARY
 endif # LIBRARY_NAME
 
 ifneq (,$(filter-out %.$(LIB_SUFFIX),$(SHARED_LIBRARY_LIBS)))
 $(error SHARED_LIBRARY_LIBS must contain .$(LIB_SUFFIX) files only)
 endif
 
 # Create dependencies on static (and shared EXTRA_DSO_LIBS) libraries
-DO_EXPAND_LIBS = $(foreach f,$(1),$(if $(filter %.$(LIB_SUFFIX),$(f)),$(if $(wildcard $(f).$(LIBS_DESC_SUFFIX)),$(f).$(LIBS_DESC_SUFFIX),$(if $(wildcard $(f)),$(f)))))
-LIBS_DEPS = $(call DO_EXPAND_LIBS,$(filter %.$(LIB_SUFFIX),$(LIBS) $(if $(PROGRAM)$(SIMPLE_PROGRAMS),$(MOZ_GLUE_PROGRAM_LDFLAGS))))
-SHARED_LIBRARY_LIBS_DEPS = $(call DO_EXPAND_LIBS,$(SHARED_LIBRARY_LIBS))
+ifneq (,$(strip $(filter %.$(LIB_SUFFIX),$(LIBS) $(EXTRA_DSO_LDOPTS)) $(SHARED_LIBRARY_LIBS) $(EXTRA_DSO_LIBS)))
+$(MDDEPDIR)/libs: Makefile.in
+	@mkdir -p $(MDDEPDIR)
+	@$(EXPAND_LIBS_DEPS) LIBS_DEPS = $(filter %.$(LIB_SUFFIX),$(LIBS)) , \
+	                     SHARED_LIBRARY_LIBS_DEPS = $(SHARED_LIBRARY_LIBS) , \
+	                     DSO_LDOPTS_DEPS = $(EXTRA_DSO_LIBS) $(filter %.$(LIB_SUFFIX), $(EXTRA_DSO_LDOPTS)) > $@
+
+ifneq (,$(wildcard $(MDDEPDIR)/libs))
+include $(MDDEPDIR)/libs
+endif
+
+$(MDDEPDIR)/libs: $(wildcard $(filter %.$(LIBS_DESC_SUFFIX),$(LIBS_DEPS) $(SHARED_LIBRARY_LIBS_DEPS) $(DSO_LDOPTS_DEPS)))
+
+EXTRA_DEPS += $(MDDEPDIR)/libs
+endif
+
 HOST_LIBS_DEPS = $(filter %.$(LIB_SUFFIX),$(HOST_LIBS))
-DSO_LDOPTS_DEPS = $(call DO_EXPAND_LIBS,$(EXTRA_DSO_LIBS) $(filter %.$(LIB_SUFFIX), $(EXTRA_DSO_LDOPTS)))
 
 # Dependencies which, if modified, should cause everything to rebuild
 GLOBAL_DEPS += Makefile Makefile.in $(DEPTH)/config/autoconf.mk $(topsrcdir)/config/config.mk
 
 ##############################################
 include $(topsrcdir)/config/makefiles/target_libs.mk
 
 ##############################################
--- a/config/tests/unit-expandlibs.py
+++ b/config/tests/unit-expandlibs.py
@@ -32,16 +32,17 @@ config_unix = {
     'IMPORT_LIB_SUFFIX': '',
     'LIBS_DESC_SUFFIX': '.desc',
     'EXPAND_LIBS_LIST_STYLE': 'linkerscript',
 }
 
 config = sys.modules['expandlibs_config'] = imp.new_module('expandlibs_config')
 
 from expandlibs import LibDescriptor, ExpandArgs, relativize
+from expandlibs_deps import ExpandLibsDeps, split_args
 from expandlibs_gen import generate
 from expandlibs_exec import ExpandArgsMore, SectionFinder
 
 def Lib(name):
     return config.LIB_PREFIX + name + config.LIB_SUFFIX
 
 def Obj(name):
     return name + config.OBJ_SUFFIX
@@ -186,16 +187,40 @@ class TestExpandArgs(TestExpandInit):
         self.touch([self.tmpfile('libx', Lib('x'))])
         args = ExpandArgs(['foo', '-bar'] + self.arg_files + [self.tmpfile('liby', Lib('y'))])
         self.assertRelEqual(args, ['foo', '-bar'] + self.files + self.liby_files + [self.tmpfile('libx', Lib('x'))]) 
 
         self.touch([self.tmpfile('liby', Lib('y'))])
         args = ExpandArgs(['foo', '-bar'] + self.arg_files + [self.tmpfile('liby', Lib('y'))])
         self.assertRelEqual(args, ['foo', '-bar'] + self.files + [self.tmpfile('liby', Lib('y'))])
 
+class TestExpandLibsDeps(TestExpandInit):
+    def test_expandlibsdeps(self):
+        '''Test library expansion for dependencies'''
+        # Dependency list for a library with a descriptor is equivalent to
+        # the arguments expansion, to which we add each descriptor
+        args = self.arg_files + [self.tmpfile('liby', Lib('y'))]
+        self.assertRelEqual(ExpandLibsDeps(args), ExpandArgs(args) + [self.tmpfile('libx', Lib('x') + config.LIBS_DESC_SUFFIX), self.tmpfile('liby', Lib('y') + config.LIBS_DESC_SUFFIX)])
+
+        # When a library exists at the same time as a descriptor, the
+        # descriptor is not a dependency
+        self.touch([self.tmpfile('libx', Lib('x'))])
+        args = self.arg_files + [self.tmpfile('liby', Lib('y'))]
+        self.assertRelEqual(ExpandLibsDeps(args), ExpandArgs(args) + [self.tmpfile('liby', Lib('y') + config.LIBS_DESC_SUFFIX)])
+
+        self.touch([self.tmpfile('liby', Lib('y'))])
+        args = self.arg_files + [self.tmpfile('liby', Lib('y'))]
+        self.assertRelEqual(ExpandLibsDeps(args), ExpandArgs(args))
+
+class TestSplitArgs(unittest.TestCase):
+    def test_split_args(self):
+        self.assertEqual(split_args(['a', '=', 'b', 'c']), {'a': ['b', 'c']})
+        self.assertEqual(split_args(['a', '=', 'b', 'c', ',', 'd', '=', 'e', 'f', 'g', ',', 'h', '=', 'i']),
+                         {'a': ['b', 'c'], 'd': ['e', 'f', 'g'], 'h': ['i']})
+
 class TestExpandArgsMore(TestExpandInit):
     def test_makelist(self):
         '''Test grouping object files in lists'''
         # ExpandArgsMore does the same as ExpandArgs
         with ExpandArgsMore(['foo', '-bar'] + self.arg_files + [self.tmpfile('liby', Lib('y'))]) as args:
             self.assertRelEqual(args, ['foo', '-bar'] + self.files + self.liby_files + self.libx_files) 
 
             # But also has an extra method replacing object files with a list
--- a/js/src/config/config.mk
+++ b/js/src/config/config.mk
@@ -778,17 +778,17 @@ ifdef TIERS
 DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
 STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
 endif
 
 OPTIMIZE_JARS_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/optimizejars.py)
 
 CREATE_PRECOMPLETE_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/createprecomplete.py)
 
-EXPAND_LIBS = $(PYTHON) -I$(DEPTH)/config $(topsrcdir)/config/expandlibs.py
+EXPAND_LIBS_DEPS = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_deps.py
 EXPAND_LIBS_EXEC = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_exec.py
 EXPAND_LIBS_GEN = $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(DEPTH)/config $(topsrcdir)/config/expandlibs_gen.py
 EXPAND_AR = $(EXPAND_LIBS_EXEC) --extract -- $(AR)
 EXPAND_CC = $(EXPAND_LIBS_EXEC) --uselist -- $(CC)
 EXPAND_CCC = $(EXPAND_LIBS_EXEC) --uselist -- $(CCC)
 EXPAND_LD = $(EXPAND_LIBS_EXEC) --uselist -- $(LD)
 EXPAND_MKSHLIB_ARGS = --uselist
 ifdef SYMBOL_ORDER
new file mode 100644
--- /dev/null
+++ b/js/src/config/expandlibs_deps.py
@@ -0,0 +1,83 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is a build helper for libraries
+#
+# The Initial Developer of the Original Code is
+# the Mozilla Foundation
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Mike Hommey <mh@glandium.org>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+'''expandlibs_deps.py takes a list of key/value pairs and prints, for each
+key, a list of all dependencies corresponding to the corresponding value.
+Libraries are thus expanded, and their descriptor is also made part of this
+list.
+The list of key/value is passed on the command line in the form
+"var1 = value , var2 = value ..."
+'''
+import sys
+import os
+from expandlibs import ExpandArgs, relativize
+import expandlibs_config as conf
+
+class ExpandLibsDeps(ExpandArgs):
+    def _expand_desc(self, arg):
+        objs = super(ExpandLibsDeps, self)._expand_desc(arg)
+        if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
+            objs += [relativize(arg + conf.LIBS_DESC_SUFFIX)]
+        return objs
+
+def split_args(args):
+    '''Transforms a list in the form
+    ['var1', '=', 'value', ',', 'var2', '=', 'other', 'value', ',' ...]
+    into a corresponding dict 
+    { 'var1': ['value'],
+      'var2': ['other', 'value'], ... }'''
+
+    result = {}
+    while args:
+        try:
+            i = args.index(',')
+            l = args[:i]
+            args[:i + 1] = []
+        except:
+            l = args
+            args = []
+        if l[1] != '=':
+            raise RuntimeError('Expected "var = value" format')
+        result[l[0]] = l[2:]
+    return result
+
+if __name__ == '__main__':
+    for key, value in split_args(sys.argv[1:]).iteritems():
+        expanded = ExpandLibsDeps(value)
+        if os.sep == '\\':
+            expanded = [o.replace(os.sep, '/') for o in expanded]
+        print "%s = %s" % (key, " ".join(expanded))
--- a/js/src/config/rules.mk
+++ b/js/src/config/rules.mk
@@ -773,21 +773,33 @@ endif # IS_COMPONENT
 endif # EXPORT_LIBRARY
 endif # LIBRARY_NAME
 
 ifneq (,$(filter-out %.$(LIB_SUFFIX),$(SHARED_LIBRARY_LIBS)))
 $(error SHARED_LIBRARY_LIBS must contain .$(LIB_SUFFIX) files only)
 endif
 
 # Create dependencies on static (and shared EXTRA_DSO_LIBS) libraries
-DO_EXPAND_LIBS = $(foreach f,$(1),$(if $(filter %.$(LIB_SUFFIX),$(f)),$(if $(wildcard $(f).$(LIBS_DESC_SUFFIX)),$(f).$(LIBS_DESC_SUFFIX),$(if $(wildcard $(f)),$(f)))))
-LIBS_DEPS = $(call DO_EXPAND_LIBS,$(filter %.$(LIB_SUFFIX),$(LIBS) $(if $(PROGRAM)$(SIMPLE_PROGRAMS),$(MOZ_GLUE_PROGRAM_LDFLAGS))))
-SHARED_LIBRARY_LIBS_DEPS = $(call DO_EXPAND_LIBS,$(SHARED_LIBRARY_LIBS))
+ifneq (,$(strip $(filter %.$(LIB_SUFFIX),$(LIBS) $(EXTRA_DSO_LDOPTS)) $(SHARED_LIBRARY_LIBS) $(EXTRA_DSO_LIBS)))
+$(MDDEPDIR)/libs: Makefile.in
+	@mkdir -p $(MDDEPDIR)
+	@$(EXPAND_LIBS_DEPS) LIBS_DEPS = $(filter %.$(LIB_SUFFIX),$(LIBS)) , \
+	                     SHARED_LIBRARY_LIBS_DEPS = $(SHARED_LIBRARY_LIBS) , \
+	                     DSO_LDOPTS_DEPS = $(EXTRA_DSO_LIBS) $(filter %.$(LIB_SUFFIX), $(EXTRA_DSO_LDOPTS)) > $@
+
+ifneq (,$(wildcard $(MDDEPDIR)/libs))
+include $(MDDEPDIR)/libs
+endif
+
+$(MDDEPDIR)/libs: $(wildcard $(filter %.$(LIBS_DESC_SUFFIX),$(LIBS_DEPS) $(SHARED_LIBRARY_LIBS_DEPS) $(DSO_LDOPTS_DEPS)))
+
+EXTRA_DEPS += $(MDDEPDIR)/libs
+endif
+
 HOST_LIBS_DEPS = $(filter %.$(LIB_SUFFIX),$(HOST_LIBS))
-DSO_LDOPTS_DEPS = $(call DO_EXPAND_LIBS,$(EXTRA_DSO_LIBS) $(filter %.$(LIB_SUFFIX), $(EXTRA_DSO_LDOPTS)))
 
 # Dependencies which, if modified, should cause everything to rebuild
 GLOBAL_DEPS += Makefile Makefile.in $(DEPTH)/config/autoconf.mk $(topsrcdir)/config/config.mk
 
 ##############################################
 include $(topsrcdir)/config/makefiles/target_libs.mk
 
 ##############################################