Bug 1041960 - Remove subtiers from make backend. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 23 Jul 2014 13:28:54 +0900
changeset 195620 9f4d4f62db1c1c73ee9547889c95c26de89127b0
parent 195619 5c157de1ee6ce6288d08981c599855e505b676d1
child 195621 9565dc1c02c6a4917f1d8f68ba42654f4ee588c9
push id46645
push usermh@glandium.org
push dateWed, 23 Jul 2014 04:33:55 +0000
treeherdermozilla-inbound@c71f854e6358 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1041960
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1041960 - Remove subtiers from make backend. r=gps
config/recurse.mk
python/mozbuild/mozbuild/backend/recursivemake.py
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -46,17 +46,16 @@ compile binaries libs export tools::
 # Carefully avoid $(eval) type of rule generation, which makes pymake slower
 # than necessary.
 # Get current tier and corresponding subtiers from the data in root.mk.
 CURRENT_TIER := $(filter $(foreach tier,compile binaries libs export tools,recurse_$(tier) $(tier)-deps),$(MAKECMDGOALS))
 ifneq (,$(filter-out 0 1,$(words $(CURRENT_TIER))))
 $(error $(CURRENT_TIER) not supported on the same make command line)
 endif
 CURRENT_TIER := $(subst recurse_,,$(CURRENT_TIER:-deps=))
-CURRENT_SUBTIERS := $($(CURRENT_TIER)_subtiers)
 
 # The rules here are doing directory traversal, so we don't want further
 # recursion to happen when running make -C subdir $tier. But some make files
 # further call make -C something else, and sometimes expect recursion to
 # happen in that case (see browser/metro/locales/Makefile.in for example).
 # Conveniently, every invocation of make increases MAKELEVEL, so only stop
 # recursion from happening at current MAKELEVEL + 1.
 ifdef CURRENT_TIER
@@ -65,45 +64,34 @@ export NO_RECURSE_MAKELEVEL=1
 else
 export NO_RECURSE_MAKELEVEL=$(word $(MAKELEVEL),2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20)
 endif
 endif
 
 # Get all directories traversed for all subtiers in the current tier, or use
 # directly the $(*_dirs) variables available in root.mk when there is no
 # TIERS (like for js/src).
-TIER_DIRS = $(or $($(1)_dirs),$(foreach subtier,$($(1)_subtiers),$($(1)_subtier_$(subtier))))
-CURRENT_DIRS := $(call TIER_DIRS,$(CURRENT_TIER))
+CURRENT_DIRS := $($(CURRENT_TIER)_dirs)
 
 ifneq (,$(filter binaries libs,$(CURRENT_TIER)))
 WANT_STAMPS = 1
 STAMP_TOUCH = $(TOUCH) $(@D)/binaries
 endif
 
-# Subtier delimiter rules
-$(addprefix subtiers/,$(addsuffix _start/$(CURRENT_TIER),$(CURRENT_SUBTIERS))): subtiers/%_start/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,subtiers/%_start))
-	@$(STAMP_TOUCH)
-
-$(addprefix subtiers/,$(addsuffix _finish/$(CURRENT_TIER),$(CURRENT_SUBTIERS))): subtiers/%_finish/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,subtiers/%_finish))
-	@$(STAMP_TOUCH)
-
-$(addprefix subtiers/,$(addsuffix /$(CURRENT_TIER),$(CURRENT_SUBTIERS))): %/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,%))
-	@$(STAMP_TOUCH)
-
-GARBAGE_DIRS += subtiers
-
 # Recursion rule for all directories traversed for all subtiers in the
 # current tier.
-# root.mk defines subtier_of_* variables, that map a normalized subdir path to
-# a subtier name (e.g. subtier_of_memory_jemalloc = base)
 $(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER):
-	$(call SUBMAKE,$(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER)),$*)
+	$(call SUBMAKE,$(if $(filter $*,$(staticdirs)),,$(CURRENT_TIER)),$*)
 # Ensure existing stamps are up-to-date, but don't create one if submake didn't create one.
 	$(if $(wildcard $@),@$(STAMP_TOUCH))
 
+ifndef STAMP_TOUCH
+.PHONY: $(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS))
+endif
+
 # Dummy rules for possibly inexisting dependencies for the above tier targets
 $(addsuffix /Makefile,$(CURRENT_DIRS)) $(addsuffix /backend.mk,$(CURRENT_DIRS)):
 
 # The export tier requires nsinstall, which is built from config. So every
 # subdirectory traversal needs to happen after traversing config.
 ifeq ($(CURRENT_TIER),export)
 $(addsuffix /$(CURRENT_TIER),$(filter-out config,$(CURRENT_DIRS))): config/$(CURRENT_TIER)
 endif
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -477,17 +477,18 @@ class RecursiveMakeBackend(CommonBackend
             self.log(logging.DEBUG, 'fill_root_mk', {
                 'number': len(skip), 'tier': tier
                 }, 'Ignoring {number} directories during {tier}')
 
         # Traverse directories in parallel, and skip static dirs
         def parallel_filter(current, subdirs):
             all_subdirs = subdirs.parallel + subdirs.dirs + \
                           subdirs.tests + subdirs.tools
-            if current in self._may_skip[tier]:
+            if current in self._may_skip[tier] \
+                    or current.startswith('subtiers/'):
                 current = None
             # subtiers/*_start and subtiers/*_finish, under subtiers/*, are
             # kept sequential. Others are all forced parallel.
             if current and current.startswith('subtiers/') and all_subdirs and \
                     all_subdirs[0].startswith('subtiers/'):
                 return current, [], all_subdirs
             return current, all_subdirs, []
 
@@ -495,25 +496,27 @@ class RecursiveMakeBackend(CommonBackend
         def compile_filter(current, subdirs):
             current, parallel, sequential = parallel_filter(current, subdirs)
             return current, subdirs.static + parallel, sequential
 
         # Skip tools dirs during libs traversal. Because of bug 925236 and
         # possible other unknown race conditions, don't parallelize the libs
         # tier.
         def libs_filter(current, subdirs):
-            if current in self._may_skip[tier]:
+            if current in self._may_skip[tier] \
+                    or current.startswith('subtiers/'):
                 current = None
             return current, [], subdirs.parallel + \
                 subdirs.dirs + subdirs.tests
 
         # Because of bug 925236 and possible other unknown race conditions,
         # don't parallelize the tools tier.
         def tools_filter(current, subdirs):
-            if current in self._may_skip[tier]:
+            if current in self._may_skip[tier] \
+                    or current.startswith('subtiers/'):
                 current = None
             return current, subdirs.parallel, \
                 subdirs.dirs + subdirs.tests + subdirs.tools
 
         # compile, binaries and tools tiers use the same traversal as export
         filters = {
             'export': parallel_filter,
             'compile': compile_filter,
@@ -524,62 +527,29 @@ class RecursiveMakeBackend(CommonBackend
 
         root_deps_mk = Makefile()
 
         # Fill the dependencies for traversal of each tier.
         for tier, filter in filters.items():
             main, all_deps = \
                 self._traversal.compute_dependencies(filter)
             for dir, deps in all_deps.items():
-                rule = root_deps_mk.create_rule(['%s/%s' % (dir, tier)])
                 if deps is not None:
+                    rule = root_deps_mk.create_rule(['%s/%s' % (dir, tier)])
                     rule.add_dependencies('%s/%s' % (d, tier) for d in deps if d)
             rule = root_deps_mk.create_rule(['recurse_%s' % tier])
             if main:
                 rule.add_dependencies('%s/%s' % (d, tier) for d in main)
 
         root_mk = Makefile()
 
         # Fill root.mk with the convenience variables.
-        for tier, filter in filters.items() + [('all', self._traversal.default_filter)]:
-            # Gather filtered subtiers for the given tier
-            all_direct_subdirs = reduce(lambda x, y: x + y,
-                                        self._traversal.get_subdirs(''), [])
-            direct_subdirs = [d for d in all_direct_subdirs
-                              if filter(d, self._traversal.get_subdirs(d))[0]]
-            subtiers = [d.replace('subtiers/', '') for d in direct_subdirs
-                        if d.startswith('subtiers/')]
-
-            if tier != 'all':
-                # Gather filtered directories for the given tier
-                dirs = [d for d in direct_subdirs if not d.startswith('subtiers/')]
-                if dirs:
-                    # For build systems without tiers (js/src), output a list
-                    # of directories for each tier.
-                    all_dirs = self._traversal.traverse('', filter)
-                    root_mk.add_statement('%s_dirs := %s' % (tier, ' '.join(all_dirs)))
-                    continue
-                if subtiers:
-                    # Output the list of filtered subtiers for the given tier.
-                    root_mk.add_statement('%s_subtiers := %s' % (tier, ' '.join(subtiers)))
-
-            for subtier in subtiers:
-                # subtier_dirs[0] is 'subtiers/%s_start' % subtier, skip it
-                subtier_dirs = list(self._traversal.traverse('subtiers/%s_start' % subtier, filter))[1:]
-                if tier == 'all':
-                    for dir in subtier_dirs:
-                        # Output convenience variables to be able to map directories
-                        # to subtier names from Makefiles.
-                        stamped = dir.replace('/', '_')
-                        root_mk.add_statement('subtier_of_%s := %s' % (stamped, subtier))
-
-                else:
-                    # Output the list of filtered directories for each tier/subtier
-                    # pair.
-                    root_mk.add_statement('%s_subtier_%s := %s' % (tier, subtier, ' '.join(subtier_dirs)))
+        for tier, filter in filters.items():
+            all_dirs = self._traversal.traverse('', filter)
+            root_mk.add_statement('%s_dirs := %s' % (tier, ' '.join(all_dirs)))
 
         root_mk.add_statement('$(call include_deps,root-deps.mk)')
 
         with self._write_file(
                 mozpath.join(self.environment.topobjdir, 'root.mk')) as root:
             root_mk.dump(root, removal_guard=False)
 
         with self._write_file(
@@ -794,42 +764,34 @@ class RecursiveMakeBackend(CommonBackend
         def relativize(dirs):
             return [mozpath.normpath(mozpath.join(backend_file.relobjdir, d))
                 for d in dirs]
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
             # For pseudo derecursification, subtiers are treated as pseudo
             # directories, with a special hierarchy:
-            # - subtier1 - subtier1_start - dirA - dirAA
-            # |          |                |      + dirAB
-            # |          |                ...
-            # |          |                + dirB
-            # |          + subtier1_finish
-            # + subtier2 - subtier2_start ...
-            # ...        + subtier2_finish
-            self._traversal.add('subtiers/%s' % tier,
-                                dirs=['subtiers/%s_start' % tier,
-                                      'subtiers/%s_finish' % tier])
-
+            # - subtier1 + dirA - dirAA
+            # |          |      + dirAB
+            # |          ...
+            # |          + dirB
+            # + subtier2 ...
             if dirs:
                 fh.write('tier_%s_dirs += %s\n' % (tier, ' '.join(dirs)))
                 fh.write('DIRS += $(tier_%s_dirs)\n' % tier)
-                self._traversal.add('subtiers/%s_start' % tier,
+                self._traversal.add('subtiers/%s' % tier,
                                     dirs=relativize(dirs))
 
             # tier_static_dirs should have the same keys as tier_dirs.
             if obj.tier_static_dirs[tier]:
-                fh.write('tier_%s_staticdirs += %s\n' % (
-                    tier, ' '.join(obj.tier_static_dirs[tier])))
-                self._traversal.add('subtiers/%s_start' % tier,
+                fh.write('staticdirs += %s\n' % (
+                    ' '.join(obj.tier_static_dirs[tier])))
+                self._traversal.add('subtiers/%s' % tier,
                                     static=relativize(obj.tier_static_dirs[tier]))
 
-            self._traversal.add('subtiers/%s_start' % tier)
-            self._traversal.add('subtiers/%s_finish' % tier)
             self._traversal.add('', dirs=['subtiers/%s' % tier])
 
         if obj.dirs:
             fh.write('DIRS := %s\n' % ' '.join(obj.dirs))
             self._traversal.add(backend_file.relobjdir, dirs=relativize(obj.dirs))
 
         if obj.parallel_dirs:
             fh.write('PARALLEL_DIRS := %s\n' % ' '.join(obj.parallel_dirs))