Bug 915535 - Remove directory tracking from build system. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 27 Mar 2014 10:36:16 +0900
changeset 175594 870f0aa64918823d4730d3b66de89a6c6ac06b17
parent 175593 77ca7f1eda031c512c1813a2245309a7378acb7d
child 175595 f204394cac0c804a3f48d55c172246fe4e47d2cb
push id26494
push usercbook@mozilla.com
push dateThu, 27 Mar 2014 13:09:48 +0000
treeherdermozilla-central@d2ecc6d31622 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs915535
milestone31.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 915535 - Remove directory tracking from build system. r=gps It's not used except with pymake, anyways.
config/recurse.mk
config/rules.mk
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozbuild/resources/html-build-viewer/index.html
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -72,42 +72,36 @@ CURRENT_DIRS := $(call TIER_DIRS,$(CURRE
 
 ifneq (,$(filter binaries libs,$(CURRENT_TIER)))
 WANT_STAMPS = 1
 STAMP_TOUCH = $(TOUCH) $(@D)/binaries
 endif
 
 # Subtier delimiter rules
 $(addprefix subtiers/,$(addsuffix _start/$(CURRENT_TIER),$(CURRENT_SUBTIERS))): subtiers/%_start/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,subtiers/%_start))
-	$(call BUILDSTATUS,SUBTIER_START $(CURRENT_TIER) $* $(if $(BUG_915535_FIXED),$($(CURRENT_TIER)_subtier_$*)))
+	$(call BUILDSTATUS,SUBTIER_START $(CURRENT_TIER) $*)
 	@$(STAMP_TOUCH)
 
 $(addprefix subtiers/,$(addsuffix _finish/$(CURRENT_TIER),$(CURRENT_SUBTIERS))): subtiers/%_finish/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,subtiers/%_finish))
 	$(call BUILDSTATUS,SUBTIER_FINISH $(CURRENT_TIER) $*)
 	@$(STAMP_TOUCH)
 
 $(addprefix subtiers/,$(addsuffix /$(CURRENT_TIER),$(CURRENT_SUBTIERS))): %/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,%))
 	@$(STAMP_TOUCH)
 
 GARBAGE_DIRS += subtiers
 
 # Recursion rule for all directories traversed for all subtiers in the
 # current tier.
 # root.mk defines subtier_of_* variables, that map a normalized subdir path to
 # a subtier name (e.g. subtier_of_memory_jemalloc = base)
 $(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER):
-ifdef BUG_915535_FIXED
-	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
-endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 # Ensure existing stamps are up-to-date, but don't create one if submake didn't create one.
 	$(if $(wildcard $@),@$(STAMP_TOUCH))
-ifdef BUG_915535_FIXED
-	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
-endif
 
 # Dummy rules for possibly inexisting dependencies for the above tier targets
 $(addsuffix /Makefile,$(CURRENT_DIRS)) $(addsuffix /backend.mk,$(CURRENT_DIRS)):
 
 # The export tier requires nsinstall, which is built from config. So every
 # subdirectory traversal needs to happen after traversing config.
 ifeq ($(CURRENT_TIER),export)
 $(addsuffix /$(CURRENT_TIER),$(filter-out config,$(CURRENT_DIRS))): config/$(CURRENT_TIER)
@@ -155,17 +149,17 @@ else
 # Tier traversal handling
 #########################
 
 ifdef TIERS
 
 libs export tools::
 	$(call BUILDSTATUS,TIER_START $@ $(filter-out $(if $(filter export,$@),,precompile),$(TIERS)))
 	$(foreach tier,$(TIERS), $(if $(filter-out libs_precompile tools_precompile,$@_$(tier)), \
-		$(call BUILDSTATUS,SUBTIER_START $@ $(tier) $(if $(filter libs,$@),$(tier_$(tier)_staticdirs)) $(tier_$(tier)_dirs)) \
+		$(call BUILDSTATUS,SUBTIER_START $@ $(tier)) \
 		$(if $(filter libs,$@),$(foreach dir, $(tier_$(tier)_staticdirs), $(call TIER_DIR_SUBMAKE,$@,$(tier),$(dir),,1))) \
 		$(foreach dir, $(tier_$(tier)_dirs), $(call TIER_DIR_SUBMAKE,$@,$(tier),$(dir),$@)) \
 		$(call BUILDSTATUS,SUBTIER_FINISH $@ $(tier))))
 	$(call BUILDSTATUS,TIER_FINISH $@)
 
 else
 
 define CREATE_SUBTIER_TRAVERSAL_RULE
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -371,19 +371,17 @@ endif
 
 define SUBMAKE # $(call SUBMAKE,target,directory,static)
 +@$(UPDATE_TITLE)
 +$(MAKE) $(if $(2),-C $(2)) $(1)
 
 endef # The extra line is important here! don't delete it
 
 define TIER_DIR_SUBMAKE
-$(call BUILDSTATUS,TIERDIR_START  $(1) $(2) $(3))
 $(call SUBMAKE,$(4),$(3),$(5))
-$(call BUILDSTATUS,TIERDIR_FINISH $(1) $(2) $(3))
 
 endef # Ths empty line is important.
 
 ifneq (,$(strip $(DIRS)))
 LOOP_OVER_DIRS = \
   $(foreach dir,$(DIRS),$(call SUBMAKE,$@,$(dir)))
 endif
 
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -58,27 +58,23 @@ BuildOutputResult = namedtuple('BuildOut
 class TierStatus(object):
     """Represents the state and progress of tier traversal.
 
     The build system is organized into linear phases called tiers. Each tier
     executes in the order it was defined, 1 at a time.
 
     Tiers can have subtiers. Subtiers can execute in any order. Some subtiers
     execute sequentially. Others are concurrent.
-
-    Subtiers can have directories. Directories can execute in any order, just
-    like subtiers.
     """
 
     def __init__(self, resources):
         """Accepts a SystemResourceMonitor to record results against."""
         self.tiers = OrderedDict()
         self.active_tier = None
         self.active_subtiers = set()
-        self.active_dirs = {}
         self.resources = resources
 
     def set_tiers(self, tiers):
         """Record the set of known tiers."""
         for tier in tiers:
             self.tiers[tier] = dict(
                 begin_time=None,
                 finish_time=None,
@@ -94,93 +90,50 @@ class TierStatus(object):
         t['begin_time'] = time.time()
         self.resources.begin_phase(self._phase(tier))
         for subtier in subtiers:
             t['subtiers'][subtier] = dict(
                 begin_time=None,
                 finish_time=None,
                 duration=None,
                 concurrent=False,
-                dirs=OrderedDict(),
-                dirs_complete=0,
             )
 
         self.active_tier = tier
         self.active_subtiers = set()
-        self.active_dirs = {}
 
     def finish_tier(self, tier):
         """Record that execution of a tier has finished."""
         t = self.tiers[tier]
         t['finish_time'] = time.time()
         t['duration'] = self.resources.finish_phase(self._phase(tier))
 
-    def begin_subtier(self, tier, subtier, dirs):
+    def begin_subtier(self, tier, subtier):
         """Record that execution of a subtier has begun."""
         self.resources.begin_phase(self._phase(tier, subtier))
 
         st = self.tiers[tier]['subtiers'][subtier]
         st['begin_time'] = time.time()
 
-        for d in dirs:
-            st['dirs'][d] = dict(
-                begin_time=None,
-                finish_time=None,
-                duration=None,
-                concurrent=False,
-            )
-
         if self.active_subtiers:
             st['concurrent'] = True
 
         self.active_subtiers.add(subtier)
 
     def finish_subtier(self, tier, subtier):
         """Record that execution of a subtier has finished."""
         st = self.tiers[tier]['subtiers'][subtier]
         st['finish_time'] = time.time()
 
         self.active_subtiers.remove(subtier)
         if self.active_subtiers:
             st['concurrent'] = True
 
-        # A subtier may not have directories.
-        try:
-            del self.active_dirs[subtier]
-        except KeyError:
-            pass
-
         st['duration'] = self.resources.finish_phase(self._phase(tier, subtier))
 
-    def begin_dir(self, tier, subtier, d):
-        """Record that execution of a directory has begun."""
-        self.resources.begin_phase(self._phase(tier, subtier, d))
-        entry = self.tiers[tier]['subtiers'][subtier]['dirs'][d]
-        entry['begin_time'] = time.time()
-
-        self.active_dirs.setdefault(subtier, set()).add(d)
-
-        if len(self.active_dirs[subtier]) > 1:
-            entry['concurrent'] = True
-
-    def finish_dir(self, tier, subtier, d):
-        """Record that execution of a directory has finished."""
-        st = self.tiers[tier]['subtiers'][subtier]
-        st['dirs_complete'] += 1
-
-        entry = st['dirs'][d]
-        entry['finish_time'] = time.time()
-
-        self.active_dirs[subtier].remove(d)
-        if self.active_dirs[subtier]:
-            entry['concurrent'] = True
-
-        entry['duration'] = self.resources.finish_phase(self._phase(tier,
-            subtier, d))
-
     def tier_status(self):
         for tier, state in self.tiers.items():
             active = self.active_tier == tier
             finished = state['finish_time'] is not None
 
             yield tier, active, finished
 
     def current_subtier_status(self):
@@ -188,21 +141,16 @@ class TierStatus(object):
             return
 
         for subtier, state in self.tiers[self.active_tier]['subtiers'].items():
             active = subtier in self.active_subtiers
             finished = state['finish_time'] is not None
 
             yield subtier, active, finished
 
-    def current_dirs_status(self):
-        for subtier, dirs in self.active_dirs.items():
-            st = self.tiers[self.active_tier]['subtiers'][subtier]
-            yield subtier, st['dirs'].keys(), dirs, st['dirs_complete']
-
     def tiered_resource_usage(self):
         """Obtains an object containing resource usage for tiers.
 
         The returned object is suitable for serialization.
         """
         o = []
 
         for tier, state in self.tiers.items():
@@ -218,35 +166,21 @@ class TierStatus(object):
 
             for subtier, state in state['subtiers'].items():
                 st_entry = dict(
                     name=subtier,
                     start=state['begin_time'],
                     end=state['finish_time'],
                     duration=state['duration'],
                     concurrent=state['concurrent'],
-                    dirs=[],
                 )
 
                 self.add_resources_to_dict(st_entry, phase=self._phase(tier,
                     subtier))
 
-                for d, state in state['dirs'].items():
-                    d_entry = dict(
-                        name=d,
-                        start=state['begin_time'],
-                        end=state['finish_time'],
-                        duration=state['duration'],
-                    )
-
-                    self.add_resources_to_dict(d_entry, phase=self._phase(tier,
-                        subtier, d))
-
-                    st_entry['dirs'].append(d_entry)
-
                 t_entry['subtiers'].append(st_entry)
 
             o.append(t_entry)
 
         return o
 
     def add_resources_to_dict(self, entry, start=None, end=None, phase=None):
         """Helper function to append resource information to a dict."""
@@ -271,22 +205,20 @@ class TierStatus(object):
 
             d['cpu_times_fields'] = list(cpu_times._fields)
             d['io_fields'] = list(usage.io._fields)
             d['virt_fields'] = list(usage.virt._fields)
             d['swap_fields'] = list(usage.swap._fields)
 
             return d
 
-    def _phase(self, tier, subtier=None, d=None):
+    def _phase(self, tier, subtier=None):
         parts = [tier]
         if subtier:
             parts.append(subtier)
-        if d:
-            parts.append(d)
 
         return '_'.join(parts)
 
 
 class BuildMonitor(MozbuildObject):
     """Monitors the output of the build."""
 
     def init(self, warnings_path):
@@ -358,27 +290,20 @@ class BuildMonitor(MozbuildObject):
                 tier = args[0]
                 subtiers = args[1:]
                 self.tiers.begin_tier(tier, subtiers)
             elif action == 'TIER_FINISH':
                 tier, = args
                 self.tiers.finish_tier(tier)
             elif action == 'SUBTIER_START':
                 tier, subtier = args[0:2]
-                dirs = args[2:]
-                self.tiers.begin_subtier(tier, subtier, dirs)
+                self.tiers.begin_subtier(tier, subtier)
             elif action == 'SUBTIER_FINISH':
                 tier, subtier = args
                 self.tiers.finish_subtier(tier, subtier)
-            elif action == 'TIERDIR_START':
-                tier, subtier, d = args
-                self.tiers.begin_dir(tier, subtier, d)
-            elif action == 'TIERDIR_FINISH':
-                tier, subtier, d = args
-                self.tiers.finish_dir(tier, subtier, d)
             else:
                 raise Exception('Unknown build status: %s' % action)
 
             return BuildOutputResult(None, update_needed, False)
 
         warning = None
 
         try:
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -157,43 +157,16 @@ class BuildProgressFooter(object):
         for subtier, active, finished in tiers.current_subtier_status():
             if active:
                 parts.extend([('underline_yellow', subtier), ' '])
             elif finished:
                 parts.extend([('green', subtier), ' '])
             else:
                 parts.extend([subtier, ' '])
 
-        if tiers.active_dirs:
-            parts.extend([('bold', 'DIRECTORIES'), ': '])
-            have_dirs = False
-
-            for subtier, all_dirs, active_dirs, complete in tiers.current_dirs_status():
-                if len(all_dirs) < 2:
-                    continue
-
-                have_dirs = True
-
-                parts.extend([
-                    '%02d' % (complete + 1),
-                    '/',
-                    '%02d' % len(all_dirs),
-                    ' ',
-                    '(',
-                ])
-                if active_dirs:
-                    commas = [', '] * (len(active_dirs) - 1)
-                    magenta_dirs = [('magenta', d) for d in active_dirs]
-                    parts.extend(i.next() for i in itertools.cycle((iter(magenta_dirs),
-                                                                    iter(commas))))
-                parts.append(')')
-
-            if not have_dirs:
-                parts = parts[0:-2]
-
         # We don't want to write more characters than the current width of the
         # terminal otherwise wrapping may result in weird behavior. We can't
         # simply truncate the line at terminal width characters because a)
         # non-viewable escape characters count towards the limit and b) we
         # don't want to truncate in the middle of an escape sequence because
         # subsequent output would inherit the escape sequence.
         max_width = self._t.width
         written = 0
--- a/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
+++ b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
@@ -212,32 +212,16 @@ BuildResources.prototype = Object.freeze
     for (var i = 0; i < t.subtiers.length; i++) {
       var entry = t.subtiers[i];
 
       if (entry.name == subtier) {
         return entry;
       }
     }
   },
-
-  getDirectory: function (tier, subtier, directory) {
-    var st = this.getSubtier(tier, subtier);
-
-    if (!st) {
-      return;
-    }
-
-    for (var i = 0; i < st.dirs.length; i++) {
-      var entry = st.dirs[i];
-
-      if (entry.name == directory) {
-        return entry;
-      }
-    }
-  },
 });
 
 function updateResourcesGraph() {
   //var selected = document.getElementById("resourceType");
   //var what = selected[selected.selectedIndex].value;
   var what = "cpu";
 
   renderResources("resource_graph", currentResources, what);
@@ -386,18 +370,18 @@ function renderResources(id, resources, 
     .call(xAxis)
     ;
 
   svg.append("g")
     .attr("class", "y axis")
     .call(yAxis)
     ;
 
-  // Now we render a timeline of sorts of the tiers, subtiers, and directories.
-  // There are 3 rows of rectangles that visualize divisions between the
+  // Now we render a timeline of sorts of the tiers, subtiers.
+  // There are 2 rows of rectangles that visualize divisions between the
   // different items. We use the same x scale as the resource graph so times
   // line up properly.
   svg = d3.select("#" + id).append("svg")
     .attr("width", width + margin.left + margin.right)
     .attr("height", 100 + margin.top + margin.bottom)
     .append("g")
       .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
     ;
@@ -433,73 +417,41 @@ function renderResources(id, resources, 
         .attr("x", x_start)
         .attr("y", 60)
         .attr("height", 30)
         .attr("width", draw_width)
         .attr("class", classes)
         .attr("tier", t)
         .attr("subtier", subtier.name)
         ;
-
-      subtier.dirs.forEach(function (dir) {
-        var x_start = x(dir.start - resources.offset);
-        var x_end = x(dir.end - resources.offset);
-        var draw_width = x_end - x_start;
-
-        var classes = "timeline directory";
-
-        if (draw_width < 6) {
-          classes += " short";
-        }
-
-        svg.append("rect")
-          .attr("x", x_start)
-          .attr("y", 100)
-          .attr("height", 30)
-          .attr("width", draw_width)
-          .attr("class", classes)
-          .attr("tier", t)
-          .attr("subtier", subtier.name)
-          .attr("directory", dir.name)
-          ;
-      });
     });
   });
 
   function getEntry(element) {
     var tier = element.getAttribute("tier");
     var subtier = element.getAttribute("subtier");
-    var directory = element.getAttribute("directory");
 
     var entry = resources.getTier(tier);
     entry.tier = tier;
 
     if (subtier) {
       entry = resources.getSubtier(tier, subtier);
       entry.tier = tier;
       entry.subtier = subtier;
     }
 
-    if (directory) {
-      entry = resources.getDirectory(tier, subtier, directory);
-      entry.tier = tier;
-      entry.subtier = subtier;
-      entry.directory = directory;
-    }
-
     return entry;
   }
 
   d3.selectAll(".timeline")
   .on("mouseenter", function () {
       var entry = getEntry(this);
 
       d3.select("#tt_tier").html(entry.tier);
       d3.select("#tt_subtier").html(entry.subtier || "n/a");
-      d3.select("#tt_directory").html(entry.directory || "n/a");
       d3.select("#tt_duration").html(entry.duration || "n/a");
       d3.select("#tt_cpu_percent").html(entry.cpu_percent || "n/a");
 
       d3.select("#tooltip").style("display", "");
     })
     .on("mouseleave", function () {
       var tooltip = d3.select("#tooltip");
       tooltip.style("display", "none");
@@ -532,17 +484,16 @@ document.addEventListener("DOMContentLoa
 
     </script>
     <h3>Build Resource Usage Report</h3>
 
     <div id="tooltip" style="display: none;">
       <table border="0">
         <tr><td>Tier</td><td id="tt_tier"></td></tr>
         <tr><td>Subtier</td><td id="tt_subtier"></td></tr>
-        <tr><td>Directory</td><td id="tt_directory"></td></tr>
         <tr><td>Duration</td><td id="tt_duration"></td></tr>
         <tr><td>CPU %</td><td id="tt_cpu_percent"></td></tr>
       </table>
     </div>
 
     <!--
     <select id="resourceType" onchange="updateResourcesGraph();">
       <option value="cpu">CPU</option>