Bug 915535 - Remove subtier tracking from build system. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 27 Mar 2014 10:36:25 +0900
changeset 175595 f204394cac0c804a3f48d55c172246fe4e47d2cb
parent 175594 870f0aa64918823d4730d3b66de89a6c6ac06b17
child 175596 e75e1373d6344af456b19db907c0f51ec4b9c300
push id26494
push usercbook@mozilla.com
push dateThu, 27 Mar 2014 13:09:48 +0000
treeherdermozilla-central@d2ecc6d31622 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs915535
milestone31.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 915535 - Remove subtier tracking from build system. r=gps It's currently inaccurate, and soon, the very notion of subtier is going to blow away in the compile/binaries targets.
config/recurse.mk
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozbuild/resources/html-build-viewer/index.html
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -31,17 +31,17 @@ include root.mk
 # In practice this disables it when recursing under js/src, which confuses mach.
 ifndef TIERS
 BUILDSTATUS =
 endif
 
 # Main rules (export, compile, binaries, libs and tools) call recurse_* rules.
 # This wrapping is only really useful for build status.
 compile binaries libs export tools::
-	$(call BUILDSTATUS,TIER_START $@ $($@_subtiers))
+	$(call BUILDSTATUS,TIER_START $@)
 	+$(MAKE) recurse_$@
 	$(call BUILDSTATUS,TIER_FINISH $@)
 
 # Carefully avoid $(eval) type of rule generation, which makes pymake slower
 # than necessary.
 # Get current tier and corresponding subtiers from the data in root.mk.
 CURRENT_TIER := $(filter $(foreach tier,compile binaries libs export tools,recurse_$(tier) $(tier)-deps),$(MAKECMDGOALS))
 ifneq (,$(filter-out 0 1,$(words $(CURRENT_TIER))))
@@ -72,21 +72,19 @@ CURRENT_DIRS := $(call TIER_DIRS,$(CURRE
 
 ifneq (,$(filter binaries libs,$(CURRENT_TIER)))
 WANT_STAMPS = 1
 STAMP_TOUCH = $(TOUCH) $(@D)/binaries
 endif
 
 # Subtier delimiter rules
 $(addprefix subtiers/,$(addsuffix _start/$(CURRENT_TIER),$(CURRENT_SUBTIERS))): subtiers/%_start/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,subtiers/%_start))
-	$(call BUILDSTATUS,SUBTIER_START $(CURRENT_TIER) $*)
 	@$(STAMP_TOUCH)
 
 $(addprefix subtiers/,$(addsuffix _finish/$(CURRENT_TIER),$(CURRENT_SUBTIERS))): subtiers/%_finish/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,subtiers/%_finish))
-	$(call BUILDSTATUS,SUBTIER_FINISH $(CURRENT_TIER) $*)
 	@$(STAMP_TOUCH)
 
 $(addprefix subtiers/,$(addsuffix /$(CURRENT_TIER),$(CURRENT_SUBTIERS))): %/$(CURRENT_TIER): $(if $(WANT_STAMPS),$(call mkdir_deps,%))
 	@$(STAMP_TOUCH)
 
 GARBAGE_DIRS += subtiers
 
 # Recursion rule for all directories traversed for all subtiers in the
@@ -147,22 +145,20 @@ compile binaries libs export tools::
 else
 #########################
 # Tier traversal handling
 #########################
 
 ifdef TIERS
 
 libs export tools::
-	$(call BUILDSTATUS,TIER_START $@ $(filter-out $(if $(filter export,$@),,precompile),$(TIERS)))
+	$(call BUILDSTATUS,TIER_START $@)
 	$(foreach tier,$(TIERS), $(if $(filter-out libs_precompile tools_precompile,$@_$(tier)), \
-		$(call BUILDSTATUS,SUBTIER_START $@ $(tier)) \
 		$(if $(filter libs,$@),$(foreach dir, $(tier_$(tier)_staticdirs), $(call TIER_DIR_SUBMAKE,$@,$(tier),$(dir),,1))) \
 		$(foreach dir, $(tier_$(tier)_dirs), $(call TIER_DIR_SUBMAKE,$@,$(tier),$(dir),$@)) \
-		$(call BUILDSTATUS,SUBTIER_FINISH $@ $(tier))))
 	$(call BUILDSTATUS,TIER_FINISH $@)
 
 else
 
 define CREATE_SUBTIER_TRAVERSAL_RULE
 PARALLEL_DIRS_$(1) = $$(addsuffix _$(1),$$(PARALLEL_DIRS))
 
 .PHONY: $(1) $$(PARALLEL_DIRS_$(1))
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -55,133 +55,71 @@ BuildOutputResult = namedtuple('BuildOut
     ('warning', 'state_changed', 'for_display'))
 
 
 class TierStatus(object):
     """Represents the state and progress of tier traversal.
 
     The build system is organized into linear phases called tiers. Each tier
     executes in the order it was defined, 1 at a time.
-
-    Tiers can have subtiers. Subtiers can execute in any order. Some subtiers
-    execute sequentially. Others are concurrent.
     """
 
     def __init__(self, resources):
         """Accepts a SystemResourceMonitor to record results against."""
         self.tiers = OrderedDict()
         self.active_tier = None
-        self.active_subtiers = set()
         self.resources = resources
 
     def set_tiers(self, tiers):
         """Record the set of known tiers."""
         for tier in tiers:
             self.tiers[tier] = dict(
                 begin_time=None,
                 finish_time=None,
                 duration=None,
-                subtiers=OrderedDict(),
             )
 
-    def begin_tier(self, tier, subtiers):
+    def begin_tier(self, tier):
         """Record that execution of a tier has begun."""
         t = self.tiers[tier]
         # We should ideally use a monotonic clock here. Unfortunately, we won't
         # have one until Python 3.
         t['begin_time'] = time.time()
-        self.resources.begin_phase(self._phase(tier))
-        for subtier in subtiers:
-            t['subtiers'][subtier] = dict(
-                begin_time=None,
-                finish_time=None,
-                duration=None,
-                concurrent=False,
-            )
-
+        self.resources.begin_phase(tier)
         self.active_tier = tier
-        self.active_subtiers = set()
 
     def finish_tier(self, tier):
         """Record that execution of a tier has finished."""
         t = self.tiers[tier]
         t['finish_time'] = time.time()
-        t['duration'] = self.resources.finish_phase(self._phase(tier))
-
-    def begin_subtier(self, tier, subtier):
-        """Record that execution of a subtier has begun."""
-        self.resources.begin_phase(self._phase(tier, subtier))
-
-        st = self.tiers[tier]['subtiers'][subtier]
-        st['begin_time'] = time.time()
-
-        if self.active_subtiers:
-            st['concurrent'] = True
-
-        self.active_subtiers.add(subtier)
-
-    def finish_subtier(self, tier, subtier):
-        """Record that execution of a subtier has finished."""
-        st = self.tiers[tier]['subtiers'][subtier]
-        st['finish_time'] = time.time()
-
-        self.active_subtiers.remove(subtier)
-        if self.active_subtiers:
-            st['concurrent'] = True
-
-        st['duration'] = self.resources.finish_phase(self._phase(tier, subtier))
+        t['duration'] = self.resources.finish_phase(tier)
 
     def tier_status(self):
         for tier, state in self.tiers.items():
             active = self.active_tier == tier
             finished = state['finish_time'] is not None
 
             yield tier, active, finished
 
-    def current_subtier_status(self):
-        if self.active_tier not in self.tiers:
-            return
-
-        for subtier, state in self.tiers[self.active_tier]['subtiers'].items():
-            active = subtier in self.active_subtiers
-            finished = state['finish_time'] is not None
-
-            yield subtier, active, finished
-
     def tiered_resource_usage(self):
         """Obtains an object containing resource usage for tiers.
 
         The returned object is suitable for serialization.
         """
         o = []
 
         for tier, state in self.tiers.items():
             t_entry = dict(
                 name=tier,
                 start=state['begin_time'],
                 end=state['finish_time'],
                 duration=state['duration'],
-                subtiers=[],
             )
 
-            self.add_resources_to_dict(t_entry, phase=self._phase(tier))
-
-            for subtier, state in state['subtiers'].items():
-                st_entry = dict(
-                    name=subtier,
-                    start=state['begin_time'],
-                    end=state['finish_time'],
-                    duration=state['duration'],
-                    concurrent=state['concurrent'],
-                )
-
-                self.add_resources_to_dict(st_entry, phase=self._phase(tier,
-                    subtier))
-
-                t_entry['subtiers'].append(st_entry)
+            self.add_resources_to_dict(t_entry, phase=tier)
 
             o.append(t_entry)
 
         return o
 
     def add_resources_to_dict(self, entry, start=None, end=None, phase=None):
         """Helper function to append resource information to a dict."""
         cpu_percent = self.resources.aggregate_cpu_percent(start=start,
@@ -205,23 +143,16 @@ class TierStatus(object):
 
             d['cpu_times_fields'] = list(cpu_times._fields)
             d['io_fields'] = list(usage.io._fields)
             d['virt_fields'] = list(usage.virt._fields)
             d['swap_fields'] = list(usage.swap._fields)
 
             return d
 
-    def _phase(self, tier, subtier=None):
-        parts = [tier]
-        if subtier:
-            parts.append(subtier)
-
-        return '_'.join(parts)
-
 
 class BuildMonitor(MozbuildObject):
     """Monitors the output of the build."""
 
     def init(self, warnings_path):
         """Create a new monitor.
 
         warnings_path is a path of a warnings database to use.
@@ -283,27 +214,20 @@ class BuildMonitor(MozbuildObject):
             action = args.pop(0)
             update_needed = True
 
             if action == 'TIERS':
                 self.tiers.set_tiers(args)
                 update_needed = False
             elif action == 'TIER_START':
                 tier = args[0]
-                subtiers = args[1:]
-                self.tiers.begin_tier(tier, subtiers)
+                self.tiers.begin_tier(tier)
             elif action == 'TIER_FINISH':
                 tier, = args
                 self.tiers.finish_tier(tier)
-            elif action == 'SUBTIER_START':
-                tier, subtier = args[0:2]
-                self.tiers.begin_subtier(tier, subtier)
-            elif action == 'SUBTIER_FINISH':
-                tier, subtier = args
-                self.tiers.finish_subtier(tier, subtier)
             else:
                 raise Exception('Unknown build status: %s' % action)
 
             return BuildOutputResult(None, update_needed, False)
 
         warning = None
 
         try:
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -148,25 +148,16 @@ class BuildProgressFooter(object):
         for tier, active, finished in tiers.tier_status():
             if active:
                 parts.extend([('underline_yellow', tier), ' '])
             elif finished:
                 parts.extend([('green', tier), ' '])
             else:
                 parts.extend([tier, ' '])
 
-        parts.extend([('bold', 'SUBTIER'), ':', ' '])
-        for subtier, active, finished in tiers.current_subtier_status():
-            if active:
-                parts.extend([('underline_yellow', subtier), ' '])
-            elif finished:
-                parts.extend([('green', subtier), ' '])
-            else:
-                parts.extend([subtier, ' '])
-
         # We don't want to write more characters than the current width of the
         # terminal otherwise wrapping may result in weird behavior. We can't
         # simply truncate the line at terminal width characters because a)
         # non-viewable escape characters count towards the limit and b) we
         # don't want to truncate in the middle of an escape sequence because
         # subsequent output would inherit the escape sequence.
         max_width = self._t.width
         written = 0
--- a/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
+++ b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
@@ -196,32 +196,16 @@ BuildResources.prototype = Object.freeze
     for (var i = 0; i < this.data.tiers.length; i++) {
       var t = this.data.tiers[i];
 
       if (t.name == tier) {
         return t;
       }
     }
   },
-
-  getSubtier: function (tier, subtier) {
-    var t = this.getTier(tier);
-
-    if (!t) {
-      return;
-    }
-
-    for (var i = 0; i < t.subtiers.length; i++) {
-      var entry = t.subtiers[i];
-
-      if (entry.name == subtier) {
-        return entry;
-      }
-    }
-  },
 });
 
 function updateResourcesGraph() {
   //var selected = document.getElementById("resourceType");
   //var what = selected[selected.selectedIndex].value;
   var what = "cpu";
 
   renderResources("resource_graph", currentResources, what);
@@ -370,18 +354,18 @@ function renderResources(id, resources, 
     .call(xAxis)
     ;
 
   svg.append("g")
     .attr("class", "y axis")
     .call(yAxis)
     ;
 
-  // Now we render a timeline of sorts of the tiers, subtiers.
-  // There are 2 rows of rectangles that visualize divisions between the
+  // Now we render a timeline of sorts of the tiers
+  // There is a row of rectangles that visualize divisions between the
   // different items. We use the same x scale as the resource graph so times
   // line up properly.
   svg = d3.select("#" + id).append("svg")
     .attr("width", width + margin.left + margin.right)
     .attr("height", 100 + margin.top + margin.bottom)
     .append("g")
       .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
     ;
@@ -397,61 +381,32 @@ function renderResources(id, resources, 
     svg.append("rect")
       .attr("x", x_start)
       .attr("y", 20)
       .attr("height", 30)
       .attr("width", x_end - x_start)
       .attr("class", "timeline tier")
       .attr("tier", t)
       ;
-
-    tier.subtiers.forEach(function (subtier) {
-      var x_start = x(subtier.start - resources.offset);
-      var x_end = x(subtier.end - resources.offset);
-      var draw_width = x_end - x_start;
-
-      var classes = "timeline subtier";
-      if (draw_width < 6) {
-        classes += " short";
-      }
-
-      svg.append("rect")
-        .attr("x", x_start)
-        .attr("y", 60)
-        .attr("height", 30)
-        .attr("width", draw_width)
-        .attr("class", classes)
-        .attr("tier", t)
-        .attr("subtier", subtier.name)
-        ;
-    });
   });
 
   function getEntry(element) {
     var tier = element.getAttribute("tier");
-    var subtier = element.getAttribute("subtier");
 
     var entry = resources.getTier(tier);
     entry.tier = tier;
 
-    if (subtier) {
-      entry = resources.getSubtier(tier, subtier);
-      entry.tier = tier;
-      entry.subtier = subtier;
-    }
-
     return entry;
   }
 
   d3.selectAll(".timeline")
   .on("mouseenter", function () {
       var entry = getEntry(this);
 
       d3.select("#tt_tier").html(entry.tier);
-      d3.select("#tt_subtier").html(entry.subtier || "n/a");
       d3.select("#tt_duration").html(entry.duration || "n/a");
       d3.select("#tt_cpu_percent").html(entry.cpu_percent || "n/a");
 
       d3.select("#tooltip").style("display", "");
     })
     .on("mouseleave", function () {
       var tooltip = d3.select("#tooltip");
       tooltip.style("display", "none");
@@ -483,17 +438,16 @@ document.addEventListener("DOMContentLoa
 }, false);
 
     </script>
     <h3>Build Resource Usage Report</h3>
 
     <div id="tooltip" style="display: none;">
       <table border="0">
         <tr><td>Tier</td><td id="tt_tier"></td></tr>
-        <tr><td>Subtier</td><td id="tt_subtier"></td></tr>
         <tr><td>Duration</td><td id="tt_duration"></td></tr>
         <tr><td>CPU %</td><td id="tt_cpu_percent"></td></tr>
       </table>
     </div>
 
     <!--
     <select id="resourceType" onchange="updateResourcesGraph();">
       <option value="cpu">CPU</option>