Bug 970738: Integrate jacuzzi allocator with prioritizeBuilders r=bhearsum
authorChris AtLee <catlee@mozilla.com>
Mon, 24 Feb 2014 11:30:03 -0500
changeset 9876 743f092b924b9b20d6c2021de4ce8f93f780f16f
parent 9875 c21b876f060ec3b8cdbe73057ca256f4d07e8e8c
child 9877 9890933910b5c4f1097ec73c830fa8779dcc3106
push id7331
push usercatlee@mozilla.com
push dateMon, 24 Feb 2014 16:30:14 +0000
reviewersbhearsum
bugs970738
Bug 970738: Integrate jacuzzi allocator with prioritizeBuilders r=bhearsum
mozilla/builder_master.cfg
mozilla/master_common.py
mozilla/universal_master_sqlite.cfg
--- a/mozilla/builder_master.cfg
+++ b/mozilla/builder_master.cfg
@@ -20,16 +20,18 @@ reload(config)
 reload(thunderbird_config)
 reload(b2g_config)
 reload(master_localconfig)
 reload(build.paths)
 
 from master_common import BuildmasterConfig
 from build.paths import getRealpath
 
+master_common.prioritizeBuilders.check_jacuzzis = True
+
 c = BuildmasterConfig
 for key, value in master_localconfig.BuildmasterConfig.items():
     if key in c:
         c[key].extend(value)
     else:
         c[key] = value
 
 # Create our QueueDir objects
--- a/mozilla/master_common.py
+++ b/mozilla/master_common.py
@@ -174,20 +174,37 @@ def prioritizeBuilders(buildmaster, buil
 
     # Annotate our list of builders with their priority
     builders = map(lambda builder: (builderPriority(builder, requests[builder.name]), builder), builders)
     builders.sort()
     log("prioritized %i builder(s): %s", len(builders), [(p, b.name) for (p, b) in builders])
 
     # For each set of slaves, create a list of (priority, builder) for that set
     # of slaves
+    # If we're checking the jacuzzi allocations, then limit the available
+    # slaves by whatever the jacuzzi allocation is.
+    # If we don't incorporate the jacuzzi allocations here, we could end up
+    # with lower priority builders being discarded below which have available
+    # slaves attached and allocated.
+    from buildbotcustom.misc import J
     builders_by_slaves = {}
     for b in builders:
-        slaves = frozenset(s.slave.slavename for s in b[1].slaves if s.slave.slavename in avail_slaves)
-        builders_by_slaves.setdefault(slaves, []).append(b)
+        slaves = [s for s in b[1].slaves if s.slave.slavename in avail_slaves]
+        if getattr(prioritizeBuilders, 'check_jacuzzis', False):
+            try:
+                # Filter the available slaves through the jacuzzi bubbles..
+                slaves = J.get_slaves(b[1].name, slaves)
+            except Exception:
+                twlog.err("handled exception talking to jacuzzi; trying to carry on")
+
+        if slaves:
+            slaves = frozenset(s.slave.slavename for s in slaves)
+            builders_by_slaves.setdefault(slaves, []).append(b)
+        else:
+            log('removed builder %s with no allocated slaves available' % b[1].name)
     log("assigned into %i slave set(s)", len(builders_by_slaves))
 
     # Find the set of builders with the highest priority for each set of slaves
     # If there are multiple builders with the same priority, keep all of them,
     # but discard builders with lower priority.
     # By removing lower priority builders, we avoid the situation where a slave
     # connects when the master is partway through iterating through the full
     # set of builders and assigns work to lower priority builders while there's
--- a/mozilla/universal_master_sqlite.cfg
+++ b/mozilla/universal_master_sqlite.cfg
@@ -20,16 +20,18 @@ reload(config)
 reload(thunderbird_config)
 reload(b2g_config)
 reload(master_localconfig)
 reload(build.paths)
 
 from master_common import BuildmasterConfig
 from build.paths import getRealpath
 
+master_common.prioritizeBuilders.check_jacuzzis = True
+
 c = BuildmasterConfig
 for key, value in master_localconfig.BuildmasterConfig.items():
     if key in c:
         c[key].extend(value)
     else:
         c[key] = value
 
 # Create our QueueDir objects