Bug 1027054 - Use different logic to adjust amount of instances to be started for jacuzzied slaves. r=catlee
authorRail Aliiev <rail@mozilla.com>
Wed, 18 Jun 2014 10:46:32 -0400
changeset 452 65465eaa3ccde58a10ca1846ca9f252c4bc58898
parent 451 c87f6652a4ed87e4a584080f492560a9fb5c35c0
child 453 f54b46cd7ea894663b28d7c579b33f65079a7311
push id447
push userraliiev@mozilla.com
push dateWed, 18 Jun 2014 14:46:36 +0000
reviewerscatlee
bugs1027054
Bug 1027054 - Use different logic to adjust amount of instances to be started for jacuzzied slaves. r=catlee
scripts/aws_watch_pending.py
--- a/scripts/aws_watch_pending.py
+++ b/scripts/aws_watch_pending.py
@@ -35,16 +35,17 @@ from cloudtools.jacuzzi import get_alloc
 from cloudtools.aws.ami import get_ami
 from cloudtools.aws.vpc import get_avail_subnet
 
 log = logging.getLogger()
 
 # Number of seconds from an instance's launch time for it to be considered
 # 'fresh'
 FRESH_INSTANCE_DELAY = 20 * 60
+FRESH_INSTANCE_DELAY_JACUZZI = 10 * 60
 
 
 def find_pending(db):
     inspector = Inspector(db)
     # Newer buildbot has a "buildrequest_claims" table
     if "buildrequest_claims" in inspector.get_table_names():
         query = sa.text("""
         SELECT buildername, id FROM
@@ -403,25 +404,33 @@ def aws_watch_pending(dburl, regions, bu
             running = aws_get_slaveset_instances(running, slaveset)
             # Filter by create_type
             if create_type == 'spot':
                 running = aws_get_spot_instances(running)
             else:
                 running = aws_get_ondemand_instances(running)
 
             # Get instances launched recently
-            fresh = aws_get_fresh_instances(running, time.time() - FRESH_INSTANCE_DELAY)
+            if slaveset:
+                # jaccuzied slaves, use shorter delay
+                fresh = aws_get_fresh_instances(running, time.time() - FRESH_INSTANCE_DELAY_JACUZZI)
+            else:
+                fresh = aws_get_fresh_instances(running, time.time() - FRESH_INSTANCE_DELAY)
             log.info("%i running for %s %s %s (%i fresh)", len(running), create_type, moz_instance_type, slaveset, len(fresh))
             # TODO: This logic is probably too simple
             # Reduce the number of required slaves by the number of freshly
             # started instaces, plus 10% of those that have been running a
             # while
             num_fresh = len(fresh)
-            num_old = len(running) - num_fresh
-            delta = num_fresh + (num_old / 10)
+            # reduce number of required slaves by number of fresh instances
+            delta = num_fresh
+            if not slaveset:
+                # if not in jacuzzi, reduce by 10% of already running instances
+                num_old = len(running) - num_fresh
+                delta += num_old / 10
             log.info("reducing required count for %s %s %s by %i (%i running; need %i)", create_type, moz_instance_type, slaveset, delta, len(running), count)
             d[moz_instance_type, slaveset] = max(0, count - delta)
             if d[moz_instance_type, slaveset] == 0:
                 log.info("removing requirement for %s %s %s", create_type, moz_instance_type, slaveset)
                 to_delete.add((moz_instance_type, slaveset))
 
             # If slaveset is not None, and all our slaves are running, we should
             # remove it from the set of things to try and start instances for