cache results for spot request sanity checks r=rail
authorChris AtLee <catlee@mozilla.com>
Thu, 13 Mar 2014 10:56:31 -0400
changeset 341 a701860945e05fbf56f8b5283ef21fef17b84c1d
parent 340 1b9a08be70fcd3125c8e3220f8f967d8651c64f0
child 342 e037c6715f7c09fba90aa745b7975201cb052c37
push id341
push usercatlee@mozilla.com
push dateThu, 13 Mar 2014 14:56:45 +0000
reviewersrail
cache results for spot request sanity checks r=rail
scripts/aws_watch_pending.py
--- a/scripts/aws_watch_pending.py
+++ b/scripts/aws_watch_pending.py
@@ -2,16 +2,17 @@
 """
 Watches pending jobs and starts or creates EC2 instances if required
 """
 import re
 import time
 import datetime
 import random
 from collections import defaultdict
+from repoze.lru import lru_cache
 
 try:
     import simplejson as json
     assert json
 except ImportError:
     import json
 
 from boto.exception import BotoServerError
@@ -31,35 +32,57 @@ site.addsitedir(os.path.join(os.path.dir
 from cloudtools.aws import get_aws_connection, INSTANCE_CONFIGS_DIR, \
     aws_time_to_datetime
 from cloudtools.aws.spot import CANCEL_STATUS_CODES, \
     TERMINATED_BY_AWS_STATUS_CODES
 
 log = logging.getLogger()
 
 
+@lru_cache(10)
+def get_all_spot_requests(region):
+    log.info("getting all spot requests for %s", region)
+    conn = get_aws_connection(region)
+    spot_requests = conn.get_all_spot_instance_requests()
+    return spot_requests
+
+
+@lru_cache(100)
+def get_spot_requests(region, instance_type, availability_zone):
+    log.info("getting filtered spot requests for %s (%s)", availability_zone, instance_type)
+    all_requests = get_all_spot_requests(region)
+    retval = []
+    if not all_requests:
+        return retval
+
+    for r in all_requests:
+        if r.launch_specification.instance_type != instance_type:
+            continue
+        if r.launched_availability_zone != availability_zone:
+            continue
+        retval.append(r)
+    return retval
+
+
+@lru_cache(100)
 def usable_choice(choice, minutes=15):
     """Sanity check recent spot requests"""
     region = choice.region
     az = choice.availability_zone
     instance_type = choice.instance_type
     bid_price = choice.bid_price
     current_price = choice.current_price
     log.debug("Sanity checking %s in %s", instance_type, az)
 
     # if price is higher than 80% of the bid price do not use the choice
     if current_price > bid_price * 0.8:
         log.debug("Price is higher than 80%% of ours, %s", choice)
         return False
 
-    conn = get_aws_connection(region)
-    filters = {
-        "launch.instance-type": instance_type,
-        "launched-availability-zone": az}
-    spot_requests = conn.get_all_spot_instance_requests(filters=filters)
+    spot_requests = get_spot_requests(region, instance_type, az)
     if not spot_requests:
         log.debug("No available spot requests in last %sm", minutes)
         return True
     # filter out requests older than 15 min
     # first, get the tzinfo of one of the requests
     delta = datetime.timedelta(minutes=minutes)
     recent_spot_requests = []
     for r in spot_requests: