Author: tmckay
Date: 2013-03-19 19:27:17 +0000 (Tue, 19 Mar 2013)
New Revision: 5748
Modified:
trunk/mint/python/mint/aviary/collector.py
trunk/sage/python/sage/aviary/aviaryoperations.py
Log:
Adding chunking for calls to get_attributes to lessen CPU usage.
Add strip_host flag on BaseCollectorQuery (temporary) that can
be set True to patch against Aviary
Modified: trunk/mint/python/mint/aviary/collector.py
===================================================================
--- trunk/mint/python/mint/aviary/collector.py 2013-03-19 15:04:34 UTC (rev 5747)
+++ trunk/mint/python/mint/aviary/collector.py 2013-03-19 19:27:17 UTC (rev 5748)
@@ -42,7 +42,7 @@
self.ep = ep
self.names = []
-
+ self.strip_host = False
self.objects = dict()
self.recycle_interval = timedelta(seconds=10)
@@ -73,6 +73,25 @@
def get_objects(self, no_summary):
raise Exception("NotImplemented")
+ def get_attributes(self, ids):
+ ads = []
+ final = False
+ stat = False
+ chunk_size = 3
+
+ chunks = [ids[i:i+chunk_size] for i in range(0, len(ids), chunk_size)]
+
+ for chunk in chunks:
+ stat, res = self.ep.app.aviary_ops.get_attributes(self.ep.url,
+ chunk,
+ self.names,
+ self.strip_host)
+ if stat == 0:
+ final = 0
+ ads.extend(res)
+
+ return final, ads
+
def do_the_job(self, foreman):
# If the endpoint has been abandoned, then we should delete the objects
@@ -121,9 +140,7 @@
else:
# get the classad in addition to the ids and summaries
log.debug("Gathering attributes")
- stat, ads = self.ep.app.aviary_ops.get_attributes(self.ep.url,
- ids,
- self.names)
+ stat, ads = self.get_attributes(ids)
if stat == 0:
log.debug("Queueing updates, attribute based")
for ad in ads:
@@ -480,8 +497,10 @@
# Initialize the name index for paging
# We are extending the basic collector endpoint here
self.ep.idx = None
- self.ep.size = 8
+ self.ep.size = 16
+ self.strip_host = False
+
self.names = ["CondorPlatform",
"CondorVersion",
"DaemonStartTime",
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2013-03-19 15:04:34 UTC (rev 5747)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2013-03-19 19:27:17 UTC (rev 5748)
@@ -1305,7 +1305,7 @@
def get_slot_id(self, host, page_size, mode=None, offset=None, no_summary=False):
return self._get_obj_by_id(host, page_size, mode, offset, "getSlotID",
no_summary)
- def get_attributes(self, host, ids, names=[]):
+ def get_attributes(self, host, ids, names=[], strip_host=False):
if not type(ids) in (list, tuple):
ids = [ids]
@@ -1325,8 +1325,15 @@
host,
"getAttributes")
+
+ def fixname(name):
+ if strip_host and "@" in name:
+ name = name[name.find("@")+1:]
+ return name
+
args = []
for id in ids:
+ id.name = fixname(id.name)
args.append({"id":id,
"names": names})