Author: tmckay
Date: 2013-03-12 21:36:19 +0000 (Tue, 12 Mar 2013)
New Revision: 5737
Modified:
trunk/mint/python/mint/aviary/collector.py
trunk/sage/python/sage/aviary/aviaryoperations.py
Log:
Switch slot query to use paging
Modified: trunk/mint/python/mint/aviary/collector.py
===================================================================
--- trunk/mint/python/mint/aviary/collector.py 2013-03-11 13:38:20 UTC (rev 5736)
+++ trunk/mint/python/mint/aviary/collector.py 2013-03-12 21:36:19 UTC (rev 5737)
@@ -96,7 +96,8 @@
if self.no_summary:
# We're going to get everything from classads, we
# did not ask for summaries
- ids = [o.id for o in objs if o.status.code == "OK"]
+ ids = [o.id for o in objs \
+ if not hasattr(o, "status_code") or o.status.code ==
"OK"]
else:
# We'll get things from classads and/or summaries.
# Save the summaries in a dictionary so we can look them up
@@ -115,7 +116,8 @@
log.debug("Queueing updates, summary based")
for i in ids:
update = self.make_update(i, [], summaries)
- self.ep.app.update_thread.enqueue(update)
+ if update:
+ self.ep.app.update_thread.enqueue(update)
else:
# get the classad in addition to the ids and summaries
log.debug("Gathering attributes")
@@ -128,7 +130,8 @@
if ad.status.code == "OK":
update = self.make_update(ad.id,
ad.ad.attrs, summaries)
- self.ep.app.update_thread.enqueue(update)
+ if update:
+ self.ep.app.update_thread.enqueue(update)
# Remember the objects that we have just seen
for id in ids:
@@ -474,6 +477,11 @@
def __init__(self, ep):
super(QuerySlot, self).__init__(ep)
+ # Initialize the name index for paging
+ # We are extending the basic collector endpoint here
+ self.ep.idx = None
+ self.ep.size = 8
+
self.names = ["CondorPlatform",
"CondorVersion",
"DaemonStartTime",
@@ -568,5 +576,23 @@
return SlotObj(self.ep.id, id)
def get_objects(self, no_summary):
- return self.ep.app.aviary_ops.get_slot(self.ep.url, no_summary=no_summary)
+ if self.ep.idx is None:
+ mode = None
+ else:
+ mode = "AFTER"
+ status, ids = self.ep.app.aviary_ops.get_slot_id(self.ep.url,
+ self.ep.size,
+ mode,
+ self.ep.idx,
+ no_summary=no_summary)
+
+ if ids.remaining == 0:
+ self.ep.idx = None
+ else:
+ self.ep.idx = ids.results[-1].id
+
+ # We can't return the whole thing because the caller expects
+ # only a list, not the "remaining"
+ return status, ids.results
+
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2013-03-11 13:38:20 UTC (rev 5736)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2013-03-12 21:36:19 UTC (rev 5737)
@@ -1302,8 +1302,8 @@
def get_master_id(self, host, page_size, mode=None, offset=None):
return self._get_obj_by_id(host, page_size, mode, offset,
"getMasterID")
- def get_slot_id(self, host, page_size, mode=None, offset=None):
- return self._get_obj_by_id(host, page_size, mode, offset, "getSlotID")
+ def get_slot_id(self, host, page_size, mode=None, offset=None, no_summary=False):
+ return self._get_obj_by_id(host, page_size, mode, offset, "getSlotID",
no_summary)
def get_attributes(self, host, ids, names=[]):
if not type(ids) in (list, tuple):
@@ -1340,7 +1340,7 @@
self.client_pool.return_object(client)
return res;
- def _get_obj_by_id(self, host, page_size, mode, offset, meth_name):
+ def _get_obj_by_id(self, host, page_size, mode, offset, meth_name,
no_summary=False):
assert mode is None or mode in ("BEFORE", "AFTER")
@@ -1366,14 +1366,19 @@
self._setup_client(client,
self.servers,
host,
- "getMasterID")
+ meth_name)
+ if no_summary:
+ client.set_enable_attributes(True)
+ client.set_attributes({"includeSummaries": "false"})
+
res = self._call_sync(self.call_client_retry,
my_process_results,
client,
- "getMasterID",
+ meth_name,
page_size, mode, resID)
+ client.set_enable_attributes(False)
self.client_pool.return_object(client)
return res;