r5341 - trunk/cumin/python/cumin/grid
by croberts@fedoraproject.org
Author: croberts
Date: 2012-04-23 16:34:12 +0000 (Mon, 23 Apr 2012)
New Revision: 5341
Modified:
trunk/cumin/python/cumin/grid/slot.py
Log:
Removing reference to charts.py
Modified: trunk/cumin/python/cumin/grid/slot.py
===================================================================
--- trunk/cumin/python/cumin/grid/slot.py 2012-04-23 16:09:53 UTC (rev 5340)
+++ trunk/cumin/python/cumin/grid/slot.py 2012-04-23 16:34:12 UTC (rev 5341)
@@ -10,7 +10,6 @@
from cumin.stat import *
from cumin.widgets import *
from cumin.OpenFlashChart import *
-from cumin.charts import *
strings = StringCatalog(__file__)
log = logging.getLogger("cumin.slot")
12 years
r5340 - trunk/cumin/python/cumin
by croberts@fedoraproject.org
Author: croberts
Date: 2012-04-23 16:09:53 +0000 (Mon, 23 Apr 2012)
New Revision: 5340
Removed:
trunk/cumin/python/cumin/charts.py
Log:
Removing charts.py, no longer used
Deleted: trunk/cumin/python/cumin/charts.py
===================================================================
--- trunk/cumin/python/cumin/charts.py 2012-04-23 14:44:21 UTC (rev 5339)
+++ trunk/cumin/python/cumin/charts.py 2012-04-23 16:09:53 UTC (rev 5340)
@@ -1,139 +0,0 @@
-import logging
-
-from cairo import *
-from random import random
-from time import time
-from math import sqrt, pi, ceil
-
-from formats import *
-from util import *
-
-log = logging.getLogger("cumin.chart")
-
-
-class HeatMapChart(object):
- def __init__(self, width=400, height=400):
- self.width = width # final width
- self.height = height
- self.max_width = width # max width
- self.max_height = height
- self.cols = 0 # columns
- self.rows = 0
- self.max_size = 28 # size of each slot
- self.surface = None
- self.gap = 3
-
- def plot_colored_shape(self, interior, shape, width, height):
- surface = ImageSurface(FORMAT_ARGB32, int(width), int(height))
- cr = Context(surface)
- cr.set_line_width(1)
-
- cr.set_source_rgba(1.0, 1.0, 1.0, 1.0)
- cr.rectangle(0, 0, width - 1, height - 1)
- cr.fill()
-
- cr.set_line_width(2)
- self._plot_shape(cr, interior, shape, 0, 0, width, height)
- return surface
-
- def plot_colored_rect(self, interior, width, height):
- surface = ImageSurface(FORMAT_ARGB32, int(width), int(height))
- cr = Context(surface)
- cr.set_line_width(1)
-
- cr.set_source_rgb(*interior)
- cr.rectangle(0, 0, width - 1, height - 1)
- cr.fill()
- return surface
-
- def plot_state(self, state, width, height):
- surface = ImageSurface(FORMAT_ARGB32, int(width), int(height))
- cr = Context(surface)
- cr.set_line_width(2)
-
- self._plot_square(cr, (0.7, 0.7, 0.7), state, 0, 0, width, height)
- return surface
-
- def _plot_square(self, cr, colors, state, x, y, width, height):
- if state == "Unclaimed":
- shape = "box"
- elif state == "Claimed":
- shape = "solid"
- elif state in ("Owner", "Unavailable"):
- shape = "diagonal"
- elif state in ("Matched", "Preempting", "Preempting/Matched"):
- shape = "filled_triangle"
- else:
- shape = "box"
-
- self._plot_shape(cr, colors, shape, x, y, width, height)
-
- def _plot_shape(self, cr, colors, shape, x, y, width, height):
- cr.set_source_rgb(*colors)
- #cr.move_to(x, y)
- cr.rectangle(x, y, width - self.gap, height - self.gap)
-
- if shape == "box": # leave empty
- cr.stroke()
- elif shape == "solid": # solid fill
- cr.fill()
- elif shape == "diagonal": # diagonal line
- cr.move_to(x, y)
- cr.line_to(x + width - self.gap, y + height - self.gap)
- cr.stroke()
- elif shape == "filled_triangle": # triangle
- cr.stroke()
- cr.move_to(x + width - self.gap, y)
- cr.line_to(x, y + height - self.gap)
- cr.line_to(x + width - self.gap, y + height - self.gap)
- cr.close_path()
- cr.fill()
-
- def plot_shapes(self, shape_info, zl):
- count = len(shape_info)
- slot_size = self.slot_size(count, zl)
- x = 0
- y = 0
- col = 0
- self.rows = 1
- # the width and height depend on the number of slots
- self.surface = ImageSurface(FORMAT_ARGB32, int(self.width), int(self.height))
- cr = Context(self.surface)
-
- cr.set_line_width( zl == 1 and 1 or 2 )
-
- for slot in shape_info:
- interior = slot["color"]
- shape = slot["shape"]
-
- if col >= self.cols:
- x = 0
- y = y + slot_size
- col = 0
- self.rows = self.rows + 1
-
- self._plot_shape(cr, interior, shape, x, y, slot_size, slot_size)
-
- x = x + slot_size
- col = col + 1
- return slot_size
-
- def slot_size(self, count, zoom=1):
- sq = sqrt(count)
- cols = int(sq)
- if sq > cols:
- cols = cols + 1
- size = int(self.width * zoom / cols)
- if size < 2:
- size = 2
- if size > self.max_size * zoom:
- size = self.max_size * zoom
-
- self.width = int((size * cols) + 1)
- self.height = int((ceil(count * 1.0 / cols) * size) + 1)
- self.cols = cols
- return size
-
- def write(self, writer):
- self.surface.write_to_png(writer)
-
12 years
r5339 - in trunk: cumin/python/cumin wooly/python/wooly
by croberts@fedoraproject.org
Author: croberts
Date: 2012-04-23 14:44:21 +0000 (Mon, 23 Apr 2012)
New Revision: 5339
Modified:
trunk/cumin/python/cumin/stat.strings
trunk/wooly/python/wooly/pages.strings
Log:
Removing references to the no-longer-used swfobject.js.
Modified: trunk/cumin/python/cumin/stat.strings
===================================================================
--- trunk/cumin/python/cumin/stat.strings 2012-04-20 20:18:52 UTC (rev 5338)
+++ trunk/cumin/python/cumin/stat.strings 2012-04-23 14:44:21 UTC (rev 5339)
@@ -368,26 +368,6 @@
</div>
YOU ARE HERE!
</div>
-<script type="text/javascript">
-//<![CDATA[
- var flashversion = swfobject.getFlashPlayerVersion();
- if (flashversion.major < 9) {
- wooly.addPageUpdateListener(function () {if (document.images["{id}"].className == "Loading") return; cumin.updateChart("{id}");});
- cumin.setupChart('{id}', '{fullpage_href}');
- } else {
- cumin.forceChart = function () {
- var chart = cumin.getChart('{id}');
- if (chart) {
- chart.height = window.getSize().y * 0.96;
- }
- }
- swfobject.embedSWF("resource?name=open-flash-chart.swf", "{id}_chart", "96%", "{height}", "9.0.0", "",
- {"data-file":"{href}"}, {wmode: "opaque"}, null, cumin.forceChart);
- wooly.addPageUpdateListener(function () { cumin.updateFlashChart('{id}'); });
- window.addEvent('resize', cumin.forceChart);
- }
-//]]>
-</script>
[PieFlashChart.html]
<div class="StatValueChart" id="{id}">
@@ -395,13 +375,3 @@
<div id="{id}_chart">
</div>
</div>
-<script type="text/javascript">
-//<![CDATA[
- var flashversion = swfobject.getFlashPlayerVersion();
- if (flashversion.major >= 9) {
- swfobject.embedSWF("resource?name=open-flash-chart.swf", "{id}_chart", "{width}", "{height}", "9.0.0", "",
- {"data-file":"{href}"}, {wmode: 'opaque'});
- }
-//]]>
-</script>
-
Modified: trunk/wooly/python/wooly/pages.strings
===================================================================
--- trunk/wooly/python/wooly/pages.strings 2012-04-20 20:18:52 UTC (rev 5338)
+++ trunk/wooly/python/wooly/pages.strings 2012-04-23 14:44:21 UTC (rev 5339)
@@ -18,7 +18,6 @@
<script type="text/javascript" src="resource?name=wooly.js"> </script>
<script type="text/javascript" src="resource?name=app.js"> </script>
<script type="text/javascript" src="{base_name}.js"> </script>
- <script type="text/javascript" src="resource?name=swfobject.js"></script>
<!--[if lt IE 9]><script language="javascript" type="text/javascript" src="resource?name=excanvas.js"></script><![endif]-->
<script type="text/javascript" src="resource?name=jquery.js"></script>
<script type="text/javascript" src="resource?name=jquery.jqplot.js"></script>
12 years
r5338 - trunk/cumin/model/plumage
by tmckay@fedoraproject.org
Author: tmckay
Date: 2012-04-20 20:18:52 +0000 (Fri, 20 Apr 2012)
New Revision: 5338
Modified:
trunk/cumin/model/plumage/rosemary.xml
Log:
Correct persistent data setting
Modified: trunk/cumin/model/plumage/rosemary.xml
===================================================================
--- trunk/cumin/model/plumage/rosemary.xml 2012-04-20 20:18:25 UTC (rev 5337)
+++ trunk/cumin/model/plumage/rosemary.xml 2012-04-20 20:18:52 UTC (rev 5338)
@@ -1,5 +1,5 @@
<model>
- <package name="com.redhat.grid.plumage" persistent="time">
+ <package name="com.redhat.grid.plumage" persistent="timestamp">
<class name="OSUtil">
<source>
<database>condor_stats</database>
12 years
r5337 - trunk/mint/python/mint/plumage
by croberts@fedoraproject.org
Author: croberts
Date: 2012-04-20 20:18:25 +0000 (Fri, 20 Apr 2012)
New Revision: 5337
Modified:
trunk/mint/python/mint/plumage/REPORTING-README
Log:
correcting data purge command.
Modified: trunk/mint/python/mint/plumage/REPORTING-README
===================================================================
--- trunk/mint/python/mint/plumage/REPORTING-README 2012-04-20 20:17:40 UTC (rev 5336)
+++ trunk/mint/python/mint/plumage/REPORTING-README 2012-04-20 20:18:25 UTC (rev 5337)
@@ -18,7 +18,7 @@
Any data that has been loaded will immediately show up in the charts in the cumin UI. There are threads that maintain an archive load (starting with current records and moving back in time) and another thread that loads current records every 5 min.
Note: If you want to quickly purge all of the reporting data, you can run the following command:
-cumin-report --del-on-start --init-only
+cumin-report --delete-on-start --init-only
Dependencies:
12 years
r5336 - trunk/mint/python/mint/plumage
by croberts@fedoraproject.org
Author: croberts
Date: 2012-04-20 20:17:40 +0000 (Fri, 20 Apr 2012)
New Revision: 5336
Modified:
trunk/mint/python/mint/plumage/REPORTING-README
Log:
Adding note regarding cumin-report data purge.
Modified: trunk/mint/python/mint/plumage/REPORTING-README
===================================================================
--- trunk/mint/python/mint/plumage/REPORTING-README 2012-04-20 20:15:05 UTC (rev 5335)
+++ trunk/mint/python/mint/plumage/REPORTING-README 2012-04-20 20:17:40 UTC (rev 5336)
@@ -17,6 +17,9 @@
cumin-report pulls data from the ODS (mongoDB) in the background. A full data load can be millions and millions of records and could take a considerable amount of time.
Any data that has been loaded will immediately show up in the charts in the cumin UI. There are threads that maintain an archive load (starting with current records and moving back in time) and another thread that loads current records every 5 min.
+Note: If you want to quickly purge all of the reporting data, you can run the following command:
+cumin-report --del-on-start --init-only
+
Dependencies:
The CuminReporting feature has a dependency on
12 years
r5335 - trunk/mint/python/mint/plumage
by croberts@fedoraproject.org
Author: croberts
Date: 2012-04-20 20:15:05 +0000 (Fri, 20 Apr 2012)
New Revision: 5335
Modified:
trunk/mint/python/mint/plumage/REPORTING-README
Log:
Adding a note regarding expire-threshold to the README file.
Modified: trunk/mint/python/mint/plumage/REPORTING-README
===================================================================
--- trunk/mint/python/mint/plumage/REPORTING-README 2012-04-20 19:59:14 UTC (rev 5334)
+++ trunk/mint/python/mint/plumage/REPORTING-README 2012-04-20 20:15:05 UTC (rev 5335)
@@ -11,6 +11,9 @@
# To enable reporting features, uncomment the following line.
# reports: report
+Also, by default, cumin will expire data samples in the database after 1 day. In order to allow the reporting feature to store data for a longer period of time, the following can be added to the [report] section of cumin.conf.
+expire-threshold: 1y #for 1 year of data retention, 30d would give 30 days, and so on.
+
cumin-report pulls data from the ODS (mongoDB) in the background. A full data load can be millions and millions of records and could take a considerable amount of time.
Any data that has been loaded will immediately show up in the charts in the cumin UI. There are threads that maintain an archive load (starting with current records and moving back in time) and another thread that loads current records every 5 min.
12 years
r5334 - trunk/mint/python/mint/plumage
by croberts@fedoraproject.org
Author: croberts
Date: 2012-04-20 19:59:14 +0000 (Fri, 20 Apr 2012)
New Revision: 5334
Modified:
trunk/mint/python/mint/plumage/session.py
Log:
Removing an unnecessary sleep/log message.
Modified: trunk/mint/python/mint/plumage/session.py
===================================================================
--- trunk/mint/python/mint/plumage/session.py 2012-04-20 19:55:11 UTC (rev 5333)
+++ trunk/mint/python/mint/plumage/session.py 2012-04-20 19:59:14 UTC (rev 5334)
@@ -266,8 +266,6 @@
obj = ObjectUpdate(self.app.model, record, self.cls)
self.app.update_thread.enqueue(obj)
- log.info("PlumageSessionThread--catch-up: pass completed")
- sleep(300)
log.info("CatchUpPlumageSessionThread--catch-up: catch-up run completed for records newer than %s and older than %s" % (newest, datetime.now() - timedelta(seconds=300) + UTC_DIFF))
else:
log.info("CatchUpPlumageSessionThread: Skipping catch-up, no records present (probably first-run)")
12 years
r5333 - trunk/mint/python/mint/plumage
by tmckay@fedoraproject.org
Author: tmckay
Date: 2012-04-20 19:55:11 +0000 (Fri, 20 Apr 2012)
New Revision: 5333
Modified:
trunk/mint/python/mint/plumage/session.py
Log:
Add handlers for failed import, mongo server not up, general errors for graceful failure.
Modified: trunk/mint/python/mint/plumage/session.py
===================================================================
--- trunk/mint/python/mint/plumage/session.py 2012-04-20 18:49:12 UTC (rev 5332)
+++ trunk/mint/python/mint/plumage/session.py 2012-04-20 19:55:11 UTC (rev 5333)
@@ -1,5 +1,4 @@
from __future__ import division
-
import time
from mint.util import MintDaemonThread
@@ -8,12 +7,15 @@
from random import randrange
from datetime import timedelta, datetime
-from pymongo import Connection
-from pymongo.collection import Collection
-from pymongo.database import Database
+try:
+ from pymongo import Connection
+ from pymongo.errors import AutoReconnect
+ from pymongo.collection import Collection
+ from pymongo.database import Database
+ imports_ok = True
+except:
+ imports_ok = False
-from dateutil.parser import *
-
import logging
log = logging.getLogger("mint.plumage.session")
@@ -37,6 +39,9 @@
log.info("Initializing %s", self)
def init_classes(self):
+ if not imports_ok:
+ return
+
# Apply the package filter to the class list
if len(self.app.classes):
black_list = set()
@@ -88,6 +93,9 @@
cls))
def start(self):
log.info("Starting %s", self)
+ if not imports_ok:
+ log.info("!!! Imports failed for pymongo, there will be no data !!!")
+
for t in self.threads:
t.start()
@@ -104,58 +112,68 @@
super(PlumageSessionThread, self).__init__(app)
self.cls = cls
- self.connection = Connection(server_host, server_port) #Needs to be from config
- self.database = Database(self.connection, self.cls.database)
- self.collection = Collection(self.database, self.cls.collection)
+ self.server_host = server_host
+ self.server_port = server_port
+
+ try:
+ self.connection = Connection(server_host, server_port)
+ except AutoReconnect:
+ self.connection = None
+ log.info("%s could not connect to mongo server at %s:%s, "\
+ "autoreconnect is on" % (self.__class__.__name__,
+ server_host,
+ server_port))
+ self.database = None
+ self.collection = None
+
+ def _init(self):
+ if self.database is None:
+ self.database = Database(self.connection, self.cls.database)
+
+ if self.collection is None:
+ self.collection = Collection(self.database, self.cls.collection)
+
+ def _check_connection(self):
+ # Loop waiting for the server if it's not there. Autoreconnect
+ # will hook us up...
+ while not self.stop_requested:
+ try:
+ if self.connection is None:
+ self.connection = Connection(self.server_host,
+ self.server_port)
+ else:
+ self.connection.server_info()
+ break
+ except AutoReconnect, e:
+ log.info("%s waiting for server, %s, sleeping..."\
+ % (self.__class__.__name__, str(e)))
+ for i in range(10):
+ sleep(1)
+ if self.stop_requested:
+ break
def run(self):
+ self._check_connection()
+ if self.stop_requested:
+ return
- # We create objects here. Tag them with the right class,
- # probably specified to us from a config option (with a corresponding
- # query specification in the xml)
- (oldest, newest) = self.app.update_thread.get_first_and_last_sample_timestamp(self.cls)
- if oldest is None:
- # if we have no oldest record (first run), start at "5 min ago" and start loading everything
- oldest = datetime.now() - timedelta(seconds=300)
- oldest = oldest + UTC_DIFF
-
- log.info("PlumageSessionThread--history: Loading records older than %s" % oldest)
- times = sorted(self.collection.find({"ts": {'$lt': oldest}}).distinct('ts'), reverse=True)
- sample_times = map(lambda i: times[i],filter(lambda i: i%5 == 0,range(len(times))))
+ try:
+ # Make sure we have a database
+ self._init()
- for time in sample_times:
- record = OSUtil()
- record.host= "%s:%s" % (self.connection.host, self.connection.port)
- record.total = len(self.collection.find({'ts':time}).distinct('n'))
- record.used = len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
- record.unused = len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
- record.owner = len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
- record.efficiency = (record.used/(record.total-record.owner))*100
- record.ts = time - UTC_DIFF
-
- obj = ObjectUpdate(self.app.model, record, self.cls)
- self.app.update_thread.enqueue(obj)
-
- log.info("PlumageSessionThread--history: run completed")
-
-
-class CurrentPlumageSessionThread(PlumageSessionThread):
- def run(self):
- while True:
- if self.stop_requested:
- break
-
+ # We create objects here. Tag them with the right class,
+ # probably specified to us from a config option (with a corresponding
+ # query specification in the xml)
(oldest, newest) = self.app.update_thread.get_first_and_last_sample_timestamp(self.cls)
- if newest is not None:
- most_recent = max((datetime.now() - timedelta(seconds=300) + UTC_DIFF), (newest + UTC_DIFF))
- else:
- most_recent = datetime.now() - timedelta(seconds=300) + UTC_DIFF
+ if oldest is None:
+ # if we have no oldest record (first run), start at "5 min ago" and start loading everything
+ oldest = datetime.now() - timedelta(seconds=300)
+ oldest = oldest + UTC_DIFF
- log.info("PlumageSessionThread--current: Loading records newer than %s" % most_recent)
+ log.info("PlumageSessionThread--history: Loading records older than %s" % oldest)
+ times = sorted(self.collection.find({"ts": {'$lt': oldest}}).distinct('ts'), reverse=True)
+ sample_times = map(lambda i: times[i],filter(lambda i: i%5 == 0,range(len(times))))
- times = sorted(self.collection.find({"ts": {'$gt': most_recent}}).distinct('ts'), reverse=True)
- sample_times = map(lambda i: times[i],filter(lambda i: i%5 == 0,range(len(times))))
-
for time in sample_times:
record = OSUtil()
record.host= "%s:%s" % (self.connection.host, self.connection.port)
@@ -165,10 +183,53 @@
record.owner = len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
record.efficiency = (record.used/(record.total-record.owner))*100
record.ts = time - UTC_DIFF
-
+
obj = ObjectUpdate(self.app.model, record, self.cls)
self.app.update_thread.enqueue(obj)
+
+ log.info("PlumageSessionThread--history: run completed")
+
+ except Exception, e:
+ log.info("%s got exception %s, exiting" % (self.__class__.__name__, str(e)))
+
+class CurrentPlumageSessionThread(PlumageSessionThread):
+ def run(self):
+ while True:
+ self._check_connection()
+ if self.stop_requested:
+ break
+
+ # Make sure we have a db
+ try:
+ self._init()
+
+ (oldest, newest) = self.app.update_thread.get_first_and_last_sample_timestamp(self.cls)
+ if newest is not None:
+ most_recent = max((datetime.now() - timedelta(seconds=300) + UTC_DIFF), (newest + UTC_DIFF))
+ else:
+ most_recent = datetime.now() - timedelta(seconds=300) + UTC_DIFF
+
+ log.info("PlumageSessionThread--current: Loading records newer than %s" % most_recent)
+
+ times = sorted(self.collection.find({"ts": {'$gt': most_recent}}).distinct('ts'), reverse=True)
+ sample_times = map(lambda i: times[i],filter(lambda i: i%5 == 0,range(len(times))))
+
+ for time in sample_times:
+ record = OSUtil()
+ record.host= "%s:%s" % (self.connection.host, self.connection.port)
+ record.total = len(self.collection.find({'ts':time}).distinct('n'))
+ record.used = len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
+ record.unused = len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
+ record.owner = len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
+ record.efficiency = (record.used/(record.total-record.owner))*100
+ record.ts = time - UTC_DIFF
+
+ obj = ObjectUpdate(self.app.model, record, self.cls)
+ self.app.update_thread.enqueue(obj)
+
log.info("PlumageSessionThread--current: pass completed")
+ except Exception, e:
+ log.info("%s got exception %s, sleeping" % (self.__class__.__name__, str(e)))
sleep(300)
@@ -178,27 +239,37 @@
'''
class CatchUpPlumageSessionThread(PlumageSessionThread):
def run(self):
- (oldest, newest) = self.app.update_thread.get_first_and_last_sample_timestamp(self.cls)
- if newest is not None:
- log.info("CatchUpPlumageSessionThread: Starting for records newer than %s" % newest)
-
- times = sorted(self.collection.find({"ts": {'$gt': newest + UTC_DIFF, '$lt': datetime.now() - timedelta(seconds=300) + UTC_DIFF}}).distinct('ts'), reverse=True)
- sample_times = map(lambda i: times[i],filter(lambda i: i%5 == 0,range(len(times))))
-
- for time in sample_times:
- record = OSUtil()
- record.host= "%s:%s" % (self.connection.host, self.connection.port)
- record.total = len(self.collection.find({'ts':time}).distinct('n'))
- record.used = len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
- record.unused = len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
- record.owner = len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
- record.efficiency = (record.used/(record.total-record.owner))*100
- record.ts = time - UTC_DIFF
-
- obj = ObjectUpdate(self.app.model, record, self.cls)
- self.app.update_thread.enqueue(obj)
- log.info("PlumageSessionThread--catch-up: pass completed")
- sleep(300)
- log.info("CatchUpPlumageSessionThread--catch-up: catch-up run completed for records newer than %s and older than %s" % (newest, datetime.now() - timedelta(seconds=300) + UTC_DIFF))
- else:
- log.info("CatchUpPlumageSessionThread: Skipping catch-up, no records present (probably first-run)")
\ No newline at end of file
+ self._check_connection()
+ if self.stop_requested:
+ return
+
+ try:
+ # Make sure we have a db
+ self._init()
+
+ (oldest, newest) = self.app.update_thread.get_first_and_last_sample_timestamp(self.cls)
+ if newest is not None:
+ log.info("CatchUpPlumageSessionThread: Starting for records newer than %s" % newest)
+
+ times = sorted(self.collection.find({"ts": {'$gt': newest + UTC_DIFF, '$lt': datetime.now() - timedelta(seconds=300) + UTC_DIFF}}).distinct('ts'), reverse=True)
+ sample_times = map(lambda i: times[i],filter(lambda i: i%5 == 0,range(len(times))))
+
+ for time in sample_times:
+ record = OSUtil()
+ record.host= "%s:%s" % (self.connection.host, self.connection.port)
+ record.total = len(self.collection.find({'ts':time}).distinct('n'))
+ record.used = len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
+ record.unused = len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
+ record.owner = len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
+ record.efficiency = (record.used/(record.total-record.owner))*100
+ record.ts = time - UTC_DIFF
+
+ obj = ObjectUpdate(self.app.model, record, self.cls)
+ self.app.update_thread.enqueue(obj)
+ log.info("PlumageSessionThread--catch-up: pass completed")
+ sleep(300)
+ log.info("CatchUpPlumageSessionThread--catch-up: catch-up run completed for records newer than %s and older than %s" % (newest, datetime.now() - timedelta(seconds=300) + UTC_DIFF))
+ else:
+ log.info("CatchUpPlumageSessionThread: Skipping catch-up, no records present (probably first-run)")
+ except Exception, e:
+ log.info("%s got exception %s, exiting" % (self.__class__.__name__, str(e)))
12 years
r5332 - in trunk: cumin/model/plumage cumin/model/upgrades mint/python/mint/plumage
by croberts@fedoraproject.org
Author: croberts
Date: 2012-04-20 18:49:12 +0000 (Fri, 20 Apr 2012)
New Revision: 5332
Modified:
trunk/cumin/model/plumage/plumage.xml
trunk/cumin/model/plumage/rosemary.xml
trunk/cumin/model/upgrades/1.2_to_1.3
trunk/mint/python/mint/plumage/session.py
Log:
Adding the plumage_host as a property of the OSUtil class.
Modified: trunk/cumin/model/plumage/plumage.xml
===================================================================
--- trunk/cumin/model/plumage/plumage.xml 2012-04-20 18:18:31 UTC (rev 5331)
+++ trunk/cumin/model/plumage/plumage.xml 2012-04-20 18:49:12 UTC (rev 5332)
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<schema package="com.redhat.grid.plumage" package_type="generic">
<class name="OSUtil">
+ <property name="host" type="sstr" />
<statistic name="ts" type="absTime" unit="uint32" />
<statistic name="total" type="uint32" uint="uint32" />
<statistic name="used" type="uint32" uint="uint32" />
Modified: trunk/cumin/model/plumage/rosemary.xml
===================================================================
--- trunk/cumin/model/plumage/rosemary.xml 2012-04-20 18:18:31 UTC (rev 5331)
+++ trunk/cumin/model/plumage/rosemary.xml 2012-04-20 18:49:12 UTC (rev 5332)
@@ -6,6 +6,10 @@
<collection>samples.machine</collection>
</source>
+ <property name="host">
+ <title>Host</title>
+ </property>
+
<statistic name="ts" timestamp="y">
</statistic>
Modified: trunk/cumin/model/upgrades/1.2_to_1.3
===================================================================
--- trunk/cumin/model/upgrades/1.2_to_1.3 2012-04-20 18:18:31 UTC (rev 5331)
+++ trunk/cumin/model/upgrades/1.2_to_1.3 2012-04-20 18:49:12 UTC (rev 5332)
@@ -1,5 +1,5 @@
#!/bin/bash
echo 'create schema "com.redhat.grid.plumage"' | psql -d cumin -U cumin -h localhost
echo 'create sequence "com.redhat.grid.plumage"."OSUtil_id_seq"' | psql -d cumin -U cumin -h localhost
-echo 'create table "com.redhat.grid.plumage"."OSUtil" ("_id" int8 not null,"ts" timestamp not null,"total" int8 not null,"used" int8 not null,"unused" int8 not null,"owner" int8 not null,"efficiency" float4 not null,constraint "OSUtil_pk" primary key ("_id"))'| psql -d cumin -U cumin -h localhost
+echo 'create table "com.redhat.grid.plumage"."OSUtil" ("_id" int8 not null,"host" text not null,"ts" timestamp not null,"total" int8 not null,"used" int8 not null,"unused" int8 not null,"owner" int8 not null,"efficiency" float4 not null,constraint "OSUtil_pk" primary key ("_id"))'| psql -d cumin -U cumin -h localhost
echo 'create table "com.redhat.grid.plumage"."OSUtil_samples" ("ts" timestamp not null,"total" int8 not null,"used" int8 not null,"unused" int8 not null,"owner" int8 not null,"efficiency" float4 not null)'| psql -d cumin -U cumin -h localhost
Modified: trunk/mint/python/mint/plumage/session.py
===================================================================
--- trunk/mint/python/mint/plumage/session.py 2012-04-20 18:18:31 UTC (rev 5331)
+++ trunk/mint/python/mint/plumage/session.py 2012-04-20 18:49:12 UTC (rev 5332)
@@ -125,6 +125,7 @@
for time in sample_times:
record = OSUtil()
+ record.host= "%s:%s" % (self.connection.host, self.connection.port)
record.total = len(self.collection.find({'ts':time}).distinct('n'))
record.used = len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
record.unused = len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
@@ -157,6 +158,7 @@
for time in sample_times:
record = OSUtil()
+ record.host= "%s:%s" % (self.connection.host, self.connection.port)
record.total = len(self.collection.find({'ts':time}).distinct('n'))
record.used = len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
record.unused = len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
@@ -167,7 +169,7 @@
obj = ObjectUpdate(self.app.model, record, self.cls)
self.app.update_thread.enqueue(obj)
log.info("PlumageSessionThread--current: pass completed")
- sleep(300)
+ sleep(300)
'''This thread is meant to run once at startup. It will figure out the newest record in the cumin
@@ -185,6 +187,7 @@
for time in sample_times:
record = OSUtil()
+ record.host= "%s:%s" % (self.connection.host, self.connection.port)
record.total = len(self.collection.find({'ts':time}).distinct('n'))
record.used = len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
record.unused = len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
12 years