Author: croberts
Date: 2012-05-31 12:18:57 +0000 (Thu, 31 May 2012)
New Revision: 5394
Modified:
trunk/cumin/model/plumage/plumage.xml
trunk/cumin/model/plumage/rosemary.xml
trunk/cumin/model/upgrades/1.2_to_1.3
trunk/mint/python/mint/plumage/session.py
Log:
Changes to cumin-report to extract more data from the plumage database for use in charts.
Modified: trunk/cumin/model/plumage/plumage.xml
===================================================================
--- trunk/cumin/model/plumage/plumage.xml 2012-05-30 19:52:07 UTC (rev 5393)
+++ trunk/cumin/model/plumage/plumage.xml 2012-05-31 12:18:57 UTC (rev 5394)
@@ -7,6 +7,14 @@
<statistic name="used" type="uint32" uint="uint32"
/>
<statistic name="unused" type="uint32" uint="uint32"
/>
<statistic name="owner" type="uint32" uint="uint32"
/>
- <statistic name="efficiency" type="float" />
+ <statistic name="efficiency" type="float" />
+ <statistic name="freemem" type="uint32"
uint="uint32" />
+ <statistic name="usedmem" type="uint32"
uint="uint32" />
+ <statistic name="availmem" type="uint32"
uint="uint32" />
+ <statistic name="totalmem" type="uint32"
uint="uint32" />
+ <statistic name="freecpu" type="uint32"
uint="uint32" />
+ <statistic name="usedcpu" type="uint32"
uint="uint32" />
+ <statistic name="availcpu" type="uint32"
uint="uint32" />
+ <statistic name="totalcpu" type="uint32"
uint="uint32" />
</class>
-</schema>
+</schema>
\ No newline at end of file
Modified: trunk/cumin/model/plumage/rosemary.xml
===================================================================
--- trunk/cumin/model/plumage/rosemary.xml 2012-05-30 19:52:07 UTC (rev 5393)
+++ trunk/cumin/model/plumage/rosemary.xml 2012-05-31 12:18:57 UTC (rev 5394)
@@ -36,7 +36,39 @@
<statistic name="efficiency">
<title>Efficiency percentage</title>
</statistic>
+
+ <statistic name="freemem">
+ <title>Free memory (pool)</title>
+ </statistic>
+
+ <statistic name="usedmem">
+ <title>Used memory (pool)</title>
+ </statistic>
+
+ <statistic name="availmem">
+ <title>Available memory (pool)</title>
+ </statistic>
+
+ <statistic name="totalmem">
+ <title>Total memory (pool)</title>
+ </statistic>
+ <statistic name="freecpu">
+ <title>Free cpus (pool)</title>
+ </statistic>
+
+ <statistic name="usedcpu">
+ <title>Used cpus (pool)</title>
+ </statistic>
+
+ <statistic name="availcpu">
+ <title>Available cpus (pool)</title>
+ </statistic>
+
+ <statistic name="totalcpu">
+ <title>Total cpus (pool)</title>
+ </statistic>
+
</class>
</package>
Modified: trunk/cumin/model/upgrades/1.2_to_1.3
===================================================================
--- trunk/cumin/model/upgrades/1.2_to_1.3 2012-05-30 19:52:07 UTC (rev 5393)
+++ trunk/cumin/model/upgrades/1.2_to_1.3 2012-05-31 12:18:57 UTC (rev 5394)
@@ -3,3 +3,20 @@
echo 'create sequence
"com.redhat.grid.plumage"."OSUtil_id_seq"' | psql -d cumin -U
cumin -h localhost
echo 'create table "com.redhat.grid.plumage"."OSUtil"
("_id" int8 not null,"host" text not null,"ts" timestamp not
null,"total" int8 not null,"used" int8 not null,"unused"
int8 not null,"owner" int8 not null,"efficiency" float4 not
null,constraint "OSUtil_pk" primary key ("_id"))'| psql -d cumin
-U cumin -h localhost
echo 'create table "com.redhat.grid.plumage"."OSUtil_samples"
("ts" timestamp not null,"total" int8 not null,"used" int8
not null,"unused" int8 not null,"owner" int8 not
null,"efficiency" float4 not null)'| psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil" ADD COLUMN
freemem int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil" ADD COLUMN
usedmem int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil" ADD COLUMN
availmem int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil" ADD COLUMN
totalmem int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil" ADD COLUMN
freecpu int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil" ADD COLUMN
usedcpu int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil" ADD COLUMN
availcpu int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil" ADD COLUMN
totalcpu int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil_samples" ADD
COLUMN freemem int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil_samples" ADD
COLUMN usedmem int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil_samples" ADD
COLUMN availmem int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil_samples" ADD
COLUMN totalmem int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil_samples" ADD
COLUMN freecpu int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil_samples" ADD
COLUMN usedcpu int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil_samples" ADD
COLUMN availcpu int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+echo 'ALTER TABLE "com.redhat.grid.plumage"."OSUtil_samples" ADD
COLUMN totalcpu int8 NOT NULL DEFAULT 0' | psql -d cumin -U cumin -h localhost
+
Modified: trunk/mint/python/mint/plumage/session.py
===================================================================
--- trunk/mint/python/mint/plumage/session.py 2012-05-30 19:52:07 UTC (rev 5393)
+++ trunk/mint/python/mint/plumage/session.py 2012-05-31 12:18:57 UTC (rev 5394)
@@ -1,11 +1,13 @@
from __future__ import division
import time
+import inspect
from mint.util import MintDaemonThread
from time import sleep
from update import ObjectUpdate
from random import randrange
from datetime import timedelta, datetime
+from bson.code import Code
try:
from pymongo import Connection
@@ -22,6 +24,25 @@
UTC_DIFF = datetime.utcnow() - datetime.now() # needed because everything in mongo is
UTC
+timestamp_mapper = Code("""
+ function () {
+ emit({ts:this.ts, st:this.st}, {mem: this.mem, cpu: this.cpu});
+ }
+ """)
+
+timestamp_reducer = Code("""
+ function (key, values) {
+ var memsum = 0;
+ var cpusum = 0;
+ values.forEach(function(doc) {
+ memsum += doc.mem;
+ cpusum += doc.cpu;
+ });
+ return {mem: memsum, cpu: cpusum}; //format of this needs to be
same as second param of emit
+ }
+ """)
+
+
class OSUtil(object):
pass
@@ -153,7 +174,40 @@
sleep(1)
if self.stop_requested:
break
+
+ def fillOSUtilStats(self, time, temptable):
+ record = OSUtil()
+ record.host= "%s:%s" % (self.connection.host, self.connection.port)
+ record.total =
len(self.collection.find({'ts':time}).distinct('n'))
+ record.used =
len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
+ record.unused =
len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
+ record.owner =
len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
+ reduced = self.collection.map_reduce(timestamp_mapper, timestamp_reducer,
temptable, query={'ts':time,})
+ reduced_collection = Collection(self.database,temptable)
+ record.freemem =
int(reduced_collection.find({'_id.st':{'$nin':['Claimed','Owner']}})[0]['value']['mem'])
+ record.freecpu =
int(reduced_collection.find({'_id.st':{'$nin':['Claimed','Owner']}})[0]['value']['cpu'])
+ record.usedmem =
int(reduced_collection.find({'_id.st':'Claimed'})[0]['value']['mem'])
+ record.usedcpu =
int(reduced_collection.find({'_id.st':'Claimed'})[0]['value']['cpu'])
+
+ record.availmem =
int(reduced_collection.find({'_id.st':'Claimed'})[0]['value']['mem'])
+
int(reduced_collection.find({'_id.st':'Unclaimed'})[0]['value']['mem'])
+ record.availcpu =
int(reduced_collection.find({'_id.st':'Claimed'})[0]['value']['cpu'])
+
int(reduced_collection.find({'_id.st':'Unclaimed'})[0]['value']['cpu'])
+
+ record.totalmem =
int(reduced_collection.find({'_id.st':'Claimed'})[0]['value']['mem'])
+ \
+
int(reduced_collection.find({'_id.st':'Unclaimed'})[0]['value']['mem'])
+ \
+
int(reduced_collection.find({'_id.st':'Owner'})[0]['value']['mem'])
+
+ record.totalcpu =
int(reduced_collection.find({'_id.st':'Claimed'})[0]['value']['cpu'])
+ \
+
int(reduced_collection.find({'_id.st':'Unclaimed'})[0]['value']['cpu'])
+ \
+
int(reduced_collection.find({'_id.st':'Owner'})[0]['value']['cpu'])
+
+ if(record.total != record.owner):
+ record.efficiency = (record.used/(record.total-record.owner))*100
+ else:
+ record.efficiency = 0
+ record.ts = time - UTC_DIFF
+ return record
+
def run(self):
self._check_connection()
if self.stop_requested:
@@ -177,17 +231,7 @@
sample_times = map(lambda i: times[i],filter(lambda i: i%5 ==
0,range(len(times))))
for time in sample_times:
- record = OSUtil()
- record.host= "%s:%s" % (self.connection.host,
self.connection.port)
- record.total =
len(self.collection.find({'ts':time}).distinct('n'))
- record.used =
len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
- record.unused =
len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
- record.owner =
len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
- if(record.total != record.owner):
- record.efficiency = (record.used/(record.total-record.owner))*100
- else:
- record.efficiency = 0
- record.ts = time - UTC_DIFF
+ record = self.fillOSUtilStats(time, "history")
obj = ObjectUpdate(self.app.model, record, self.cls)
self.app.update_thread.enqueue(obj)
@@ -220,17 +264,7 @@
sample_times = map(lambda i: times[i],filter(lambda i: i%5 ==
0,range(len(times))))
for time in sample_times:
- record = OSUtil()
- record.host= "%s:%s" % (self.connection.host,
self.connection.port)
- record.total =
len(self.collection.find({'ts':time}).distinct('n'))
- record.used =
len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
- record.unused =
len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
- record.owner =
len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
- if(record.total != record.owner):
- record.efficiency =
(record.used/(record.total-record.owner))*100
- else:
- record.efficiency = 0
- record.ts = time - UTC_DIFF
+ record = self.fillOSUtilStats(time, "current")
obj = ObjectUpdate(self.app.model, record, self.cls)
self.app.update_thread.enqueue(obj)
@@ -263,17 +297,7 @@
sample_times = map(lambda i: times[i],filter(lambda i: i%5 ==
0,range(len(times))))
for time in sample_times:
- record = OSUtil()
- record.host= "%s:%s" % (self.connection.host,
self.connection.port)
- record.total =
len(self.collection.find({'ts':time}).distinct('n'))
- record.used =
len(self.collection.find({'ts':time,'st':{'$nin':['Unclaimed','Owner']}}).distinct('n'))
- record.unused =
len(self.collection.find({'ts':time,'st':'Unclaimed'}).distinct('n'))
- record.owner =
len(self.collection.find({'ts':time,'st':'Owner'}).distinct('n'))
- if(record.total != record.owner):
- record.efficiency =
(record.used/(record.total-record.owner))*100
- else:
- record.efficiency = 0
- record.ts = time - UTC_DIFF
+ record = self.fillOSUtilStats(time, "catchup")
obj = ObjectUpdate(self.app.model, record, self.cls)
self.app.update_thread.enqueue(obj)