r4541 - in trunk/cumin: python/cumin python/cumin/grid resources
by eallen@fedoraproject.org
Author: eallen
Date: 2011-02-21 20:11:08 +0000 (Mon, 21 Feb 2011)
New Revision: 4541
Modified:
trunk/cumin/python/cumin/charts.py
trunk/cumin/python/cumin/grid/slot.py
trunk/cumin/python/cumin/grid/slot.strings
trunk/cumin/resources/slots.swf
Log:
Implement BZ 647500 - Categorize slots into more natural categories.
- Updated action script code to handle categories
- Updated png generation code to match the new categories
- Updated json generation code to group slots into new categories
- Updated hover popup for swf and png to display the new categories
- Updated legend for sef and png to show the new categories
Modified: trunk/cumin/python/cumin/charts.py
===================================================================
--- trunk/cumin/python/cumin/charts.py 2011-02-21 19:30:33 UTC (rev 4540)
+++ trunk/cumin/python/cumin/charts.py 2011-02-21 20:11:08 UTC (rev 4541)
@@ -22,6 +22,18 @@
self.surface = None
self.gap = 3
+ def plot_colored_shape(self, interior, shape, width, height):
+ surface = ImageSurface(FORMAT_ARGB32, int(width), int(height))
+ cr = Context(surface)
+ cr.set_line_width(1)
+
+ cr.set_source_rgba(1.0, 1.0, 1.0, 1.0)
+ cr.rectangle(0, 0, width - 1, height - 1)
+ cr.fill()
+
+ self._plot_shape(cr, interior, shape, 0, 0, width, height)
+ return surface
+
def plot_colored_rect(self, interior, width, height):
surface = ImageSurface(FORMAT_ARGB32, int(width), int(height))
cr = Context(surface)
@@ -41,19 +53,33 @@
return surface
def _plot_square(self, cr, colors, state, x, y, width, height):
+ if state == "Unclaimed":
+ shape = "box"
+ elif state == "Claimed":
+ shape = "solid"
+ elif state in ("Owner", "Unavailable"):
+ shape = "diagonal"
+ elif state in ("Matched", "Preempting", "Preempting/Matched"):
+ shape = "filled_triangle"
+ else:
+ shape = "box"
+
+ self._plot_shape(cr, colors, shape, x, y, width, height)
+
+ def _plot_shape(self, cr, colors, shape, x, y, width, height):
cr.set_source_rgb(*colors)
#cr.move_to(x, y)
cr.rectangle(x, y, width - self.gap, height - self.gap)
- if state == "Unclaimed": # leave empty
+ if shape == "box": # leave empty
cr.stroke()
- elif state == "Claimed": # solid fill
+ elif shape == "solid": # solid fill
cr.fill()
- elif state in ("Owner", "Unavailable"): # diagonal line
+ elif shape == "diagonal": # diagonal line
cr.move_to(x, y)
cr.line_to(x + width - self.gap, y + height - self.gap)
cr.stroke()
- elif state in ("Matched", "Preempting", "Preempting/Matched"): # triangle
+ elif shape == "filled_triangle": # triangle
cr.stroke()
cr.move_to(x + width - self.gap, y)
cr.line_to(x, y + height - self.gap)
@@ -61,8 +87,8 @@
cr.close_path()
cr.fill()
- def plot_slots(self, slots, zl):
- count = len(slots)
+ def plot_shapes(self, shape_info, zl):
+ count = len(shape_info)
slot_size = self.slot_size(count, zl)
x = 0
y = 0
@@ -74,8 +100,9 @@
cr.set_line_width( zl == 1 and 1 or 2 )
- for slot in slots:
- interior, state = slot[:2]
+ for slot in shape_info:
+ interior = slot["color"]
+ shape = slot["shape"]
if col >= self.cols:
x = 0
@@ -83,7 +110,7 @@
col = 0
self.rows = self.rows + 1
- self._plot_square(cr, interior, state, x, y, slot_size, slot_size)
+ self._plot_shape(cr, interior, shape, x, y, slot_size, slot_size)
x = x + slot_size
col = col + 1
Modified: trunk/cumin/python/cumin/grid/slot.py
===================================================================
--- trunk/cumin/python/cumin/grid/slot.py 2011-02-21 19:30:33 UTC (rev 4540)
+++ trunk/cumin/python/cumin/grid/slot.py 2011-02-21 20:11:08 UTC (rev 4541)
@@ -1,6 +1,5 @@
import logging
-from wooly import *
from wooly.widgets import *
from wooly.forms import *
from wooly.resources import *
@@ -48,8 +47,15 @@
self.where_exprs.default = list()
- def render_sql_order_by(self, session):
- return "Order by \"System\",\"Name\""
+ def sort_records(self, records):
+ try:
+ for rec in records:
+ rec["category"], rec["category_group"] = SlotMapPage.get_cat_and_group(rec["State"], rec["Activity"])
+ recs = sorted(records, key=lambda x:"%s%s%s" %
+ (x["category_group"], x["category"], x["Name"]))
+ except Exception:
+ return records
+ return recs
class SlotView(CuminView):
def __init__(self, app, name, slot):
@@ -111,6 +117,28 @@
class SlotMapPage(Page):
""" handles the slots.vis request """
+ categories = {("Owner", None): ("Unavailable", "Owner"),
+ ("Unclaimed", None): ("Available", "Unclaimed"),
+ ("Matched", None): ("Starting", "Transitioning"),
+ ("Claimed", "Idle"): ("Starting", "Transitioning"),
+ ("Claimed", "Busy"): ("Running", "Busy"),
+ ("Claimed", "Retiring"): ("Running", "Busy"),
+ ("Claimed", "Suspended"): ("Paused", "Busy"),
+ ("Preempting", None): ("Stopping", "Transitioning"),
+ ("Unknown", None): ("Unknown", "Unknown")}
+
+ shapes = {"Unavailable": {"shape": "filled_triangle", "color": (.6,.6,.6)},
+ "Available": {"shape": "box", "color": (.6,.6,.6)},
+ "Stopping": {"shape": "diagonal", "color": (.8,0,0)},
+ "Starting": {"shape": "diagonal", "color": (0,0,.8)},
+ "Paused": {"shape": "solid", "color": (.4,.4,1)},
+ "Running": {"shape": "solid", "color": (0,.8,0)},
+ "Busy": {"shape": "solid", "color": (0,.8,0)},
+ "Transitioning": {"shape": "diagonal", "color": (.8,0,.8)},
+ "Unclaimed": {"shape": "box", "color": (.6,.6,.6)},
+ "Owner": {"shape": "filled_triangle", "color": (.6,.6,.6)},
+ "Unknown": {"shape": "solid", "color": (1,0,0)}}
+
interiors = {"Idle": (.7,.7,.7),
"Busy": (0.0, 0.4, 0.0),
"Suspended": (1,0,0),
@@ -120,7 +148,6 @@
"Retiring": (.8,.2,.8),
"Unknown": (.8,.8,.8)}
- shapes = ["rectangle", "circle", "circle"]
max_width = 400
max_png_age = 30
max_visible_slots = 200 # start drill-down mode after this number
@@ -134,12 +161,9 @@
self.zoom_level.default = 1
self.add_parameter(self.zoom_level)
- self.activity = Parameter(app, "act")
- self.add_parameter(self.activity)
+ self.category = Parameter(app, "category")
+ self.add_parameter(self.category)
- self.state = Parameter(app, "state")
- self.add_parameter(self.state)
-
self.json = Parameter(app, "json")
self.add_parameter(self.json)
@@ -152,70 +176,67 @@
self.groups = ListParameter(app, "group", group)
self.add_parameter(self.groups)
- self.cache = ImageCache()
self.slots = SlotDataSet(app)
+ @classmethod
+ def get_categories(cls):
+ return cls.categories
+
+ @classmethod
+ def get_unique_category_groups(cls):
+ categories = cls.categories
+ d = dict()
+ for x in categories:
+ _, group = categories[x]
+ if not group == "Unknown":
+ d[group] = None
+ return d.keys()
+
def get_content_type(self, session):
return self.json.get(session) and "text/plain" or "image/png"
def get_cache_control(self, session):
return "no-cache"
- def render_activity(self, session, activity):
+ def render_category(self, session, category):
map = HeatMapChart()
- surface = map.plot_colored_rect(self.interiors[activity], 12, 12)
+ surface = map.plot_colored_shape(self.shapes[category]["color"],
+ self.shapes[category]["shape"], 20, 20)
writer = Writer()
surface.write_to_png(writer)
return writer.to_string()
- def render_state(self, session, state):
- map = HeatMapChart()
- surface = map.plot_state(state, 12, 12)
- writer = Writer()
- surface.write_to_png(writer)
- return writer.to_string()
-
def do_render(self, session):
- activity = self.activity.get(session)
- if activity:
- return self.render_activity(session, activity)
+ # render the legend icon
+ category = self.category.get(session)
+ if category:
+ return self.render_category(session, category)
- state = self.state.get(session)
- if state:
- return self.render_state(session, state)
-
+ # generate the json data for the flash chart
if self.json.get(session):
return self.render_json(session)
zl = self.zoom_level.get(session)
- # determine if cached copy is recent enough
- cached_png, args = self.get_cached(session, zl)
- if cached_png:
- self.set_cookie(session, args)
- return cached_png
-
map = HeatMapChart(self.max_width, self.max_width)
cursor = self.slots.execute(session)
- records = cursor.fetchall()
+ records = cursor_to_rows(cursor)
if len(records) == 0:
return
- columns = [x[0] for x in cursor.description]
- activity = columns.index("Activity")
- state = columns.index("State")
+ records = self.slots.sort_records(records)
+ shape_info = list()
+ for rec in records:
+ cat = rec["category"]
+ shape_info.append(self.shapes[cat])
+ size = map.plot_shapes(shape_info, zl)
- interiors = self.interiors.copy()
- interiors[None] = interiors["Unknown"]
- slots = [(interiors[x[activity]], x[state]) for x in records]
- size = map.plot_slots(slots, zl)
+ args = (size, map.width, map.height, len(shape_info), map.rows, map.cols)
+ cookie = "|".join((str(x) for x in args))
+ session.set_cookie("slot_info", cookie)
- args = (size, map.width, map.height, len(slots), map.rows, map.cols)
- self.set_cookie(session, args)
- self.cache_it(session, zl, map, args)
-
writer = Writer()
map.write(writer)
return writer.to_string()
@@ -231,16 +252,35 @@
d[atr].append(i)
return d
- def get_activityStates(self, records, plist):
- """ for each unique combination of activity:state, return a dictionary entry
- that counts the number of records with that combo """
+ @classmethod
+ def get_cat_and_group(cls, state, activity):
+ act = activity
+ if state in ("Owner", "Unclaimed", "Matched", "Preempting", "Unknown"):
+ # activity doesn't matter for these states
+ act = None
+ try:
+ cat, cat_group = cls.categories[(state, act)]
+ except Exception, e:
+ log.exception(e)
+ cat, cat_group = ("%s:%s" % (state, activity), "Unknown")
+ return cat, cat_group
+
+ def get_cats(self, records, plist):
d = dict()
for i in plist:
- activityState = "%s:%s" % (records[i]["Activity"], records[i]["State"])
- if not activityState in d:
- d[activityState] = 0
- d[activityState] += 1
+ state = records[i]["State"]
+ activity = records[i]["Activity"]
+
+ cat, cat_group = SlotMapPage.get_cat_and_group(state, activity)
+ if not cat_group in d:
+ d[cat_group] = dict()
+ if not cat in d[cat_group]:
+ d[cat_group][cat] = 0
+ d[cat_group]["_count"] = 0
+ d[cat_group]["_count"] += 1
+ d[cat_group][cat] += 1
+
return d
def render_json(self, session):
@@ -250,6 +290,7 @@
if len(records) == 0:
return "[]"
+ records = self.slots.sort_records(records)
groups = self.groups.get(session)
slot_count = len(records)
@@ -257,10 +298,11 @@
root.name = self.object_description
root.value = self.object_param.get(session)._id
root.slots = slot_count
- root.vis = self.json.get(session)
+ json = self.json.get(session)
+ root.vis = json
+ root.shape_info = self.shapes
root.activity_colors = self.interiors
expanded = self.expanded.get(session)
- json = self.json.get(session)
leaves = True
(root.tree, root.back) = self.treeify(records, range(slot_count), groups, 0,
expanded, leaves, slot_count, json)
@@ -273,11 +315,13 @@
if level == len(groups):
if leaves:
for i in sorted(plist, key=lambda x:"%s%s%s" %
- (records[x]["Activity"], records[x]["State"], records[x]["Name"])):
+ (records[x]["category_group"], records[x]["category"], records[x]["Name"])):
el = Element()
el.job_id = records[i]["JobId"] and records[i]["JobId"] or ""
el.activity = records[i]["Activity"] and records[i]["Activity"] or "Unknown"
el.state = records[i]["State"] and records[i]["State"] or "Unknown"
+ el.category = records[i]["category"]
+ el.category_group = records[i]["category_group"]
el.value = records[i]["Name"] and records[i]["Name"] or ""
el.load_avg = records[i]["LoadAvg"] and round(records[i]["LoadAvg"], 2) or 0
el.name = "slot"
@@ -287,20 +331,19 @@
else:
if not expanded:
# display summary info for all the slots under this grouping
- activityStates = self.get_activityStates(records, plist)
- for activityState in sorted(activityStates):
+ category_groups = self.get_cats(records, plist)
+ for category_group in sorted(category_groups):
el = Element()
el.name = "slot_info"
- el.a_s = activityState
- el.count = activityStates[activityState]
- el.value = "%s.%s" % (records[plist[0]][groups[level-1]], activityState) # uid
+ el.category = category_group
+ el.count = category_groups[category_group]["_count"]
+ el.value = "%s.%s" % (records[plist[0]][groups[level-1]], category_group) # uid
level_list.append(el)
return (level_list, False)
# not a leaf
group = groups[level]
level_dict = self.get_info_array(records, plist, group)
- first = True
for key in sorted(level_dict):
el = Element()
el.name = group
@@ -323,27 +366,13 @@
return (level_list, False)
- def get_cached(self, session, zl):
- filename = self.gen_filename(session)
- return self.cache.find_recent(filename, self.max_png_age)
-
- def cache_it(self, session, zl, map, args):
- filename = self.gen_filename(session)
- writer = self.cache.create_cache_file(filename, args)
- map.write(writer)
-
- def gen_filename(self, session):
- return session.marshal()
-
- def set_cookie(self, session, args):
- cookie = "|".join((str(x) for x in args))
- session.set_cookie("slot_info", cookie)
-
class SlotMap(Widget):
def __init__(self, app, name):
super(SlotMap, self).__init__(app, name)
- self.slot_info = self.SlotInfo(app, "slot_info")
+ self.slots = SlotDataSet(app)
+
+ self.slot_info = self.SlotInfo(app, "slot_info", self.slots)
self.add_child(self.slot_info)
self.slot_legend = self.SlotLegend(app, "slot_legend")
@@ -351,8 +380,6 @@
self.slot_clip_size = 400
- self.slots = SlotDataSet(app)
-
def render_image_href(self, session):
raise Exception("Not implemented")
@@ -375,18 +402,27 @@
def render_slot_clip_size(self, session):
return self.slot_clip_size
- class SlotLegend(Widget):
+ class SlotLegend(ItemSet):
def __init__(self, app, name):
super(SlotMap.SlotLegend, self).__init__(app, name)
- self.states = SlotStates(app, "slot_states")
- self.add_child(self.states)
+ categories = SlotMapPage.get_unique_category_groups()
+ for cat in sorted(categories):
+ acat = SlotCategories(app, cat)
+ self.add_child(acat)
- self.activities = SlotActivities(app, "slot_activities")
- self.add_child(self.activities)
+ def do_get_items(self, session):
+ return self.children
+ def render_category(self, session, cat):
+ return cat.name
+
+ def render_category_items(self, session, cat):
+ return cat.render(session)
+
class SlotInfo(ItemSet):
- display_names = {"JobId": ("Job ID", "", ""),
+ display_names = {"category": ("", "", ""),
+ "JobId": ("Job ID", "", ""),
"System": ("System", "", ""),
"Machine": ("Machine", "", ""),
"State": ("State", "", ""),
@@ -394,7 +430,7 @@
"Name": ("Name", "", ""),
"_id": ("Slot_ID", "slotInfo_id", "hidden_row")}
- def __init__(self, app, name):
+ def __init__(self, app, name, slot_data):
super(SlotMap.SlotInfo, self).__init__(app, name)
self.index = IntegerParameter(app, "i")
@@ -403,26 +439,25 @@
self.slot = Attribute(app, "slot")
self.add_attribute(self.slot)
+ self.slot_data = slot_data
+
self.info_div_tmpl = WidgetTemplate(self, "bg_html")
def do_render(self, session):
index = self.index.get(session)
if index is not None:
- self.parent.slots.limit.set(session, 1)
- self.parent.slots.offset.set(session, index)
+ #self.parent.slots.limit.set(session, 1)
+ #self.parent.slots.offset.set(session, index)
- cursor = self.parent.slots.execute(session)
+ cursor = self.slot_data.execute(session)
+ records = cursor_to_rows(cursor)
- data = dict()
- names = [x[0] for x in cursor.description]
- values = cursor.fetchone()
+ if len(records):
+ records = self.slot_data.sort_records(records)
- for name, value in zip(names, values):
- data[name] = value
+ self.slot.set(session, records[index])
- self.slot.set(session, data)
-
writer = Writer()
self.info_div_tmpl.render(writer, session)
return writer.to_string()
@@ -432,7 +467,9 @@
def do_get_items(self, session):
slot = self.slot.get(session)
- return ((self.display_names[x], slot[x])
+ return ((
+ x == "category" and (slot["category_group"],"","title_row") or self.display_names[x],
+ slot[x])
for x in slot if x in self.display_names)
def render_slot_info_url(self, session):
@@ -453,46 +490,27 @@
def render_item_row_class(self, session, item):
return item[0][2] and "class='%s'" % item[0][2] or ""
-class SlotStates(ItemSet):
- states = ("Unclaimed", "Claimed", "Unavailable", "Matched", "Preempting")
-
+class SlotCategories(ItemSet):
def do_get_items(self, session):
- return self.states
+ cat_group = self.name
+ categories = SlotMapPage.get_categories()
- def render_item_size(self, session, activity):
- return 12
+ d = dict()
+ for x in categories:
+ cat, group = categories[x]
+ if group is cat_group:
+ d[cat] = None
- def render_item_title(self, session, state):
- return state
+ return sorted(d.keys())
- def render_item_href(self, session, state):
- params = list()
- params.append("state=%s" % state)
+ def render_item_size(self, session, cat):
+ return 20
- return "poolslots.vis?" + ";".join(params)
+ def render_item_title(self, session, cat):
+ return cat
-class SlotActivities(ItemSet):
- activities = (("Idle", "clear"),
- ("Busy", "green"),
- ("Suspended", "red"),
- ("Vacating", "orange"),
- ("Killing", "blue"),
- ("Benchmarking", "yellow"),
- ("Retiring", "purple"),
- ("Unknown", "grey"))
-
- def do_get_items(self, session):
- return self.activities
-
- def render_item_size(self, session, activity):
- return 12
-
- def render_item_title(self, session, activity):
- return activity[0]
-
- def render_item_href(self, session, activity):
+ def render_item_href(self, session, cat):
params = list()
- params.append("act=%s" % activity[0])
+ params.append("category=%s" % cat)
return "poolslots.vis?" + ";".join(params)
-
Modified: trunk/cumin/python/cumin/grid/slot.strings
===================================================================
--- trunk/cumin/python/cumin/grid/slot.strings 2011-02-21 19:30:33 UTC (rev 4540)
+++ trunk/cumin/python/cumin/grid/slot.strings 2011-02-21 20:11:08 UTC (rev 4541)
@@ -303,12 +303,18 @@
text-align: right;
padding-right: 1em;
}
-
td.slot_info_value {
font-weight: normal;
height: 1.1em;
}
+tr.title_row td.slot_info_title, tr.title_row td.slot_info_value {
+ padding-bottom: 0.5em;
+ border-bottom: 1px solid black;
+ font-style: italic;
+ font-weight: bold;
+}
+
html > body .outerpair1 {
background: url(resource?name=upperrightfade.png) right top no-repeat;
}
@@ -359,65 +365,53 @@
</tr>
[SlotLegend.css]
-div.slot_legend {
- margin-top: 0.1em;
- margin-left: auto;
- margin-right: auto;
- width: 20em;
+div.SlotLegend {
+ -moz-border-radius: 0.5em;
+ background-color: #FCFCCC;
+ border: 1px solid #BABA00;
+ margin: 0 0 0 8em;
+ padding: 0 0 1em 2em;
+ width: 11em;
}
-
-div.slot_legend h3 {
- margin-top: 0;
+div.SlotLegend dl {
+ margin: auto;
+ padding: 0;
}
-
-div.slot_states, div.slot_activities {
- float: left;
- margin-left: 1.3em;
+div.SlotLegend dt {
+ margin: 0;
+ padding: 1em 0 0 0;
+ font-weight: bold;
}
-
-div.slot_states ul, div.slot_activities ul {
+div.SlotLegend dd {
+ margin: 0 0 0 1em;
padding: 0;
+}
+div.SlotLegend dd img {
margin: 0;
- font-size: 0.9em;
- list-style: none;
+ position: relative;
+ top: 6px;
}
-div.slot_states li, div.slot_activities li {
- padding-bottom: 0.25em;
-}
-
-div.slot_states li:hover, div.slot_activities li:hover {
- color: red;
-}
-
[SlotLegend.html]
-<div class="slot_legend">
- {slot_states}
-
- {slot_activities}
-
- <div style="clear:both"><!-- --></div>
+<div id="{id}" class="{class}">
+ <dl>
+ {items}
+ </dl>
</div>
-[SlotStates.html]
-<div class="slot_states">
- <h3>States</h3>
+[SlotLegend.item_html]
+ <dt>{category}</dt>
+ {category_items}
- <ul>{items}</ul>
-</div>
+[SlotCategories.html]
+{items}
-[SlotActivities.html]
-<div class="slot_activities">
- <h3>Activities</h3>
-
- <ul>{items}</ul>
-</div>
-
-[SlotStates.item_html]
-<li onmouseover="vis_treemap_over('state', '{item_title}')" onmouseout="vis_treemap_out('state', '{item_title}')">
+[SlotCategories.item_html]
+<dd onmouseover="vis_treemap_over('{item_title}')" onmouseout="vis_treemap_out('{item_title}')">
<img src="{item_href}" width="{item_size}" height="{item_size}" title="{item_title}" alt="{item_title}"/>
+
{item_title}
-</li>
+</dd>
[SlotActivities.item_html]
<li onmouseover="vis_treemap_over('activity', '{item_title}')" onmouseout="vis_treemap_out('activity', '{item_title}')">
Modified: trunk/cumin/resources/slots.swf
===================================================================
(Binary files differ)
13 years, 2 months
r4540 - trunk/cumin/bin
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-21 19:30:33 +0000 (Mon, 21 Feb 2011)
New Revision: 4540
Modified:
trunk/cumin/bin/cumin
Log:
Add logging to $CUMIN_HOME/master.log for startup errors,
informational messages on application start, warnings on restart.
Modified: trunk/cumin/bin/cumin
===================================================================
--- trunk/cumin/bin/cumin 2011-02-18 22:46:34 UTC (rev 4539)
+++ trunk/cumin/bin/cumin 2011-02-21 19:30:33 UTC (rev 4540)
@@ -6,7 +6,11 @@
import os
import sys
import signal
+from parsley.loggingex import *
+log = logging.getLogger("cumin.master")
+home = os.environ.get("CUMIN_HOME", os.path.normpath("/usr/share/cumin"))
+
def call_sys_exit(sig,frame):
sys.exit()
@@ -21,6 +25,19 @@
else:
signal.signal(signal.SIGTERM, func)
+def log_parse_errors(r):
+ msg = ""
+ r = os.fdopen(r, "r")
+ while True:
+ t = r.read()
+ if t == "":
+ break
+ msg += t
+ r.close()
+ if msg:
+ msg = "\n"+msg
+ log.error("Error in options"+msg)
+
def main():
# tuple indices, for clarity
@@ -28,6 +45,8 @@
SECTION = 1
PROGRAM = 2
+ enable_logging("cumin.master", logging.INFO, os.path.join(home, "master.log"))
+
parser = OptionParser()
parser.add_option("--init-only", dest="init_only", action="store_true", default=False)
@@ -40,12 +59,26 @@
help="Configuration section names for cumin-data instances."\
"\nEach value implies a separate cumin-data instance.")
- (options, args) = parser.parse_args()
+ # Trap exit from parser and save standard error for logging
+ # Then put stderr back to original value
+ err = sys.stderr
+ r, w = os.pipe()
+ sys.stderr = os.fdopen(w,"w")
+ try:
+ (options, args) = parser.parse_args()
+ except:
+ parse_errors = True
+ else:
+ parse_errors = False
+ sys.stderr.close()
+ sys.stderr = err
+
+ if parse_errors:
+ log_parse_errors(r)
+ return 1
+
if len(args) != 0:
- sys.stderr.write("cumin: extra args: ")
- for a in args:
- sys.stderr.write(a+" ")
- sys.stderr.write("\n")
+ log.error("Extra arguments:" + "".join([" "+arg for arg in args]))
return 1
# Get our list of cumin-web and data instances
@@ -58,14 +91,18 @@
apps.append([None, app, "cumin-data"])
# If we are just checking startup flags, invoke each instance
- # with "--init-only" and return status to caller
+ # with "--init-only" and return status to caller. Log failure with
+ # content of stderr from subprocess, if any.
if options.init_only:
for app in apps:
arg = [app[PROGRAM], "--init-only", "--section=" + app[SECTION]]
- app[PROCESS] = subprocess.Popen(arg)
- exit_code = app[PROCESS].wait()
- if exit_code != 0:
- sys.stderr.write("cumin: --init-only check failed\n")
+ app[PROCESS] = subprocess.Popen(arg, stderr=subprocess.PIPE)
+ errors = app[PROCESS].communicate()[1]
+ if app[PROCESS].wait() != 0:
+ msg = "Subprocess failed:" + "".join([" "+x for x in arg])
+ if len(errors) > 0:
+ msg += "\n" + errors
+ log.error(msg)
return 1
else:
# Launch and babysit
@@ -74,6 +111,11 @@
for app in apps:
if not app[PROCESS] or app[PROCESS].poll():
arg = [app[PROGRAM], "--section=" + app[SECTION]]
+ prog_string = "".join([" "+x for x in arg])
+ if app[PROCESS]:
+ log.warn("Restarting:" + prog_string)
+ else:
+ log.info("Starting:" + prog_string)
app[PROCESS] = subprocess.Popen(arg)
sleep(30)
13 years, 2 months
r4539 - in trunk/cumin: bin etc
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-18 22:46:34 +0000 (Fri, 18 Feb 2011)
New Revision: 4539
Modified:
trunk/cumin/bin/cumin
trunk/cumin/etc/sysvinit-cumin
Log:
Write notes to stderr, check for extra arguments in cumin.
Direct stderr to /dev/null in init.d checks.
Modified: trunk/cumin/bin/cumin
===================================================================
--- trunk/cumin/bin/cumin 2011-02-18 21:19:34 UTC (rev 4538)
+++ trunk/cumin/bin/cumin 2011-02-18 22:46:34 UTC (rev 4539)
@@ -21,9 +21,6 @@
else:
signal.signal(signal.SIGTERM, func)
-class InitCheckException(Exception):
- pass
-
def main():
# tuple indices, for clarity
@@ -44,6 +41,12 @@
"\nEach value implies a separate cumin-data instance.")
(options, args) = parser.parse_args()
+ if len(args) != 0:
+ sys.stderr.write("cumin: extra args: ")
+ for a in args:
+ sys.stderr.write(a+" ")
+ sys.stderr.write("\n")
+ return 1
# Get our list of cumin-web and data instances
# create list elements to hold the process object, section arg, and app
@@ -62,7 +65,8 @@
app[PROCESS] = subprocess.Popen(arg)
exit_code = app[PROCESS].wait()
if exit_code != 0:
- raise InitCheckException
+ sys.stderr.write("cumin: --init-only check failed\n")
+ return 1
else:
# Launch and babysit
try:
@@ -84,6 +88,6 @@
# run the finally block anyway.
set_exit_handler(call_sys_exit)
try:
- main()
+ sys.exit(main())
except KeyboardInterrupt:
pass
Modified: trunk/cumin/etc/sysvinit-cumin
===================================================================
--- trunk/cumin/etc/sysvinit-cumin 2011-02-18 21:19:34 UTC (rev 4538)
+++ trunk/cumin/etc/sysvinit-cumin 2011-02-18 22:46:34 UTC (rev 4539)
@@ -42,7 +42,8 @@
[ $? -eq 0 ] && success && echo && return
# Invoke in foreground with arguments to test validity, if args have been set
if [ -n "$CUMIN_OPTIONS" ] ; then
- daemon --user cumin --check $servicename $processname "--init-only "$CUMIN_OPTIONS
+ daemon --user cumin --check $servicename $processname \
+ "--init-only "$CUMIN_OPTIONS 2> /dev/null
[ $? -ne 0 ] && failure && echo && return
fi
# Launch for real now...
13 years, 2 months
r4538 - in trunk: cumin/bin cumin/etc cumin/python/cumin parsley/python/parsley
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-18 21:19:34 +0000 (Fri, 18 Feb 2011)
New Revision: 4538
Modified:
trunk/cumin/bin/cumin
trunk/cumin/etc/sysvinit-cumin
trunk/cumin/python/cumin/config.py
trunk/parsley/python/parsley/config.py
Log:
Add option for strict section checking in cumin apps. This will
raise an exception if a configuration section is referenced which
is not defined in a config file.
Give cumin/bin/cumin an --init-only option for validation on start.
Change init.d script to use --init-only option for cumin/bin/cumin
on start if $CUMIN_OPTIONS has been set before real launch.
These changes are to support error checking on service start.
BZ672583
Modified: trunk/cumin/bin/cumin
===================================================================
--- trunk/cumin/bin/cumin 2011-02-18 15:01:40 UTC (rev 4537)
+++ trunk/cumin/bin/cumin 2011-02-18 21:19:34 UTC (rev 4538)
@@ -21,6 +21,9 @@
else:
signal.signal(signal.SIGTERM, func)
+class InitCheckException(Exception):
+ pass
+
def main():
# tuple indices, for clarity
@@ -30,6 +33,8 @@
parser = OptionParser()
+ parser.add_option("--init-only", dest="init_only", action="store_true", default=False)
+
parser.add_option("--webs", dest="webs", default="web",
help="Configuration section names for cumin-web instances."\
"\nEach value implies a separate cumin-web instance.")
@@ -40,7 +45,7 @@
(options, args) = parser.parse_args()
- # Get our list cumin-web and data instances
+ # Get our list of cumin-web and data instances
# create list elements to hold the process object, section arg, and app
apps = []
for app in options.webs.split(','):
@@ -49,20 +54,30 @@
for app in options.datas.split(','):
apps.append([None, app, "cumin-data"])
- # Launch and babysit
- try:
- while True:
+ # If we are just checking startup flags, invoke each instance
+ # with "--init-only" and return status to caller
+ if options.init_only:
+ for app in apps:
+ arg = [app[PROGRAM], "--init-only", "--section=" + app[SECTION]]
+ app[PROCESS] = subprocess.Popen(arg)
+ exit_code = app[PROCESS].wait()
+ if exit_code != 0:
+ raise InitCheckException
+ else:
+ # Launch and babysit
+ try:
+ while True:
+ for app in apps:
+ if not app[PROCESS] or app[PROCESS].poll():
+ arg = [app[PROGRAM], "--section=" + app[SECTION]]
+ app[PROCESS] = subprocess.Popen(arg)
+ sleep(30)
+
+ finally:
for app in apps:
- if not app[PROCESS] or app[PROCESS].poll():
- arg = "--section=" + app[SECTION]
- app[PROCESS] = subprocess.Popen([app[PROGRAM], arg])
- sleep(30)
+ if app[PROCESS]:
+ os.kill(app[PROCESS].pid, signal.SIGTERM)
- finally:
- for app in apps:
- if app[PROCESS]:
- os.kill(app[PROCESS].pid, signal.SIGTERM)
-
if __name__ == "__main__":
# This is necessary so that on a SIGTERM we can call sys.exit()
# and cause the finally block to be executed. Ctrl-C will
Modified: trunk/cumin/etc/sysvinit-cumin
===================================================================
--- trunk/cumin/etc/sysvinit-cumin 2011-02-18 15:01:40 UTC (rev 4537)
+++ trunk/cumin/etc/sysvinit-cumin 2011-02-18 21:19:34 UTC (rev 4538)
@@ -37,16 +37,17 @@
start() {
echo -n $"Starting cumin: "
+ # If it's already running, be happy and return success
status $processname &> /dev/null
- RETVAL=$?
- if [ $RETVAL -eq 0 ] ; then
- success
- else
- daemon --user cumin --check $servicename $processname $CUMIN_OPTIONS \&
- RETVAL=$?
- [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$servicename
+ [ $? -eq 0 ] && success && echo && return
+ # Invoke in foreground with arguments to test validity, if args have been set
+ if [ -n "$CUMIN_OPTIONS" ] ; then
+ daemon --user cumin --check $servicename $processname "--init-only "$CUMIN_OPTIONS
+ [ $? -ne 0 ] && failure && echo && return
fi
- echo
+ # Launch for real now...
+ daemon --user cumin --check $servicename $processname $CUMIN_OPTIONS \&
+ [ $? -eq 0 ] && touch /var/lock/subsys/$servicename && echo
}
stop() {
Modified: trunk/cumin/python/cumin/config.py
===================================================================
--- trunk/cumin/python/cumin/config.py 2011-02-18 15:01:40 UTC (rev 4537)
+++ trunk/cumin/python/cumin/config.py 2011-02-18 21:19:34 UTC (rev 4538)
@@ -33,13 +33,13 @@
return self.parse_files(paths)
class CuminWebConfig(CuminConfig):
- def __init__(self, section_name):
+ def __init__(self, section_name, strict_section):
super(CuminWebConfig, self).__init__()
- self.create_section(section_name)
-
- def create_section(self, name):
- web = CuminConfigSection(self, name)
+ self.create_section(section_name, strict_section)
+
+ def create_section(self, name, strict_section):
+ web = CuminConfigSection(self, name, strict_section)
web.log_file.default = os.path.join(self.home, "log", name + ".log")
param = ConfigParameter(web, "update-interval", int)
@@ -68,13 +68,13 @@
param.default = 5 * 1024 #MB
class CuminDataConfig(CuminConfig):
- def __init__(self, section_name):
+ def __init__(self, section_name, strict_section):
super(CuminDataConfig, self).__init__()
- self.create_section(section_name)
+ self.create_section(section_name, strict_section)
- def create_section(self, name):
- data = CuminConfigSection(self, name)
+ def create_section(self, name, strict_section):
+ data = CuminConfigSection(self, name, strict_section)
data.log_file.default = os.path.join(self.home, "log", name + ".log")
param = ConfigParameter(data, "packages", str)
@@ -89,8 +89,8 @@
param.default = 60 * 60 # 1 hour
class CuminConfigSection(ConfigSection):
- def __init__(self, config, name):
- super(CuminConfigSection, self).__init__(config, name)
+ def __init__(self, config, name, strict_section=False):
+ super(CuminConfigSection, self).__init__(config, name, strict_section)
param = ConfigParameter(self, "database", str)
param.default = "dbname=cumin user=cumin host=localhost"
@@ -164,7 +164,9 @@
opts, args = parser.parse_args()
if hasattr(opts, "section"):
# Create config object with the given section and parse config files
- config = ConfigType(opts.section)
+ # Allow the section named in the default value of "--section" to be missing
+ # from the configuration file; specifically named sections are required.
+ config = ConfigType(opts.section, strict_section = opts.section != parser.defaults["section"])
values = config.parse()
if hasattr(values, opts.section):
# Reset defaults and parse again
Modified: trunk/parsley/python/parsley/config.py
===================================================================
--- trunk/parsley/python/parsley/config.py 2011-02-18 15:01:40 UTC (rev 4537)
+++ trunk/parsley/python/parsley/config.py 2011-02-18 21:19:34 UTC (rev 4538)
@@ -21,24 +21,20 @@
sections = ConfigValues()
for section in self.sections:
- try:
- values = section.parse(parser)
- except NoSectionError:
- values = None
-
+ values = section.parse(parser)
sections[section.name] = values
return sections
class ConfigSection(object):
- def __init__(self, config, name):
+ def __init__(self, config, name, strict_section=False):
assert isinstance(config, Config)
self.config = config
self.config.sections.append(self)
self.name = name
-
+ self.strict_section = strict_section
self.parameters = list()
def parse(self, parser):
@@ -49,6 +45,8 @@
try:
items = parser.items(section)
except NoSectionError:
+ if self.strict_section:
+ raise
continue
in_values.update(items)
13 years, 2 months
r4537 - trunk/cumin/bin
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-18 15:01:40 +0000 (Fri, 18 Feb 2011)
New Revision: 4537
Modified:
trunk/cumin/bin/cumin-admin
Log:
Remove operational logging for cumin-admin. This prevents creation
of the $CUMIN_HOME/log/data.log file by root on a clean system which
leads to file permission problems. Logging to stderr is fine.
Modified: trunk/cumin/bin/cumin-admin
===================================================================
--- trunk/cumin/bin/cumin-admin 2011-02-17 20:21:20 UTC (rev 4536)
+++ trunk/cumin/bin/cumin-admin 2011-02-18 15:01:40 UTC (rev 4537)
@@ -31,8 +31,6 @@
parser.set_usage(generate_usage())
config, values, opts, args = get_configuration(parser, CuminDataConfig)
- setup_operational_logging(opts)
-
try:
name = args[0]
except IndexError:
13 years, 2 months
r4536 - trunk/cumin
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-17 20:21:20 +0000 (Thu, 17 Feb 2011)
New Revision: 4536
Modified:
trunk/cumin/Makefile
Log:
Undo revision 4535, change to create data.log.
This needs more thought.
Modified: trunk/cumin/Makefile
===================================================================
--- trunk/cumin/Makefile 2011-02-17 20:11:45 UTC (rev 4535)
+++ trunk/cumin/Makefile 2011-02-17 20:21:20 UTC (rev 4536)
@@ -34,13 +34,6 @@
install -pm 0644 etc/cumin.* ${CUMIN_HOME}/etc
install -pm 0755 etc/sysvinit-cumin ${CUMIN_HOME}/etc
install -d ${CUMIN_HOME}/log
-
- # This log file needs to exist, otherwise if cumin-admin commands
- # are run from the root account before the cumin service is started
- # this file will have the wrong ownership.
- touch ${CUMIN_HOME}/log/data.log
- -chown cumin:cumin ${CUMIN_HOME}/log/data.log
-
bin/install-python-code python ${CUMIN_HOME}/python
bin/install-python-code ../wooly/python ${CUMIN_HOME}/python
bin/install-python-code ../rosemary/python ${CUMIN_HOME}/python
13 years, 2 months
r4535 - trunk/cumin
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-17 20:11:45 +0000 (Thu, 17 Feb 2011)
New Revision: 4535
Modified:
trunk/cumin/Makefile
Log:
Create a $CUMIN_HOME/log/data.log file owned by the
cumin user as part of make install. To prevent creation
of the same file by cumin-admin when run by root which
results in the wrong ownership.
Modified: trunk/cumin/Makefile
===================================================================
--- trunk/cumin/Makefile 2011-02-16 15:58:29 UTC (rev 4534)
+++ trunk/cumin/Makefile 2011-02-17 20:11:45 UTC (rev 4535)
@@ -34,6 +34,13 @@
install -pm 0644 etc/cumin.* ${CUMIN_HOME}/etc
install -pm 0755 etc/sysvinit-cumin ${CUMIN_HOME}/etc
install -d ${CUMIN_HOME}/log
+
+ # This log file needs to exist, otherwise if cumin-admin commands
+ # are run from the root account before the cumin service is started
+ # this file will have the wrong ownership.
+ touch ${CUMIN_HOME}/log/data.log
+ -chown cumin:cumin ${CUMIN_HOME}/log/data.log
+
bin/install-python-code python ${CUMIN_HOME}/python
bin/install-python-code ../wooly/python ${CUMIN_HOME}/python
bin/install-python-code ../rosemary/python ${CUMIN_HOME}/python
13 years, 2 months
r4534 - trunk/cumin/python/cumin
by eallen@fedoraproject.org
Author: eallen
Date: 2011-02-16 15:58:29 +0000 (Wed, 16 Feb 2011)
New Revision: 4534
Modified:
trunk/cumin/python/cumin/widgets.py
Log:
Use qmf_object_id instead of qmf_agent_id for csv export parameters since qmf_agent_id isn't unique in qmf 1.x
Modified: trunk/cumin/python/cumin/widgets.py
===================================================================
--- trunk/cumin/python/cumin/widgets.py 2011-02-16 15:54:16 UTC (rev 4533)
+++ trunk/cumin/python/cumin/widgets.py 2011-02-16 15:58:29 UTC (rev 4534)
@@ -1449,7 +1449,7 @@
self.packages = ListParameter(app, "packages", package)
self.add_parameter(self.packages)
- agent = Parameter(app, "agent")
+ agent = Parameter(app, "qmf_obj_id")
self.agents = ListParameter(app, "agents", agent)
self.add_parameter(self.agents)
@@ -1479,7 +1479,7 @@
for cls, package, agent in zip(classes, packages, agents):
rosemary_package = self.app.model._packages_by_name[package]
rosemary_cls = rosemary_package._classes_by_name[cls]
- objects.append(rosemary_cls.get_object(cursor, _qmf_agent_id=agent))
+ objects.append(rosemary_cls.get_object(cursor, _qmf_object_id=agent))
return objects
@@ -1494,7 +1494,7 @@
cls = obj._class
self.packages.add(nsession, cls._package._name)
self.classes.add(nsession, cls._name)
- self.agents.add(nsession, obj._qmf_agent_id)
+ self.agents.add(nsession, obj._qmf_object_id)
class IncrementalSearchInput(StringInput, ItemSet):
def __init__(self, app, name, field_param):
13 years, 2 months
r4533 - trunk/cumin/python/cumin/messaging
by eallen@fedoraproject.org
Author: eallen
Date: 2011-02-16 15:54:16 +0000 (Wed, 16 Feb 2011)
New Revision: 4533
Modified:
trunk/cumin/python/cumin/messaging/binding.py
trunk/cumin/python/cumin/messaging/exchange.py
trunk/cumin/python/cumin/messaging/queue.py
Log:
Enable csv export on queue and exchange bindings
Modified: trunk/cumin/python/cumin/messaging/binding.py
===================================================================
--- trunk/cumin/python/cumin/messaging/binding.py 2011-02-16 01:33:22 UTC (rev 4532)
+++ trunk/cumin/python/cumin/messaging/binding.py 2011-02-16 15:54:16 UTC (rev 4533)
@@ -1,17 +1,22 @@
import logging
-from wooly import WidgetTemplate, Writer, Attribute, Parameter, Widget
-from wooly.forms import FormInput, FormField, Form
-from wooly.parameters import DictParameter
-from cumin.formats import *
-from cumin.objectselector import *
-from cumin.sqladapter import *
-from cumin.util import sorted_by
-from cumin.widgets import *
+from rosemary.model import RosemaryObject
-import main
-from cumin.objectframe import ObjectFrameTaskForm
+from cumin.formats import fmt_shorten
+from cumin.objectframe import ObjectFrameTaskForm, ObjectFrame, ObjectFrameTask
+from cumin.objectselector import ObjectSelectorTask, ObjectSelector,\
+ ObjectLinkColumn
+from cumin.widgets import SubmitSwitch, QueueBindingField, ExchangeBindingField
+from cumin.sqladapter import ObjectSqlAdapter
+from wooly import Widget, Parameter
+from wooly.template import WidgetTemplate
+from wooly.forms import Form, FormField, FormError, MissingValueError,\
+ StringField
+from wooly.parameters import DictParameter, StringParameter, ListParameter,\
+ IntegerParameter
+from wooly.util import StringCatalog, Writer
+
strings = StringCatalog(__file__)
log = logging.getLogger("cumin.messaging.exchange")
@@ -127,6 +132,9 @@
self.add_join(exchange, binding.exchangeRef, exchange._id)
self.add_join(queue, binding.queueRef, queue._id)
+ def get_data(self, values, options):
+ return super(BindingData, self).get_data(values, options)
+
class BindingSelector(ObjectSelector):
def __init__(self, app, name):
binding = app.model.org_apache_qpid_broker.Binding
@@ -158,8 +166,6 @@
self.remove = BindingSelectionRemove(app, self)
- self.enable_csv_export()
-
class Exchange(ObjectLinkColumn):
def render_header_content(self, session):
return "Exchange"
@@ -600,7 +606,6 @@
class ExchangeBindings(ExchangeKeysField):
def render_id(self, session, *args):
- s = fmt_bytes(1)
cls = "ExchangeHiddenRow"
exchange = self.exchange.get(session)
if exchange and exchange.type == 'headers':
Modified: trunk/cumin/python/cumin/messaging/exchange.py
===================================================================
--- trunk/cumin/python/cumin/messaging/exchange.py 2011-02-16 01:33:22 UTC (rev 4532)
+++ trunk/cumin/python/cumin/messaging/exchange.py 2011-02-16 15:54:16 UTC (rev 4533)
@@ -113,6 +113,8 @@
self.exchange_column.visible = False
+ self.enable_csv_export(exchange)
+
class ExchangeInputSet(RadioInputSet):
def __init__(self, app, name):
super(ExchangeInputSet, self).__init__(app, name)
Modified: trunk/cumin/python/cumin/messaging/queue.py
===================================================================
--- trunk/cumin/python/cumin/messaging/queue.py 2011-02-16 01:33:22 UTC (rev 4532)
+++ trunk/cumin/python/cumin/messaging/queue.py 2011-02-16 15:54:16 UTC (rev 4533)
@@ -114,6 +114,8 @@
self.queue_column.visible = False
+ self.enable_csv_export(queue)
+
class QueueAdd(ObjectFrameTask):
def __init__(self, app, frame):
super(QueueAdd, self).__init__(app, frame)
13 years, 2 months
r4532 - trunk/cumin
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-16 01:33:22 +0000 (Wed, 16 Feb 2011)
New Revision: 4532
Modified:
trunk/cumin/Makefile
Log:
Ensure that $CUMIN_HOME directory exists before creating version file.
This may not be (is not) the case during rpm build.
Modified: trunk/cumin/Makefile
===================================================================
--- trunk/cumin/Makefile 2011-02-15 19:56:52 UTC (rev 4531)
+++ trunk/cumin/Makefile 2011-02-16 01:33:22 UTC (rev 4532)
@@ -42,20 +42,21 @@
find ${CUMIN_HOME} -type d
clean-version:
- @ -rm -f $(VERSION_FILE)
+ -rm -f $(VERSION_FILE)
write-version: clean-version
ifndef CUMIN_VERSION
@echo svn info failed, version file not created, this is okay
else
- @echo $(CUMIN_VERSION) > $(VERSION_FILE)
- @chmod 0644 $(VERSION_FILE)
+ install -d $(CUMIN_HOME)
+ echo $(CUMIN_VERSION) > $(VERSION_FILE)
+ chmod 0644 $(VERSION_FILE)
endif
mark-local:
#if the file exists, allow the local mark to be applied
ifeq ($(shell ls $(VERSION_FILE) 2> /dev/null),$(VERSION_FILE))
- @echo local >> $(VERSION_FILE)
+ echo local >> $(VERSION_FILE)
endif
version: write-version
13 years, 2 months