r5690 - branches/elephant/cumin/python/cumin/gridhadoop
by croberts@fedoraproject.org
Author: croberts
Date: 2013-02-12 19:35:19 +0000 (Tue, 12 Feb 2013)
New Revision: 5690
Modified:
branches/elephant/cumin/python/cumin/gridhadoop/datanode.py
branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py
branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py
branches/elephant/cumin/python/cumin/gridhadoop/namenode.py
branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py
Log:
Reworking the delete tasks to handle the implied scheduler mechanism.
Modified: branches/elephant/cumin/python/cumin/gridhadoop/datanode.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/datanode.py 2013-02-12 18:16:15 UTC (rev 5689)
+++ branches/elephant/cumin/python/cumin/gridhadoop/datanode.py 2013-02-12 19:35:19 UTC (rev 5690)
@@ -37,7 +37,7 @@
link = TaskLink(app, "dataNode_create", task)
self.links.add_child(link)
- DataNodeDelete(app, self, "dataNode_delete")
+ DataNodeDelete(app, self, "dataNode_delete", self.module)
self.enable_csv_export()
@@ -69,11 +69,13 @@
frame = self.frame.children_by_name[self.frame_path]
return frame.get_href(session, id, frame.parent.host.get(session))
-class DataNodeDelete(ObjectSelectorTask):
- def __init__(self, app, selector, name):
+class DataNodeDelete(HadoopNodeDeleteTask):
+ def __init__(self, app, selector, name, module):
super(DataNodeDelete, self).__init__(app, selector)
cls = app.model.com_redhat_cumin_grid_hadoop.DataNode
+ self.module = module
+
self.form = DataNodeDeleteForm(app, self.name, self, cls)
self.invoc = None
@@ -82,7 +84,6 @@
(hadoophost) = args
self.app.remote.stop_data_node(hadoophost, [float(id)], invoc.make_callback())
-
def get_title(self, session):
return "Remove data nodes"
Modified: branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py 2013-02-12 18:16:15 UTC (rev 5689)
+++ branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py 2013-02-12 19:35:19 UTC (rev 5690)
@@ -71,8 +71,8 @@
super(HadoopNodeDeleteForm, self).__init__(app, name, task)
self.task = task
- self.hadoophost = HadoopHostField(app, "hadoophost")
- self.add_child(self.hadoophost)
+ self.hadoophost = Parameter(app, "hadoophost")
+ self.add_parameter(self.hadoophost)
def get_selection(self, session):
ids = self.ids.get(session)
@@ -87,9 +87,9 @@
# also need to render any fields by hand here since
# the usual content block gets swapped out
content = ""
- content += '''<table class="FormFieldSet"><tbody>'''
- content += self.hadoophost.render(session)
- content += '''</tbody></table>'''
+# content += '''<table class="FormFieldSet"><tbody>'''
+# content += self.hadoophost.render(session)
+# content += '''</tbody></table>'''
content += self.content.render(session, *args)
return content
@@ -188,9 +188,6 @@
def render_title(self, session):
return "Name node"
-
- def render_help(self, session):
- return "Attach data nodes to this name node"
class NameNodeOptions(OptionInputSet):
def do_process(self, session):
@@ -263,4 +260,11 @@
def __repr__(self):
name = self.__class__.__name__
args = (name, self._class, self._id, self._save_time)
- return "%s(%s,%s,%s)" % args
\ No newline at end of file
+ return "%s(%s,%s,%s)" % args
+
+
+class HadoopNodeDeleteTask(ObjectSelectorTask):
+ def enter(self, session):
+ form_session = super(HadoopNodeDeleteTask, self).enter(session)
+ self.form.hadoophost.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"].host.get(session))
+ return form_session
\ No newline at end of file
Modified: branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py 2013-02-12 18:16:15 UTC (rev 5689)
+++ branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py 2013-02-12 19:35:19 UTC (rev 5690)
@@ -28,16 +28,18 @@
def __init__(self, app, name, module, check_owner=False):
cls = app.model.com_redhat_cumin_grid_hadoop.JobTracker
+ self.module = module
+
super(JobTrackerSelector, self).__init__(app, name, cls)
self.add_search_filter(self.table.id_col)
self.table.adapter = JobTrackerAdapter(app, cls, check_owner)
- task = JobTrackerCreate(app, name + ".create")
+ task = JobTrackerCreate(app, name + ".create", self.module)
link = TaskLink(app, "jobTracker_create", task)
self.links.add_child(link)
- JobTrackerDelete(app, self, "jobTracker_delete")
+ JobTrackerDelete(app, self, "jobTracker_delete", self.module)
self.enable_csv_export()
@@ -72,10 +74,12 @@
frame = self.frame.children_by_name[self.frame.name + self.frame_path]
return frame.get_href(session, id, frame.parent.host.get(session))
-class JobTrackerDelete(ObjectSelectorTask):
- def __init__(self, app, selector, name):
+class JobTrackerDelete(HadoopNodeDeleteTask):
+ def __init__(self, app, selector, name, module):
super(JobTrackerDelete, self).__init__(app, selector)
cls = app.model.com_redhat_cumin_grid_hadoop.JobTracker
+
+ self.module = module
self.form = JobTrackerDeleteForm(app, self.name, self, cls)
self.invoc = None
@@ -107,10 +111,12 @@
return "Remove job trackers"
class JobTrackerCreate(Task):
- def __init__(self, app, name):
+ def __init__(self, app, name, module):
super(JobTrackerCreate, self).__init__(app, name)
cls = app.model.com_redhat_cumin_grid_hadoop.JobTracker
+ self.module = module
+
self.form = JobTrackerCreateForm(app, self.name, self, cls)
self.invoc = None
@@ -188,13 +194,15 @@
def __init__(self, app, name, module):
super(JobTrackerFrame, self).__init__(app, name, module)
+ self.module = module
+
self.view = JobTrackerObjectView(app, "view", self.object)
self.replace_child(self.view)
self.task_tracker_frame = TaskTrackerFrame(app, "tasktrackerframe", module)
self.add_mode(self.task_tracker_frame)
- self.overview = JobTrackerOverview(app, self.name + ".jtoverview", self.object)
+ self.overview = JobTrackerOverview(app, self.name + ".jtoverview", self.object, self.module)
self.view.add_tab(self.overview)
def get_title(self, session):
@@ -218,9 +226,9 @@
self.add_tab(JobTrackerDetails(self.app, "details", self.object))
class JobTrackerOverview(Widget):
- def __init__(self, app, name, node):
+ def __init__(self, app, name, node, module):
super(JobTrackerOverview, self).__init__(app, name)
- self.add_child(TaskTrackerSelector(app, self.name + ".ttforjt", False))
+ self.add_child(TaskTrackerSelector(app, self.name + ".ttforjt", module, False))
def render_title(self, session):
return "Task trackers"
Modified: branches/elephant/cumin/python/cumin/gridhadoop/namenode.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/namenode.py 2013-02-12 18:16:15 UTC (rev 5689)
+++ branches/elephant/cumin/python/cumin/gridhadoop/namenode.py 2013-02-12 19:35:19 UTC (rev 5690)
@@ -1,5 +1,5 @@
from cumin.objectselector import ObjectSelector, ObjectLinkColumn, ObjectTable,\
- ObjectTableColumn, ObjectQmfTable, ObjectQmfSelectorTable, ObjectSelectorTaskForm, ObjectSelectorTask
+ ObjectTableColumn, ObjectQmfTable, ObjectQmfSelectorTable, ObjectSelectorTaskForm
from cumin.objectframe import ObjectFrame, ObjectTaskForm
from cumin.util import *
from cumin.qmfadapter import ObjectQmfAdapter
@@ -35,7 +35,7 @@
link = TaskLink(app, "nameNode_create", task)
self.links.add_child(link)
- NameNodeDelete(app, self, "nodeName_delete")
+ NameNodeDelete(app, self, "nodeName_delete", module)
self.enable_csv_export()
@@ -136,11 +136,13 @@
def render_title(self, session):
return "Create a name node"
-class NameNodeDelete(ObjectSelectorTask):
- def __init__(self, app, selector, name):
+class NameNodeDelete(HadoopNodeDeleteTask):
+ def __init__(self, app, selector, name, module):
super(NameNodeDelete, self).__init__(app, selector)
cls = app.model.com_redhat_cumin_grid_hadoop.NameNode
+ self.module = module
+
self.form = NameNodeDeleteForm(app, self.name, self, cls)
self.invoc = None
Modified: branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py 2013-02-12 18:16:15 UTC (rev 5689)
+++ branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py 2013-02-12 19:35:19 UTC (rev 5690)
@@ -23,19 +23,21 @@
return results
class TaskTrackerSelector(ObjectSelector):
- def __init__(self, app, name, check_owner=False):
+ def __init__(self, app, name, module, check_owner=False):
cls = app.model.com_redhat_cumin_grid_hadoop.TaskTracker
super(TaskTrackerSelector, self).__init__(app, name, cls)
+
+ self.module = module
self.add_search_filter(self.table.id_col)
self.table.adapter = TaskTrackerAdapter(app, cls, check_owner)
- task = TaskTrackerCreate(app, name + ".create")
+ task = TaskTrackerCreate(app, name + ".create", self.module)
link = TaskLink(app, "taskTracker_create", task)
self.links.add_child(link)
- TaskTrackerDelete(app, self, "taskTracker_delete")
+ TaskTrackerDelete(app, self, "taskTracker_delete", self.module)
self.enable_csv_export()
@@ -67,10 +69,12 @@
frame = self.frame.children_by_name[self.frame_path]
return frame.get_href(session, id, frame.parent.host.get(session))
-class TaskTrackerDelete(ObjectSelectorTask):
- def __init__(self, app, selector, name):
+class TaskTrackerDelete(HadoopNodeDeleteTask):
+ def __init__(self, app, selector, name, module):
super(TaskTrackerDelete, self).__init__(app, selector)
cls = app.model.com_redhat_cumin_grid_hadoop.TaskTracker
+
+ self.module = module
self.form = TaskTrackerDeleteForm(app, self.name, self, cls)
self.invoc = None
@@ -102,10 +106,12 @@
return "Remove task trackers"
class TaskTrackerCreate(Task):
- def __init__(self, app, name):
+ def __init__(self, app, name, module):
super(TaskTrackerCreate, self).__init__(app, name)
cls = app.model.com_redhat_cumin_grid_hadoop.TaskTracker
+ self.module = module
+
self.form = TaskTrackerCreateForm(app, self.name, self, cls)
self.invoc = None
@@ -265,6 +271,8 @@
def __init__(self, app, name, module):
super(TaskTrackerFrame, self).__init__(app, name, module)
+ self.module = module
+
self.view = TaskTrackerObjectView(app, "view", self.object)
self.replace_child(self.view)
11 years, 2 months
r5689 - in branches/tmckay: etc mint/python/mint
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-02-12 18:16:15 +0000 (Tue, 12 Feb 2013)
New Revision: 5689
Modified:
branches/tmckay/etc/Profile.bash
branches/tmckay/mint/python/mint/update.py
Log:
you know why...
Modified: branches/tmckay/etc/Profile.bash
===================================================================
--- branches/tmckay/etc/Profile.bash 2013-02-12 15:56:13 UTC (rev 5688)
+++ branches/tmckay/etc/Profile.bash 2013-02-12 18:16:15 UTC (rev 5689)
@@ -1,5 +1,5 @@
export DEVEL_HOME="$PWD"
-export DEVEL_MODULES="mint cumin basil parsley wooly rosemary sage"
+export DEVEL_MODULES="mint cumin parsley wooly rosemary sage"
# This will control certain things, like whether or not
# cumin-database prompts you for the creation of a
Modified: branches/tmckay/mint/python/mint/update.py
===================================================================
--- branches/tmckay/mint/python/mint/update.py 2013-02-12 15:56:13 UTC (rev 5688)
+++ branches/tmckay/mint/python/mint/update.py 2013-02-12 18:16:15 UTC (rev 5689)
@@ -736,11 +736,11 @@
self.update_object(cursor, stats, obj)
def get_agent_id(self):
- return self.aviary_obj.id.resource + ":" + \
- self.aviary_obj.id.name
+ return str(self.aviary_obj.id.resource) + ":" + \
+ str(self.aviary_obj.id.name)
def get_object_id(self):
- return self.aviary_obj.id.name
+ return str(self.aviary_obj.id.name)
def delete_object(self, cursor, stats, obj):
obj.delete(cursor)
@@ -1029,7 +1029,7 @@
self.aviary_obj.id.name
def get_scheduler_name(self):
- return self.aviary_obj.id._scheduler
+ return str(self.aviary_obj.id._scheduler)
class AviaryCollectorUpdate(AviaryUpdate):
@@ -1057,7 +1057,7 @@
return self.aviary_obj.id.name
def get_scheduler_name(self):
- return self.aviary_obj.id.scheduler
+ return str(self.aviary_obj.id.scheduler)
class UpdateDropped(Exception):
pass
11 years, 2 months
r5688 - in branches/tmckay/cumin: etc python/cumin
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-02-12 15:56:13 +0000 (Tue, 12 Feb 2013)
New Revision: 5688
Modified:
branches/tmckay/cumin/etc/cumin.conf
branches/tmckay/cumin/python/cumin/util.py
Log:
sigh, more of the same
Modified: branches/tmckay/cumin/etc/cumin.conf
===================================================================
--- branches/tmckay/cumin/etc/cumin.conf 2013-02-12 15:46:46 UTC (rev 5687)
+++ branches/tmckay/cumin/etc/cumin.conf 2013-02-12 15:56:13 UTC (rev 5688)
@@ -13,9 +13,7 @@
[common]
# database: dbname=cumin
-# brokers: localhost:5672
-# sasl-mech-list: [default, 'anonymous' or 'plain digest-md5' with usr/passw]
-# wallaby-broker: [default, first item in 'brokers' list]
+# wallaby-broker: http://localhost
# wallaby-refresh: 60
# log-level: info
# log-max-mb: 10
@@ -77,7 +75,7 @@
# Default cumin-data configuration at installation.
# Expiration of samples and vacuuming of the database is handled
# by the data.grid thread.
-datas: data.grid, data.grid-slots, data.grid-submissions, data.sesame
+datas: data.grid, data.grid-slots, data.grid-submissions
# To run a single cumin-data process in a small deployment set 'datas' as follows
# and optionally modify 'include-classes' and 'exclude-classes' in the [data]
Modified: branches/tmckay/cumin/python/cumin/util.py
===================================================================
--- branches/tmckay/cumin/python/cumin/util.py 2013-02-12 15:46:46 UTC (rev 5687)
+++ branches/tmckay/cumin/python/cumin/util.py 2013-02-12 15:56:13 UTC (rev 5688)
@@ -6,7 +6,6 @@
from crypt import crypt
from datetime import datetime, timedelta
-from qpid.datatypes import uuid4
from pprint import *
from random import randint
from random import sample
@@ -310,4 +309,4 @@
days = dd * 86400
hours = hh * 3600
minutes = mm * 60
- return days + hours + minutes
\ No newline at end of file
+ return days + hours + minutes
11 years, 2 months
r5687 - branches/elephant/cumin/python/cumin/gridhadoop
by croberts@fedoraproject.org
Author: croberts
Date: 2013-02-12 15:46:46 +0000 (Tue, 12 Feb 2013)
New Revision: 5687
Modified:
branches/elephant/cumin/python/cumin/gridhadoop/datanode.py
branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py
branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py
branches/elephant/cumin/python/cumin/gridhadoop/main.py
branches/elephant/cumin/python/cumin/gridhadoop/namenode.py
branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py
branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py
Log:
Additional fixes for user vs admin and frame linking. Also cleaning up comments that were no longer useful.
Modified: branches/elephant/cumin/python/cumin/gridhadoop/datanode.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/datanode.py 2013-02-12 15:46:17 UTC (rev 5686)
+++ branches/elephant/cumin/python/cumin/gridhadoop/datanode.py 2013-02-12 15:46:46 UTC (rev 5687)
@@ -144,7 +144,6 @@
if obj:
self.form.id.set(form_session, obj._id)
- ## TODO find a better way to do this
self.form.hadoophost.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"].host.get(session))
self.form.nn_id.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"]\
.children_by_name[self.module.frame.name + ".ghschedframe.namenodeframe"].id.get(session))
Modified: branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py 2013-02-12 15:46:17 UTC (rev 5686)
+++ branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py 2013-02-12 15:46:46 UTC (rev 5687)
@@ -61,7 +61,7 @@
def get_owner_filter(self, session):
if self.check_owner:
- owner_list = ["condor", session.get_user().name] #condor is a temporary hack
+ owner_list = [session.get_user().name]
else:
owner_list = None
return owner_list
Modified: branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py 2013-02-12 15:46:17 UTC (rev 5686)
+++ branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py 2013-02-12 15:46:46 UTC (rev 5687)
@@ -142,7 +142,6 @@
if obj:
self.form.id.set(form_session, obj._id)
- ## TODO find a better way to do this
self.form.hadoophost.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].host.get(session))
self.form.return_url.set(form_session, session.marshal())
self.form.show(form_session)
Modified: branches/elephant/cumin/python/cumin/gridhadoop/main.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/main.py 2013-02-12 15:46:17 UTC (rev 5686)
+++ branches/elephant/cumin/python/cumin/gridhadoop/main.py 2013-02-12 15:46:46 UTC (rev 5687)
@@ -43,12 +43,6 @@
self.scheduler_frame = HadoopSchedulerFrame(app, name + ".ghschedframe", check_owner, self.module)
self.add_mode(self.scheduler_frame)
-
-# self.name_node_frame = NameNodeFrame(app, name + ".namenodeframe")
-# self.add_mode(self.name_node_frame)
-#
-# self.job_tracker_frame = JobTrackerFrame(app, name + ".jobtrackerframe")
-# self.add_mode(self.job_tracker_frame)
def render_title(self, session):
return "Grid-hadoop"
@@ -57,12 +51,7 @@
def __init__(self, app, name, check_owner):
super(GridHadoopView, self).__init__(app, name)
-# heading = self.Heading(app, "heading")
-# self.add_child(heading)
-
self.tabs = TabbedModeSet(app, "childtabs")
self.add_child(self.tabs)
- #self.tabs.add_tab(NameNodeSelector(app, name + ".namenodes", check_owner))
- #self.tabs.add_tab(JobTrackerSelector(app, name + ".jobtrackers", check_owner))
self.tabs.add_tab(HadoopSchedulerSelector(app, name + ".hscheds"))
Modified: branches/elephant/cumin/python/cumin/gridhadoop/namenode.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/namenode.py 2013-02-12 15:46:17 UTC (rev 5686)
+++ branches/elephant/cumin/python/cumin/gridhadoop/namenode.py 2013-02-12 15:46:46 UTC (rev 5687)
@@ -49,7 +49,6 @@
def __init__(self, app, name, cls):
super(NameNodeTable, self).__init__(app, name, cls)
- #probably doesnt' work for user/grid-hadoop path
self.id_col = self.NameNodeColumn(app, "idcol", cls.Id, cls.Id, ".namenodeframe")
self.ipc_col = ObjectTableColumn(app, "ipc", cls.Ipc)
self.ipc_col.width = "20%"
Modified: branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py 2013-02-12 15:46:17 UTC (rev 5686)
+++ branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py 2013-02-12 15:46:46 UTC (rev 5687)
@@ -63,9 +63,7 @@
class SchedNameColumn(ObjectLinkColumn):
def render_cell_href(self, session, record):
id = unescape_entity(record[self.id_field.index])
- #need to make this work for user/grid-hadoop path too somehow
- # this is way less than ideal
- frame = self.page.page_widgets_by_path[self.path.rsplit(".", 7)[0] + self.frame_path]
+ frame = self.frame.children_by_name[self.frame.name + self.frame_path]
return frame.get_href(session, id, id)
Modified: branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py 2013-02-12 15:46:17 UTC (rev 5686)
+++ branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py 2013-02-12 15:46:46 UTC (rev 5687)
@@ -134,7 +134,6 @@
if obj:
self.form.id.set(form_session, obj._id)
- ## TODO find a better way to do this
self.form.hadoophost.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].host.get(session))
self.form.jt_id.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].children_by_name["gridhadoop.ghschedframe.jobtrackerframe"].id.get(session))
self.form.return_url.set(form_session, session.marshal())
11 years, 2 months
r5686 - in branches/tmckay: cumin/bin cumin/python/cumin mint/python/mint
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-02-12 15:46:17 +0000 (Tue, 12 Feb 2013)
New Revision: 5686
Added:
branches/tmckay/mint/python/mint/data_app.py
Modified:
branches/tmckay/cumin/bin/cumin-admin
branches/tmckay/cumin/python/cumin/objecttask.py
Log:
Mods to make install from package work
Modified: branches/tmckay/cumin/bin/cumin-admin
===================================================================
--- branches/tmckay/cumin/bin/cumin-admin 2013-02-12 15:12:42 UTC (rev 5685)
+++ branches/tmckay/cumin/bin/cumin-admin 2013-02-12 15:46:17 UTC (rev 5686)
@@ -108,7 +108,7 @@
authmech = [x.strip() for x in values.common.auth.split(";") if x != ""]
- app = Cumin(config.get_home(), [], opts.database, authmech=authmech)
+ app = Cumin(config.get_home(), opts.database, authmech=authmech)
app.check()
Modified: branches/tmckay/cumin/python/cumin/objecttask.py
===================================================================
--- branches/tmckay/cumin/python/cumin/objecttask.py 2013-02-12 15:12:42 UTC (rev 5685)
+++ branches/tmckay/cumin/python/cumin/objecttask.py 2013-02-12 15:46:17 UTC (rev 5686)
@@ -1,6 +1,3 @@
-from qpid.connection import *
-from qpid.util import *
-
from rosemary.model import *
from wooly.forms import *
from wooly.resources import *
Added: branches/tmckay/mint/python/mint/data_app.py
===================================================================
--- branches/tmckay/mint/python/mint/data_app.py (rev 0)
+++ branches/tmckay/mint/python/mint/data_app.py 2013-02-12 15:46:17 UTC (rev 5686)
@@ -0,0 +1,311 @@
+#!/usr/bin/python
+
+import os
+import sys
+from traceback import print_exc
+
+from cumin.config import *
+from cumin.util import *
+from parsley.loggingex import PipeLogThread
+from psycopg2 import OperationalError
+from cumin.admin import SchemaVersion, SchemaMissing
+from cumin.errors import CuminErrors
+
+def restore_IO():
+ sys.stderr = sys.__stderr__
+ sys.stdout = sys.__stdout__
+
+def process_classes(mint, values, section_name, on_empty=None):
+
+ return_code = CuminErrors.NO_ERROR
+ pkgs = set()
+ if values and len(values) > 0:
+
+ for cls_str in values.split(","):
+ pair = cls_str.strip().split(":")
+ if len(pair) == 2:
+ pname = pair[0]
+ cname = pair[1]
+ else:
+ log.error("Configuration section '%s',"\
+ " class name '%s' is badly formed"\
+ % (section_name, cls_str.strip()))
+ return_code = CuminErrors.PARSE_ERROR
+ break
+
+ try:
+ pkg = mint.model._packages_by_name[pname]
+ if cname == "*":
+ pkgs = pkgs.union(set(pkg._classes))
+ else:
+ try:
+ cls = pkg._classes_by_name[cname]
+ pkgs.add(cls)
+ except KeyError:
+ log.warning("Configuration section '%s',"\
+ " class '%s' is not contained in package"\
+ " '%s'" % (section_name, cname, pname))
+ except KeyError:
+ log.warning("Configuration section '%s',"\
+ " package '%s' not found" % (section_name, pname))
+ elif on_empty == "all":
+ for pkg in mint.model._packages:
+ pkgs = pkgs.union(set(pkg._classes))
+
+ return return_code, pkgs
+
+def adjust_return(passed_init, ret):
+ # Shift non-zer0 return codes left 1 bit
+ # and OR in whether or not init passed
+ if ret != 0:
+ ret = ret << 1 | passed_init
+ return ret
+
+def write_status_path(p, value):
+ try:
+ f = open(p, "w+")
+ if value:
+ f.write(value)
+ f.close()
+ except:
+ pass
+
+class DataAppTemplate(object):
+ def __init__(self):
+
+ if not hasattr(self, "default_data_section"):
+ raise Exception("DataAppTemplate must define "\
+ "member default_data_section")
+
+ def get_parser(self):
+ raise Exception("Not implemented")
+
+ def get_config(self, section):
+ raise Exception("Not implemented")
+
+ def get_app(self, home, values, opts):
+ raise Exception("Not implemented")
+
+def main(template):
+ assert isinstance(template, DataAppTemplate)
+
+ passed_init = 1
+ return_code = CuminErrors.NO_ERROR
+ wrote_status = False
+ config = None
+
+ # Do our own simple option check so we can redirect IO early
+ # without worrying about other options or the behavior of optParse
+ opts = check_for_options(["--section", "--daemon"], sys.argv[1:])
+ if type(opts["--section"]) is str:
+ section_name = opts["--section"]
+ else:
+ section_name = template.default_data_section
+
+ # If the --daemon option has been set, redirect stderr and stdio through
+ # pipes. Launch a thread to read those pipes and direct the content to
+ # files with rollover control.
+ pipeThread = None
+ if opts["--daemon"]:
+ try:
+ cumin_home = os.environ.get("CUMIN_HOME", os.path.normpath("/usr/share/cumin"))
+ err_file = os.path.join(cumin_home, "log", section_name+".stderr")
+ out_file = os.path.join(cumin_home, "log", section_name+".stdout")
+
+ # Open pipes and redirect
+ err_r, err_w = os.pipe()
+ out_r, out_w = os.pipe()
+ pipeThread = PipeLogThread(descriptors=[err_r, out_r],
+ paths=[err_file, out_file],
+ call_on_fail=restore_IO)
+ pipeThread.start()
+ sys.stderr = os.fdopen(err_w, "w", 0)
+ sys.stdout = os.fdopen(out_w, "w", 0)
+ except:
+ print_exc()
+ pipeThread = None
+
+ class ArgError(Exception):
+ pass
+
+ try:
+ mint = None
+ setup_initial_logging()
+
+ parser = template.get_parser()
+
+ # Add additional parameters
+ parser.add_option("--print-stats", action="store_true", default=False)
+ parser.add_option("--print-events", type="int", default=0, metavar="LEVEL")
+ parser.add_option("--section", default=template.default_data_section)
+ parser.add_option("--daemon", action="store_true", default=False)
+ parser.add_option("--no-vacuum", dest="vacuum_enabled", action="store_false")
+ parser.add_option("--no-expire", dest="expire_enabled", action="store_false")
+ parser.add_option("--delete-on-start", action="store_true", default=False)
+
+ # Get options
+ opts, args = parser.parse_args()
+
+ config = template.get_config(opts.section)
+
+ # There might be other sections returned from config.parse() but
+ # currently we don't care about them...
+ values = getattr(config.parse(), opts.section)
+
+ # Use the config values as defaults for unspecified options
+ apply_defaults(values, opts)
+
+ # If log_file is a relative path, base it from $CUMIN_HOME/log
+ opts.log_file = config.expand_log_file(opts.log_file)
+ setup_operational_logging(opts,
+ values.log_max_mb,
+ values.log_max_archives)
+
+ if len(args) != 0:
+ log.error("Extra arguments:" + "".join([" "+arg for arg in args]))
+ raise ArgError
+
+ mint = template.get_app(config.home, values, opts)
+
+ mint.print_event_level = opts.print_events
+
+ mint.expire_enabled = opts.expire_enabled
+ mint.expire_thread.interval = values.expire_interval
+ mint.expire_thread.threshold = values.expire_threshold
+
+ mint.vacuum_enabled = opts.vacuum_enabled
+ mint.vacuum_thread.interval = values.vacuum_interval
+
+ # This builds the data model, so it needs to be
+ # called before we process classes
+ mint.check()
+
+ # Handle class binding options
+ if values.include_classes or values.exclude_classes:
+ return_code, includes = process_classes(mint,
+ values.include_classes,
+ opts.section, on_empty="all")
+
+ return_code, excludes = process_classes(mint,
+ values.exclude_classes,
+ opts.section)
+
+ mint.set_classes(includes.difference(excludes))
+
+ # Finish initialization
+ mint.init()
+
+ # If we've been instructed to delete timestamp data at
+ # startup, set the flag in the update thread where the
+ # delete logic is located...
+ mint.update_thread.del_on_start = opts.delete_on_start
+
+ # ... and if furthermore the init_only flag is set,
+ # cause the delete to happen before we return
+ if opts.init_only and opts.delete_on_start:
+ mint.update_thread._delete_all_objects()
+
+ # Okay, record the init check status here
+ write_status_path(config.get_init_status_path(), str(return_code)+"\n")
+ wrote_status = True
+
+ # If init_only was set or we failed init, don't proceed...
+ if not opts.init_only and not return_code:
+ passed_init = 0
+
+ mint.start()
+ stats = mint.update_thread.stats
+ count = 0
+
+ exit_msg = "Exiting on termination string from master"
+
+ if opts.print_stats:
+ print "[Starred columns are the number of events per second]"
+
+ while True:
+ if count % 20 == 0:
+ stats.print_headings()
+
+ count += 1
+
+ stats.print_values()
+ if opts.es:
+ if poll_stdin(5, value=opts.es):
+ log.debug(exit_msg)
+ break
+ else:
+ sleep(5)
+ else:
+ while True:
+ if opts.es:
+ if poll_stdin(86400, value=opts.es):
+ log.debug(exit_msg)
+ break
+ else:
+ sleep(86400)
+
+ except KeyboardInterrupt:
+ log.info("Received shutdown signal")
+
+ except SystemExit:
+ if "--help" not in sys.argv:
+ log.error("Error in options")
+ return_code = CuminErrors.PARSE_ERROR
+
+ except ArgError:
+ return_code = CuminErrors.PARSE_ERROR
+
+ except OperationalError:
+ # Failed to talk to the database on check()
+ log.info("Run 'cumin-database check' as root for more information.")
+ return_code = CuminErrors.DATABASE_ERROR
+
+ except SchemaMissing:
+ log.info("Run 'cumin-admin create-schema' as root")
+ return_code = CuminErrors.SCHEMA_ERROR
+
+ except SchemaVersion:
+ log.info("Run 'cumin-admin upgrade-schema' as root")
+ return_code = CuminErrors.SCHEMA_VER_ERROR
+
+ except:
+ print_exc()
+ return_code = CuminErrors.UNHANDLED_ERROR
+
+ # For parse errors, for example, we won't have a config object yet
+ # but the return code will indicate an init error anyway so a
+ # parent can check the return code.
+ if not wrote_status and config:
+ write_status_path(config.get_init_status_path(), str(return_code)+"\n")
+
+ def shutdown():
+ if mint:
+ # Don't stop vacuum here. If a vacuum is happening,
+ # we want to let it complete. Stop it below.
+ mint.stop(skip_vacuum=True)
+
+ # Run shutdown with a timeout in the main thread.
+ # This guarantees exit.
+ t = ShutdownThread(shutdown)
+ t.start()
+
+ # If we get some error before options are parsed opts.tm won't be set
+ try:
+ tm = opts.tm
+ except:
+ tm = 5
+ t.join(tm)
+ if t.isAlive():
+ log.info("Shutdown thread timed out, exiting")
+ if pipeThread:
+ pipeThread.stop(immediate=True)
+
+ # Vacuum is not interruptable and could take a while.
+ # This call to stop will wait for vacuum to exit
+ if mint:
+ mint.vacuum_thread.stop()
+
+ log.info("about to call logging shutdown")
+ logging.shutdown()
+
+ return adjust_return(passed_init, return_code)
11 years, 2 months
r5685 - branches/tmckay
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-02-12 15:12:42 +0000 (Tue, 12 Feb 2013)
New Revision: 5685
Removed:
branches/tmckay/sesame/
Log:
And sesame
11 years, 2 months
r5684 - in branches/tmckay: . cumin/bin cumin/model/access cumin/python/cumin mint/python/mint sage/python/sage
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-02-12 15:11:33 +0000 (Tue, 12 Feb 2013)
New Revision: 5684
Removed:
branches/tmckay/basil/
branches/tmckay/cumin/python/cumin/messaging/
branches/tmckay/cumin/python/cumin/session.py
branches/tmckay/mace/
branches/tmckay/mint/python/mint/session.py
branches/tmckay/sage/python/sage/qmf/
Modified:
branches/tmckay/cumin/bin/cumin-data
branches/tmckay/cumin/bin/cumin-web
branches/tmckay/cumin/model/access/persona.xml
branches/tmckay/cumin/python/cumin/config.py
branches/tmckay/cumin/python/cumin/main.py
branches/tmckay/cumin/python/cumin/model.py
branches/tmckay/mint/python/mint/main.py
branches/tmckay/mint/python/mint/util.py
Log:
Remove unnecessary code
Modified: branches/tmckay/cumin/bin/cumin-data
===================================================================
--- branches/tmckay/cumin/bin/cumin-data 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/cumin/bin/cumin-data 2013-02-12 15:11:33 UTC (rev 5684)
@@ -6,7 +6,7 @@
home = os.environ.get("CUMIN_HOME", os.path.normpath("/usr/share/cumin"))
sys.path.append(os.path.join(home, "python"))
-from cumin.config import BrokeredOptionParser, CuminDataConfig
+from cumin.config import CuminOptionParser, CuminDataConfig
from mint import data_app
def make_ctrl_c(sig, frame):
@@ -20,7 +20,7 @@
super(DataTemplate, self).__init__()
def get_parser(self):
- return BrokeredOptionParser()
+ return CuminOptionParser()
def get_config(self, section):
# --section controls which section is read from the config file
@@ -31,16 +31,10 @@
def get_app(self, home, values, opts):
from mint.main import Mint
- broker_uris = [x.strip() for x in opts.brokers.split(",")]
-
mint = Mint(home,
- ("model", "model/admin"),
- broker_uris,
+ ("model", "model/admin"),
opts.database)
- if type(values.sasl_mech_list) == str:
- mint.sasl_mech_list = values.sasl_mech_list.upper()
-
mint.aviary_query_servers = values.aviary_query_servers
mint.aviary_collector_servers = values.aviary_collector_servers
mint.aviary_locator = values.aviary_locator
Modified: branches/tmckay/cumin/bin/cumin-web
===================================================================
--- branches/tmckay/cumin/bin/cumin-web 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/cumin/bin/cumin-web 2013-02-12 15:11:33 UTC (rev 5684)
@@ -33,7 +33,7 @@
# Defautl is True. Undocumented.
cumin.aviary_prefer_condor = values.aviary_prefer_condor
-def set_wallaby_configs(cumin, values, brokers):
+def set_wallaby_configs(cumin, values):
if values.wallaby_broker == "None":
values.wallaby_broker = None
@@ -162,7 +162,7 @@
setup_initial_logging()
- parser = BrokeredOptionParser()
+ parser = CuminOptionParser()
# Add additional parameters for web
parser.add_option("--host")
@@ -199,22 +199,15 @@
log.error("Extra arguments:" + "".join([" "+arg for arg in args]))
raise ArgError
- broker_uris = [x.strip() for x in opts.brokers.split(",")]
authmech = [x.strip() for x in values.auth.split(";") if x != ""]
- cumin = Cumin(config.get_home(), broker_uris, opts.database,
+ cumin = Cumin(config.get_home(), opts.database,
opts.host, opts.port,
opts.server_cert, opts.server_key,
values.force_secure_cookies,
values.persona, authmech)
- # If the mech_list was set explicitly, honor it.
- # Otherwise Cumin will default based on whether or not
- # a broker URL contains credentials.
- if type(values.sasl_mech_list) == str:
- cumin.sasl_mech_list = values.sasl_mech_list.upper()
-
set_aviary_configs(cumin, values)
- set_wallaby_configs(cumin, values, broker_uris)
+ set_wallaby_configs(cumin, values)
set_ldap_configs(cumin, values)
set_kerberos_configs(cumin, values)
Modified: branches/tmckay/cumin/model/access/persona.xml
===================================================================
--- branches/tmckay/cumin/model/access/persona.xml 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/cumin/model/access/persona.xml 2013-02-12 15:11:33 UTC (rev 5684)
@@ -1,28 +1,4 @@
<PersonaMap>
- <Persona name="default" auth="True">
- <Module name="account"/>
- <Module name="configuration"/>
- <Module name="messaging"/>
- <Module name="grid"/>
- <Module name="inventory"/>
- <Module name="usergrid"/>
-
- <GroupAccess name="nogroup">
- <MainPage name="login.html"/>
- </GroupAccess>
-
- <GroupAccess name="user">
- <MainPage name="usergrid.html"/>
- <ModuleAccess name="account"/>
- <ModuleAccess name="usergrid"/>
- </GroupAccess>
-
- <GroupAccess name="admin">
- <MainPage name="index.html"/>
- <ModuleAccess name="*"/>
- </GroupAccess>
- </Persona>
-
<Persona name="grid" auth="True">
<Module name="account"/>
<Module name="configuration"/>
@@ -45,10 +21,4 @@
<ModuleAccess name="*"/>
</GroupAccess>
</Persona>
-
- <Persona name="messaging" auth="False">
- <Module name="account"/>
- <Module name="messaging"/>
- <Module name="inventory"/>
- </Persona>
</PersonaMap>
Modified: branches/tmckay/cumin/python/cumin/config.py
===================================================================
--- branches/tmckay/cumin/python/cumin/config.py 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/cumin/python/cumin/config.py 2013-02-12 15:11:33 UTC (rev 5684)
@@ -167,13 +167,6 @@
super(BrokeredConfigSection, self).__init__(config, name,
strict_section)
- param = ConfigParameter(self, "brokers", str)
- param.default = "amqp://localhost"
-
- # Leave default set to None, which is equivalent to
- # previous behavior
- param = ConfigParameter(self, "sasl-mech-list", str)
-
param = ConfigParameter(self, "aviary-query-servers", str)
param.default = "http://localhost:9091"
@@ -315,14 +308,6 @@
self.add_option("--log-file")
self.add_option("--log-level")
-class BrokeredOptionParser(CuminOptionParser):
- def __init__(self):
- CuminOptionParser.__init__(self)
-
- # Defaults for these come later from the config
- # section named in --section, via apply_defaults()
- self.add_option("--brokers")
-
class PlumageOptionParser(CuminOptionParser):
def __init__(self):
CuminOptionParser.__init__(self)
Modified: branches/tmckay/cumin/python/cumin/main.py
===================================================================
--- branches/tmckay/cumin/python/cumin/main.py 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/cumin/python/cumin/main.py 2013-02-12 15:11:33 UTC (rev 5684)
@@ -15,7 +15,6 @@
from objectselector import *
from objecttask import *
from server import *
-from session import *
from sqladapter import *
from user import *
from util import *
@@ -23,16 +22,14 @@
from authenticator import *
from persona import CuminAuthorizator
from sage.catalog import Catalog
-from sage.qmf.qmfoperations import QmfOperations
from sage.wallaby.wallabyoperations import WallabyOperations
-from wooly import Session
from cumin.widgets import AboutPage
strings = StringCatalog(__file__)
log = logging.getLogger("cumin")
class Cumin(Application):
- def __init__(self, home, broker_uris, database_dsn,
+ def __init__(self, home, database_dsn,
host="localhost", port=45672,
server_cert="", server_key="",
force_secure_cookies=False,
@@ -54,7 +51,6 @@
model_dir = [os.path.join(self.home, x) for x in ("model/admin", "model", "model/plumage")]
self.model = CuminModel(self, model_dir)
- self.session = CuminSession(self, broker_uris)
self.database = CuminDatabase(self, database_dsn)
self.server = CuminServer(self, host, port, server_cert, server_key)
@@ -96,10 +92,6 @@
self.model.sql_logging_enabled = True
- # Space separated list of sasl authentication
- # mechanisms, according to the sasl documentation
- self.sasl_mech_list = None
-
# Aviary interface. If server values are "",
# Aviary operations for that server type will not be used.
self.aviary_job_servers = ""
@@ -134,13 +126,7 @@
def _init_persona(self, modules):
# Determine the view based on which modules are enabled
- if "messaging" in modules:
- if "grid" in modules:
- view = MainView(self, "main")
- else:
- view = MessagingMainView(self, "main")
- else:
- view = GridMainView(self, "main")
+ view = GridMainView(self, "main")
self.main_page = MainPage(self, "index.html", view)
self.add_page(self.main_page, add_to_link_set=True)
@@ -199,7 +185,7 @@
# service in the list takes precedence for any
# given op...
self.remote = Catalog()
- ops = [QmfOperations("qmf", self.session)]
+ ops = []
imports_ok = True
if self.aviary_locator or \
@@ -255,7 +241,6 @@
self.wallaby_refresh)
self.model.init()
- self.session.init()
self.database.init(schema_version_check)
self.server.init()
@@ -274,7 +259,6 @@
def start(self):
log.info("Starting %s", self)
- self.session.start()
self.server.start()
if self.wallaby is not None:
self.wallaby.start()
@@ -292,10 +276,6 @@
log.debug("Exception stopping update threads")
self.server.stop()
- try:
- self.session.stop()
- except:
- pass
def set_form_defaults(self,
request_memory,
@@ -336,20 +316,6 @@
def __init__(self, app, name):
super(GridMainView, self).__init__(app, name)
-class MessagingMainView(CuminMainView):
- def __init__(self, app, name):
- super(MessagingMainView, self).__init__(app, name)
-
- self.overview = MessagingOverviewFrame(app, "overview")
- self.add_tab(self.overview)
-
-class MainView(CuminMainView):
- def __init__(self, app, name):
- super(MainView, self).__init__(app, name)
-
- self.overview = OverviewFrame(app, "overview")
- self.add_tab(self.overview)
-
class MainPage(CuminPage, ModeSet):
def __init__(self, app, name, main_view):
super(MainPage, self).__init__(app, name)
@@ -357,7 +323,7 @@
self.main = main_view
self.add_mode(self.main)
self.set_default_frame(self.main)
- self.cumin_module = ["messaging", "grid"]
+ self.cumin_module = ["grid"]
self.page_html_class = "Cumin"
@@ -367,8 +333,6 @@
def get_title(self, session):
return "Administrator"
-# XXX Add qmf tab
-
class OverviewFrame(CuminFrame):
def __init__(self, app, name):
super(OverviewFrame, self).__init__(app, name)
@@ -376,22 +340,9 @@
self.view = self.get_overview_view(app)
self.add_mode(self.view)
- self.notice = ConfigurationNotice(app, "notice")
- self.add_mode(self.notice)
-
def get_overview_view(self, app):
return OverviewView(app, "view")
- def do_process(self, session):
- super(OverviewFrame, self).do_process(session)
-
- configured_count = len([x for x in self.app.session.qmf_brokers if x.host != "localhost"])
- connected_count = len([x for x in self.app.session.qmf_brokers if x.connected])
-
- # there were no user defined entries in cumin.conf and localhost isn't connected
- if (configured_count == 0) and (connected_count == 0):
- self.mode.set(session, self.notice)
-
def render_title(self, session):
return "Overview"
@@ -399,9 +350,6 @@
def __init__(self, app, name):
super(OverviewView, self).__init__(app, name)
- queues = TopQueueTable(app, "queues")
- self.add_child(queues)
-
systems = TopSystemTable(app, "systems")
self.add_child(systems)
systems.cumin_module = "inventory"
@@ -410,137 +358,9 @@
self.add_child(submissions)
submissions.cumin_module = "grid"
-class MessagingOverviewFrame(OverviewFrame):
- def __init__(self, app, name):
- super(MessagingOverviewFrame, self).__init__(app, name)
-
- def get_overview_view(self, app):
- return MessagingOverviewView(app, "view")
-
-class MessagingOverviewView(Widget):
- def __init__(self, app, name):
- super(MessagingOverviewView, self).__init__(app, name)
-
- queues = TopQueueTable(app, "queues")
- self.add_child(queues)
-
- systems = TopSystemTable(app, "systems")
- self.add_child(systems)
-
class ConfigurationNotice(Widget):
pass
-class TopQueueAdapter(SqlAdapter):
- def __init__(self, app, cls):
- super(TopQueueAdapter, self).__init__(app, cls.sql_table)
- self.cls = cls
-
- def init(self):
- super(TopQueueAdapter, self).init()
-
- name_col = self.table._columns_by_name["name"]
- avg_over_last_60_seconds_col = """((sum("Queue"."msgTotalEnqueues") -
- sum(s."msgTotalEnqueues")) / (count(1)-1)) / 300 as avg_60"""
- queue_id_col = self.table._columns_by_name["_id"]
- vhostRef_col = self.table._columns_by_name["_vhostRef_id"]
- vhost_table = self.app.model.org_apache_qpid_broker.Vhost.sql_table
- vhost_id_col = vhost_table._columns_by_name["_id"]
- vhost_brokerRef_col = vhost_table._columns_by_name["_brokerRef_id"]
-
- self.columns.append(name_col)
- self.columns.append(avg_over_last_60_seconds_col)
- self.columns.append(vhost_brokerRef_col)
- self.columns.append(queue_id_col)
-
- sub_query = "(%s) as s" % self.get_sub_query_text()
- SqlInnerJoin(self.query, sub_query,
- queue_id_col, "s._parent_id")
-
- SqlInnerJoin(self.query, vhost_table,
- vhostRef_col, vhost_id_col)
-
- def get_sub_query_text(self):
- samples_table = self.cls.sql_samples_table
- subquery = SqlQuery(samples_table)
- parent_col = samples_table._columns_by_name["_parent_id"]
- updated_col = samples_table._columns_by_name["_qmf_update_time"]
- enqueues_col = samples_table._columns_by_name["msgTotalEnqueues"]
-
- when = "now() - interval '600 seconds'"
- filter = SqlComparisonFilter(updated_col, when, ">=")
- subquery.add_filter(filter)
-
- columns = list()
- columns.append(parent_col)
- columns.append(enqueues_col)
-
- return subquery.emit(columns)
-
- def get_data(self, values, options):
- options.sort_column = "avg_60"
- options.sort_ascending = False
-
- options.group_column = "%s, %s, %s" % ("\"Queue\".name", "\"Queue\"._id", "\"Vhost\".\"_brokerRef_id\"")
- having = SqlComparisonFilter("count(1)", "1", ">")
- options.group_having.append(having)
-
- try:
- data = super(TopQueueAdapter, self).get_data(values, options)
- except:
- data = []
- return data
-
- def get_sql_options(self, options):
- return options
-
-class TopQueueTable(TopTable):
- def __init__(self, app, name):
- cls = app.model.org_apache_qpid_broker.Queue
- super(TopQueueTable, self).__init__(app, name, cls)
-
- frame = "main.messaging.broker.queue"
- col = self.NameColumn(app, "name", cls.name, cls._id, frame)
- self.add_column(col)
-
- col = self.add_attribute_column(cls.msgDepth)
- self.set_default_sort_column(col)
-
- def init(self):
- super(TopQueueTable, self).init()
-
- self.adapter.vhost_id_field = ObjectSqlField \
- (self.adapter, self.cls.vhostRef)
-
- filter = SqlComparisonFilter(self.cls.msgDepth.sql_column,
- "null",
- "is not")
- self.adapter.query.add_filter(filter)
-
- def render_rows(self, session):
- data = self.data.get(session)
-
- writer = Writer()
-
- for record in data:
- # if there is no broker id, don't render the row
- if record[self.adapter.vhost_id_field.index]:
- writer.write(self.row.render(session, record))
-
- return writer.to_string()
-
- class NameColumn(ObjectLinkColumn):
- def render_cell_href(self, session, record):
- branch = session.branch()
-
- frame = self.page.main.messaging.broker
- adapter = self.table.adapter
-
- frame.id.set(branch, record[adapter.vhost_id_field.index])
- frame.queue.id.set(branch, record[adapter.id_field.index])
- frame.queue.view.show(branch)
-
- return branch.marshal()
-
class TopSystemTable(TopTable):
def __init__(self, app, name):
cls = app.model.com_redhat_sesame.Sysimage
Modified: branches/tmckay/cumin/python/cumin/model.py
===================================================================
--- branches/tmckay/cumin/python/cumin/model.py 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/cumin/python/cumin/model.py 2013-02-12 15:11:33 UTC (rev 5684)
@@ -100,13 +100,6 @@
def show_main(self, session):
return self.app.main_page.main.show(session)
- def get_session_by_object(self, obj):
- agent = self.app.session.get_agent(obj._qmf_agent_id)
- broker = agent.getBroker()
- session = broker.getAmqpSession()
-
- return session
-
def get_negotiator_group_names(self, negotiator):
if not negotiator:
store = NegotiatorGroupNamesStore(self, None)
Deleted: branches/tmckay/cumin/python/cumin/session.py
===================================================================
--- branches/tmckay/cumin/python/cumin/session.py 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/cumin/python/cumin/session.py 2013-02-12 15:11:33 UTC (rev 5684)
@@ -1,201 +0,0 @@
-from model import *
-from util import *
-from mint.util import make_agent_id
-
-from qmf.console import Console, Session, ObjectId
-from rosemary.model import RosemaryObject
-from sage.util import get_sasl_mechanisms
-
-log = logging.getLogger("cumin.session")
-
-class CuminSession(object):
- def __init__(self, app, broker_uris):
- self.app = app
- self.broker_uris = broker_uris
-
- self.qmf_session = None
- self.qmf_brokers = list()
- self.qmf_agents = dict()
-
- # int seq => callable
- self.outstanding_method_calls = dict()
-
- self.lock = Lock()
-
- def add_broker(self, uri):
- mechs = get_sasl_mechanisms(uri, self.app.sasl_mech_list)
- uri_without_password = uri[uri.rfind("@") + 1:]
- log.info("Adding QMF broker at %s with mech_list %s", uri_without_password, mechs)
-
- assert self.qmf_session
-
- qmf_broker = self.qmf_session.addBroker(uri,mechanisms=mechs)
-
- self.qmf_brokers.append(qmf_broker)
-
- def check(self):
- log.info("Checking %s", self)
-
- def init(self):
- uris_without_password = [x[x.rfind("@")+1:] for x in self.broker_uris]
- log.info("Initializing %s", uris_without_password)
-
- def start(self):
- log.info("Starting %s", self)
-
- assert self.qmf_session is None
-
- self.qmf_session = Session(CuminConsole(self),
- manageConnections=True,
- rcvObjects=False,
- rcvEvents=False,
- rcvHeartbeats=False)
-
- for uri in self.broker_uris:
- self.add_broker(uri)
-
- def stop(self):
- log.info("Stopping %s", self)
-
- for qmf_broker in self.qmf_brokers:
- self.qmf_session.delBroker(qmf_broker)
-
- def get_agent(self, agent_id):
- self.lock.acquire()
- try:
- return self.qmf_agents.get(agent_id)
- finally:
- self.lock.release()
-
- def call_method(self, callback, obj, name, args):
- assert isinstance(obj, RosemaryObject)
-
- for i in range(10):
- agent = self.get_agent(obj._qmf_agent_id)
-
- if agent:
- break
-
- sleep(1)
-
- if not agent:
- raise Exception("Agent '%s' is unknown" % obj._qmf_agent_id)
-
- if obj._qmf_object_id.isdigit():
- # Translate v1 object ids
- object_id = self.gen_object_id(obj)
- else:
- object_id = obj._qmf_object_id
-
- oid = ObjectId({"_agent_name": obj._qmf_agent_id,
- "_object_name": object_id})
-
- qmf_objs = agent.getObjects(_objectId=oid)
-
- try:
- qmf_obj = qmf_objs[0]
- except IndexError:
- raise Exception("Object '%s' is unknown" % object_id)
-
- self.lock.acquire()
- try:
- seq = qmf_obj._invoke(name, args, {"_async": True})
-
- assert seq is not None
-
- self.outstanding_method_calls[seq] = callback
-
- return seq
- finally:
- self.lock.release()
-
- def gen_object_id(self, obj):
- # special case objects are queue, link , and bridge
- if obj._class in (self.app.model.org_apache_qpid_broker.Queue,
- self.app.model.org_apache_qpid_broker.Link,
- self.app.model.org_apache_qpid_broker.Bridge):
- if obj._class is self.app.model.org_apache_qpid_broker.Queue:
- key = obj.name
- else:
- if obj._class is self.app.model.org_apache_qpid_broker.Bridge:
- # bridges need the link object_id as well as their own object_id
- cls = self.app.model.org_apache_qpid_broker.Link
- cursor = self.app.database.get_read_cursor()
- link = cls.get_object(cursor, _id=obj._linkRef_id)
- if not link:
- raise Exception("Object '%s' is not a valid bridge" % obj._id)
- else:
- link = obj
- key = "qpid.%s:%s:%d" % (link.transport, link.host, link.port)
-
- if obj._class is self.app.model.org_apache_qpid_broker.Bridge:
- #org.apache.qpid.broker:bridge:org.apache.qpid.broker:link:qpid.tcp:localhost:5671,qpid.tcp:localhost:5671!amq.direct!amq.direct!r-key
- key = "%s:%s:%s,%s!%s!%s!%s" % (link._class._package._name, link._class._name.lower(),
- key, key, obj.src, obj.dest, obj.key)
- else:
- # other objects use the schema index fields
- key_args = [str(getattr(obj, x.name))
- for x in obj._class._attributes
- if x.index and not x.references]
- key = ",".join(key_args)
-
- id_args = (obj._class._package._name, obj._class._name.lower(), key)
- object_id = ":".join(id_args)
- return object_id
-
- def __repr__(self):
- uris_without_password = [x[x.rfind("@")+1:] for x in self.broker_uris]
- return "%s(%s)" % (self.__class__.__name__, uris_without_password)
-
-class CuminConsole(Console):
- def __init__(self, session):
- self.session = session
-
- def brokerConnected(self, broker):
- log.info("Broker connected %s:%s", broker.host, broker.port)
-
- def brokerConnectionFailed(self, broker):
- log.info("Broker connection failed %s:%s", broker.host, broker.port)
-
- def brokerDisconnected(self, broker):
- log.info("Broker disconnected %s:%s", broker.host, broker.port)
-
- def newPackage(self, name):
- log.debug("New package %s", name)
-
- def newClass(self, kind, classKey):
- log.debug("New class %s", classKey)
-
- def newAgent(self, qmf_agent):
- log.debug("New agent %s", qmf_agent)
-
- agent_id = make_agent_id(qmf_agent)
-
- self.session.lock.acquire()
- try:
- self.session.qmf_agents[agent_id] = qmf_agent
- finally:
- self.session.lock.release()
-
- def delAgent(self, qmf_agent):
- log.debug("Deleting agent %s", qmf_agent)
-
- agent_id = make_agent_id(qmf_agent)
-
- self.session.lock.acquire()
- try:
- del self.session.qmf_agents[agent_id]
- finally:
- self.session.lock.release()
-
- def methodResponse(self, broker, seq, response):
- log.debug("Method response for request %i received from %s",
- seq, broker)
- log.debug("Response: %s", response)
-
- self.session.lock.acquire()
- try:
- callback = self.session.outstanding_method_calls.pop(seq)
- callback(response.text, response.outArgs)
- finally:
- self.session.lock.release()
Modified: branches/tmckay/mint/python/mint/main.py
===================================================================
--- branches/tmckay/mint/python/mint/main.py 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/mint/python/mint/main.py 2013-02-12 15:11:33 UTC (rev 5684)
@@ -1,7 +1,6 @@
from database import MintDatabase
from expire import ExpireThread
from model import MintModel
-from session import MintSession
from aviary.session import MintAviarySession
from update import UpdateThread
from vacuum import VacuumThread
@@ -11,18 +10,16 @@
log = logging.getLogger("mint.main")
class Mint(object):
- def __init__(self, home, model_paths, broker_uris, database_dsn):
+ def __init__(self, home, model_paths, database_dsn):
self.home = home
model_dir = [os.path.join(self.home, x) for x in model_paths]
self.model = MintModel(self, model_dir)
self.model.sql_logging_enabled = True
- self.broker_uris = broker_uris
self.database = MintDatabase(self, database_dsn)
self.admin = CuminAdmin(self)
self.update_thread = UpdateThread(self)
- self.session = None
self.aviary_session = None
self.expire_enabled = True
@@ -33,10 +30,6 @@
self.print_event_level = 0
- # Space separated list of sasl authentication
- # mechanisms, according to the sasl documentation
- self.sasl_mech_list = None
-
# List of classes to bind in QMF. Referenced by session and update thread.
self._qmf_classes = set()
@@ -187,7 +180,6 @@
# (someday it will be ALL classes)
self._init_aviary()
- self.session = MintSession(self, self.broker_uris, self._qmf_classes)
self.database.init()
self.update_thread.init()
@@ -199,7 +191,6 @@
self.update_thread.start()
- self.session.start()
if self.aviary_session:
self.aviary_session.start()
@@ -220,7 +211,6 @@
if self.vacuum_enabled and not skip_vacuum:
self.vacuum_thread.stop()
- self.session.stop()
if self.aviary_session:
self.aviary_session.stop()
log.info("Session stopped")
Deleted: branches/tmckay/mint/python/mint/session.py
===================================================================
--- branches/tmckay/mint/python/mint/session.py 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/mint/python/mint/session.py 2013-02-12 15:11:33 UTC (rev 5684)
@@ -1,125 +0,0 @@
-from update import *
-from util import *
-
-from qmf.console import Console, Session
-from sage.util import get_sasl_mechanisms
-
-log = logging.getLogger("mint.session")
-
-class MintSession(object):
- def __init__(self, app, broker_uris, qmf_classes):
- self.app = app
- self.broker_uris = broker_uris
-
- self.qmf_classes = qmf_classes
- self.qmf_session = None
- self.qmf_brokers = list()
-
- def add_broker(self, uri):
- mechs = get_sasl_mechanisms(uri, self.app.sasl_mech_list)
- uri_without_password = uri[uri.rfind("@") + 1:]
- log.info("Adding QMF broker at %s with mech_list %s", uri_without_password, mechs)
-
- assert self.qmf_session
-
- qmf_broker = self.qmf_session.addBroker(uri, mechanisms=mechs)
- self.qmf_brokers.append(qmf_broker)
-
- def start(self):
- log.info("Starting %s", self)
-
- assert self.qmf_session is None
-
- self.qmf_session = Session(MintConsole(self.app.model),
- manageConnections=True,
- rcvObjects=True,
- rcvEvents=False,
- rcvHeartbeats=False,
- userBindings=True)
-
- self.qmf_session.bindAgent("*")
- log.info("Binding all agents")
-
- for cls in self.qmf_classes:
- pname = cls._package._name
- cname = cls._name
- self.qmf_session.bindClass(pname.lower(), cname.lower())
- log.info("Binding QMF class %s.%s" % (pname, cname))
-
- for uri in self.broker_uris:
- self.add_broker(uri)
-
- def stop(self):
- log.info("Stopping %s", self)
-
- for qmf_broker in self.qmf_brokers:
- self.qmf_session.delBroker(qmf_broker)
-
- def __repr__(self):
- uris_without_password = [x[x.rfind("@")+1:] for x in self.broker_uris]
- return "%s(%s)" % (self.__class__.__name__, uris_without_password)
-
-class MintConsole(Console):
- def __init__(self, model):
- self.model = model
-
- def brokerConnected(self, qmf_broker):
- message = "Broker %s:%i is connected"
- self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
-
- def brokerInfo(self, qmf_broker):
- message = "Broker info from %s:%i"
- self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
-
- def brokerDisconnected(self, qmf_broker):
- message = "Broker %s:%i is disconnected"
- self.model.print_event(1, message, qmf_broker.host, qmf_broker.port)
-
- def newAgent(self, qmf_agent):
- self.model.print_event(3, "Creating %s", qmf_agent)
-
- up = AgentUpdate(self.model, qmf_agent)
- self.model.app.update_thread.enqueue(up)
-
- def delAgent(self, qmf_agent):
- self.model.print_event(3, "Deleting %s", qmf_agent)
-
- up = AgentDelete(self.model, qmf_agent)
- self.model.app.update_thread.enqueue(up)
-
- def heartbeat(self, qmf_agent, timestamp):
- message = "Heartbeat from %s at %s"
- self.model.print_event(5, message, qmf_agent, timestamp)
-
- up = AgentUpdate(self.model, qmf_agent)
- self.model.app.update_thread.enqueue(up)
-
- def newPackage(self, name):
- self.model.print_event(2, "New package %s", name)
-
- def newClass(self, kind, classKey):
- self.model.print_event(2, "New class %s", classKey)
-
- def objectProps(self, broker, qmf_object):
- up = ObjectUpdate(self.model, qmf_object)
- self.model.app.update_thread.enqueue(up)
-
- def objectStats(self, broker, qmf_object):
- up = ObjectUpdate(self.model, qmf_object)
- self.model.app.update_thread.enqueue(up)
-
- def event(self, broker, event):
- self.model.print_event(4, "New event %s from %s", broker, event)
-
- def methodResponse(self, broker, seq, response):
- message = "Method response for request %i received from %s"
- self.model.print_event(3, message, seq, broker)
-
- log.debug("Response: %s", response)
-
- self.model.lock.acquire()
- try:
- callback = self.model.outstanding_method_calls.pop(seq)
- callback(response.text, response.outArgs)
- finally:
- self.model.lock.release()
Modified: branches/tmckay/mint/python/mint/util.py
===================================================================
--- branches/tmckay/mint/python/mint/util.py 2013-02-11 21:53:18 UTC (rev 5683)
+++ branches/tmckay/mint/python/mint/util.py 2013-02-12 15:11:33 UTC (rev 5684)
@@ -9,7 +9,6 @@
from datetime import datetime, timedelta
from pprint import pprint
from getpass import getpass
-from qmf.console import ObjectId
from random import sample
from tempfile import mkstemp
from threading import Thread, Lock, RLock, Condition
@@ -114,131 +113,6 @@
return password
-def make_agent_id(agent):
- broker = agent.getBroker()
-
- if not agent.isV2 or agent is broker.getBrokerAgent():
- return "!!%s:%i!!%s" % (broker.host, broker.port, agent.getAgentBank())
-
- return agent.agentBank
-
-class QmfAgentId(object):
- def __init__(self, brokerId, brokerBank, agentBank):
- assert brokerId
-
- self.brokerId = brokerId
- self.brokerBank = brokerBank
- self.agentBank = agentBank
-
- def fromObject(cls, object):
- broker = object.getBroker()
-
- brokerId = broker.getBrokerId()
- brokerBank = broker.getBrokerBank()
- agentBank = object.getObjectId().getAgentBank()
-
- return cls(brokerId, brokerBank, agentBank)
-
- def fromAgent(cls, agent):
- return agent.getAgentBank()
-
- broker = agent.getBroker()
-
- brokerId = broker.getBrokerId()
- brokerBank = broker.getBrokerBank()
- agentBank = agent.getAgentBank()
-
- print "XXX", brokerId, brokerBank, agentBank
-
- return cls(brokerId, brokerBank, agentBank)
-
- def fromString(cls, string):
- brokerId, brokerBank, agentBank = string.split(".")
-
- brokerBank = int(brokerBank)
- agentBank = int(agentBank)
-
- return cls(brokerId, brokerBank, agentBank)
-
- fromObject = classmethod(fromObject)
- fromAgent = classmethod(fromAgent)
- fromString = classmethod(fromString)
-
- def __str__(self):
- return self.agentBank
-
-class QmfObjectId(object):
- def __init__(self, id):
- self.id = id
-
- def fromObject(cls, object):
- oid = object.getObjectId()
-
- print "XXX", oid
-
- for k, v in oid.__dict__.items():
- print " ", k, v
-
- return cls(oid.first, oid.second)
-
- def fromString(cls, string):
- first, second = string.split(".")
-
- first = int(first)
- second = int(second)
-
- return cls(first, second)
-
- fromObject = classmethod(fromObject)
- fromString = classmethod(fromString)
-
- def toObjectId(self):
- return ObjectId(None, self.first, self.second)
-
- def __str__(self):
- return "%i.%i" % (self.first, self.second)
-
-class UpdateQueue(ConcurrentQueue):
- def __init__(self, maxsize=0, slotCount=1):
- self.slotCount = slotCount
- ConcurrentQueue.__init__(self, maxsize)
-
- def _init(self, maxsize):
- self.maxsize = maxsize
- self.slots = []
-
- for i in range(self.slotCount):
- self.slots.append(deque())
-
- def _qsize(self):
- size = 0
-
- for i in range(self.slotCount):
- size += len(self.slots[i])
-
- return size
-
- def _empty(self):
- return self._qsize() == 0
-
- def _full(self):
- return self.maxsize > 0 and self._qsize() == self.maxsize
-
- def _put(self, update):
- slot = update.priority
-
- if slot in range(self.slotCount):
- self.slots[slot].append(update)
- else:
- raise ValueError("Invalid priority slot")
-
- def _get(self):
- for slot in range(self.slotCount):
- if len(self.slots[slot]) > 0:
- return self.slots[slot].popleft()
-
- return None
-
password_chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
def crypt_password(password, salt=None):
11 years, 2 months
r5683 - in branches/tmckay: cumin/bin cumin/python/cumin cumin/python/cumin/grid mint/python/mint sage/python/sage/wallaby
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-02-11 21:53:18 +0000 (Mon, 11 Feb 2013)
New Revision: 5683
Modified:
branches/tmckay/cumin/bin/cumin-web
branches/tmckay/cumin/python/cumin/config.py
branches/tmckay/cumin/python/cumin/grid/tags.py
branches/tmckay/cumin/python/cumin/main.py
branches/tmckay/mint/python/mint/main.py
branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py
Log:
Intermediate commit
Modified: branches/tmckay/cumin/bin/cumin-web
===================================================================
--- branches/tmckay/cumin/bin/cumin-web 2013-02-11 21:10:39 UTC (rev 5682)
+++ branches/tmckay/cumin/bin/cumin-web 2013-02-11 21:53:18 UTC (rev 5683)
@@ -34,12 +34,11 @@
cumin.aviary_prefer_condor = values.aviary_prefer_condor
def set_wallaby_configs(cumin, values, brokers):
- if values.wallaby_broker == "":
- cumin.wallaby_broker = brokers[0]
- else:
- if values.wallaby_broker == "None":
- values.wallaby_broker = None
- cumin.wallaby_broker = values.wallaby_broker
+ if values.wallaby_broker == "None":
+ values.wallaby_broker = None
+
+ cumin.wallaby_broker = values.wallaby_broker
+
# Let 0 indicate "no timeout", since the timeout
# value is an int in the config and None can't be
# specified
Modified: branches/tmckay/cumin/python/cumin/config.py
===================================================================
--- branches/tmckay/cumin/python/cumin/config.py 2013-02-11 21:10:39 UTC (rev 5682)
+++ branches/tmckay/cumin/python/cumin/config.py 2013-02-11 21:53:18 UTC (rev 5683)
@@ -231,7 +231,7 @@
# param.default = False
param = ConfigParameter(self, "wallaby-broker", str)
- param.default = ""
+ param.default = "http://localhost"
param = ConfigParameter(self, "wallaby-refresh", int)
param.default = 60
Modified: branches/tmckay/cumin/python/cumin/grid/tags.py
===================================================================
--- branches/tmckay/cumin/python/cumin/grid/tags.py 2013-02-11 21:10:39 UTC (rev 5682)
+++ branches/tmckay/cumin/python/cumin/grid/tags.py 2013-02-11 21:53:18 UTC (rev 5683)
@@ -461,6 +461,8 @@
self.page.redirect.set(session, url)
def validate(self, session):
+ return True
+
results = self.app.wallaby.validate_configuration()
#handle the set of explanations that could come back if things didn't go well
@@ -1141,7 +1143,7 @@
return "Activate wallaby configuration"
def callback(self, results):
- if len(results[0]) != 0 or len(results[1]) != 0:
+ if results is not None:
self.invoc.status = self.invoc.FAILED
self.invoc.end()
Modified: branches/tmckay/cumin/python/cumin/main.py
===================================================================
--- branches/tmckay/cumin/python/cumin/main.py 2013-02-11 21:10:39 UTC (rev 5682)
+++ branches/tmckay/cumin/python/cumin/main.py 2013-02-11 21:53:18 UTC (rev 5683)
@@ -251,13 +251,8 @@
self.remote.add_mechanisms(ops)
# Create RPC interface for Wallaby
- # The Wallaby client API needs a broker connection. The broker host
- # for Wallaby is not necessarily the same as the broker used by
- # cumin-web for grid data, and flags set on the Session for cumin
- # may not be appropriate for the Wallaby API. So, it gets its own.
self.wallaby = WallabyOperations(self.wallaby_broker,
- self.wallaby_refresh,
- self.sasl_mech_list)
+ self.wallaby_refresh)
self.model.init()
self.session.init()
Modified: branches/tmckay/mint/python/mint/main.py
===================================================================
--- branches/tmckay/mint/python/mint/main.py 2013-02-11 21:10:39 UTC (rev 5682)
+++ branches/tmckay/mint/python/mint/main.py 2013-02-11 21:53:18 UTC (rev 5683)
@@ -94,6 +94,9 @@
self.model.com_redhat_grid.Scheduler,
self.model.com_redhat_grid.Slot]
+# supported_classes = [self.model.com_redhat_grid.Submitter,
+# self.model.com_redhat_grid.Scheduler]
+
for c in supported_classes:
if c in self._qmf_classes:
self._qmf_classes.remove(c)
Modified: branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py
===================================================================
--- branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py 2013-02-11 21:10:39 UTC (rev 5682)
+++ branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py 2013-02-11 21:53:18 UTC (rev 5683)
@@ -1,18 +1,14 @@
import logging
import time
-from qmf.console import Session
-from sage.util import CallThread, get_sasl_mechanisms
+from sage.util import CallThread, parse_URL
from threading import Lock, Condition
#import cProfile
imports_ok = True
try:
- import wallaby
- import wallaby.tagging
- import wallaby.collections
- setup = wallaby.tagging.setup
+ import wallaroo
except:
imports_ok = False
@@ -33,19 +29,20 @@
'''
Wrapper around the Wallaby client library.
'''
- def __init__(self, broker_uri, refresh_interval=None, sasl_mech_list=None):
+ def __init__(self, wallaroo_uri, refresh_interval=None):
'''
Constructor.
- broker_uri -- the URI used to connect to a QMF message broker
- where a Wallaby agent is connected. The simplest URI is just a
- hostname but a full URI can specify scheme://user/password@host:port
- or a subset of those components as long as the host is included.
+ wallaroo_uri -- the URI used to connect to a wallaroo server.
+ The simplest URI is just a hostname but a full URI can specify
+ scheme://user/password@host:port or a subset of those components
+ as long as the host is included.
+
Examples:
localhost
localhost:5672
- amqp://fred/flintsone@quarry.bedrock.com:1234
+ http://fred/flintsone@quarry.bedrock.com:1234
refresh_interval -- default refresh interval in seconds for all items
maintained by WallabyOperations' internal caching thread. A value of
@@ -53,23 +50,12 @@
item after a successful call unless the refresh() method is used.
The refresh interval may be set for items individually with the
set_interval() method.
-
- sasl_mech_list -- restricts the list of sasl mechanisms
- that will be allowed when connecting to a QMF message broker.
- If the broker URL contains no credentials, default is ANONYMOUS.
- If the broker URL does contain credentials, default is
- 'PLAIN DIGEST-MD5'
'''
- self.broker_uri = broker_uri
- self.sasl_mech_list = get_sasl_mechanisms(broker_uri,
- sasl_mech_list)
+ self.wallaroo_uri = wallaroo_uri
# A wallaby Store object
self._store = None
- # A QMF broker
- self._broker = None
-
# The cache maintenance thread
self._maintain_cache = None
@@ -80,12 +66,13 @@
# an attribute on the Wallaby Store object, with the exception of
# WBTypes.TAGS. The TAGS data is a subset of the GROUPS produced
# in this module.
- self._cache = {WBTypes.NODES: self.CacheData(refresh_interval),
- WBTypes.GROUPS: self.CacheData(refresh_interval),
- WBTypes.FEATURES: self.CacheData(refresh_interval),
- WBTypes.TAGS: self.CacheData(refresh_interval,
- synthetic=self._generate_tag_data)}
+ self._cache = {WBTypes.NODES: self.NodeData(refresh_interval),
+ WBTypes.GROUPS: self.GroupData(refresh_interval),
+ WBTypes.FEATURES: self.FeatureData(refresh_interval),
+ WBTypes.TAGS: self.TagData(refresh_interval,
+ synthetic=True)}
+
# Cache a list of nodes that are members of a tag
self._nodes_by_tag = dict()
@@ -103,9 +90,9 @@
'''
Start the caching thread.
- This thread will attempt to connect to the broker and retrieve
- a Store object from the Wallaby agent. If successful, it will
- periodically retrieve and cache data from Wallaby.
+ This thread will attempt to connect to wallaroo and retrieve
+ a Store object. If successful, it will periodically retrieve
+ and cache data from wallaroo.
Only one caching thread may run at a time. The thread may
be restarted if it has previously been stopped.
@@ -114,35 +101,35 @@
should only be called from a single thread.
retry_secs -- how often the caching thread will retry failed
- operations. This includes attempts to connect to the broker
- and retrieve a Store object as well as calls to Wallaby that
+ operations. This includes attempts to connect to wallaroo
+ and retrieve a Store object as well as calls to wallaroo that
return no data.
'''
- # The connection to the broker can actually take a long
- # time to complete. We don't want to hang a calling function,
- # so we handle the connection and retrieval of the
- # initial Store object from Wallaby in a thread.
- # (There may need to be more work here if the broker or wallaby
- # going away and coming back causes a problem, but with
- # manageConnections=True and well-known agent/object ids for
- # Wallaby it appears to recover on its own...)
- # Similarly, getting node lists etc may take a long time
- # especially over a slow network. So we use the same thread
- # to retrieve things like node lists at defined intervals.
-
# 'self' here is really a term of art since this is a local
# function, but it refers to the WallabyOperations object
# so the code reads nicely
def maintain_cache(self):
+
+ # Turn the URL into a dictionary of args that the wallaroo
+ # client.connect() method wants to see
+ url = parse_URL(self.wallaroo_uri)
+ arg_map = (("host",),
+ ("port",),
+ ("scheme",),
+ ("user", "username"),
+ ("password", "pw"))
+ connect_args = dict()
+ for arg in arg_map:
+ v = getattr(url, arg[0])
+ if v is not None:
+ connect_args[arg[-1]] = v
+
# Get initinal connection and Store obect
- self.session = Session(manageConnections=True)
- self.broker = self.session.addBroker(self.broker_uri, mechanisms=self.sasl_mech_list)
while not self._stop:
- self._store = self._get_store()
+ self._store = wallaroo.client.connect(**connect_args)
if self._store is not None:
- setup(self._store)
self._partition_group = self._store.getPartitionGroup().name
log.debug("WallabyOperations: found wallaby store object")
break
@@ -163,21 +150,15 @@
while not self._stop:
start_processing = time.time()
for attr, val in self._cache.iteritems():
- if self._stop:
- break
+ if self._stop:
+ break
- # val.remaining is the number of seconds left before
- # the next update of this data item. None is "forever".
- # Synthetic items are not retreived from the store.
- if not val.synthetic and \
- val.remaining is not None and val.remaining <= 0:
- d = get_values(attr, getattr, self._store, attr, [])
- # If the data is empty, _set_cache will leave the
- # remaining field set to 0 for the attribute so we
- # will try to get it again on our next retry.
- # Otherwise, remaining will be reset to the full
- # interval for this attribute.
- self._set_cache(attr, d)
+ # val.remaining is the number of seconds left before
+ # the next update of this data item. None is "forever".
+ # Synthetic items are not retreived from the store.
+ if not val.synthetic and \
+ val.remaining is not None and val.remaining <= 0:
+ val.get_values(attr, self)
# Now handle the synthetics. val.synthetic generates
# and stores it's own results.
@@ -187,7 +168,7 @@
if val.synthetic and \
val.remaining is not None and val.remaining <= 0:
- get_values(attr, val.synthetic, *val.args)
+ val.get_values(attr, self)
log.debug("WallabyOperations: total refresh processing time %s" \
% (time.time() - start_processing))
@@ -225,12 +206,6 @@
self._set_cache(attr, [])
self._store = None
- # Have to clean up the broker
- try:
- self.session.delBroker(self.broker)
- except:
- pass
-
#end maintain_cache
def get_values(attr, call, *args):
@@ -252,8 +227,9 @@
try:
maintain_cache(self)
log.debug("WallabyOperations: cache maintenance thread exited")
- except:
- pass
+ except Exception:
+ import traceback
+ traceback.print_exc()
if self._maintain_cache is not None and \
self._maintain_cache.isAlive():
@@ -262,7 +238,7 @@
self._stop = False
- if self.broker_uri is not None:
+ if self.wallaroo_uri is not None:
# self._maintain_cache = CallThread(cProfile.runctx('maintain_cache(self)', globals(), locals(), filename='sage.stats'), None)
self._maintain_cache = CallThread(wrap_maintain_cache, None)
self._maintain_cache.daemon = True
@@ -443,7 +419,7 @@
self._lock.acquire()
try:
if n in self._cache[WBTypes.NODES].data:
- names = self._cache[WBTypes.NODES].data[n].getTags()
+ names = self._cache[WBTypes.NODES].data[n].getLabels()
finally:
self._lock.release()
return names
@@ -461,7 +437,7 @@
self._lock.acquire()
try:
for name in names:
- self._store.addTag(name)
+ self._store.addLabel(name)
except Exception, e:
log.debug("WallabyOperations: create_tag, exception suppressed, %s" % str(e))
return False
@@ -518,7 +494,7 @@
# cast to list in case tags is a tuple, list required
start = time.time()
self._lock.acquire()
- n.modifyTags("REPLACE", list(tags), create_missing_tags=True)
+ n.modifyLabels("REPLACE", list(tags), create_missing_tags=True)
n.update()
status = True
@@ -555,7 +531,7 @@
else:
try:
start = time.time()
- g.modifyFeatures("REPLACE", features)
+ g.modifyFeatures("REPLACE", features, {})
g.update()
status = True
delta = time.time() - start
@@ -601,7 +577,7 @@
returns result of store.validateConfiguration.
'''
if self._store is not None:
- return self._store.validateConfiguration()
+ return self._store.checkBranchValidity()
else:
log.debug("WallabyOperations: validate_configuration," \
" store object not yet created")
@@ -628,23 +604,6 @@
if t is not None and t < min:
t = min
return t
-
- def _get_store(self):
- # Ideally there should only be a single Store object from a single agent.
- # We constrain the search a bit, hopefully more efficient
- store = None
- try:
- agents = [agent for agent in self.session.getAgents() \
- if "com.redhat.grid.config:Store" in agent.label]
- for agent in agents:
- s = self.session.getObjects(_agent = agent, _class = "Store")
- if len(s) > 0:
- # And finally wrap the QMF object in a wallaby.Store wrapper
- store = wallaby.Store(s[0], self.session)
- break
- except:
- pass
- return store
def _generate_tag_data(self):
# figure out the tag list and nodes per tag
@@ -655,8 +614,8 @@
if not g.name.startswith("+++") and \
self._store.isTag(g):
tags.append(g)
- nodes = g.membership()
- nodes_by_tag[g.name] = nodes
+ #nodes = g.membership()
+ nodes_by_tag[g.name] = []
self._lock.acquire()
try:
@@ -670,7 +629,7 @@
def _set_cache(self, attr, data):
self._lock.acquire()
try:
- self._cache[attr].data = self._to_dict(data)
+ self._cache[attr].data = dict(data)
self._cache[attr].reset_remaining(len(data) == 0)
finally:
self._lock.release()
@@ -710,6 +669,44 @@
else:
self.remaining = self.interval
+ class NodeData(CacheData):
+ def get_values(self, attr, ops):
+ ops._set_cache(attr,
+ [(name, ops._store.getNode(name)) for name in ops._store.nodes])
+
+ class GroupData(CacheData):
+ def get_values(self, attr, ops):
+ ops._set_cache(attr,
+ [(name, ops._store.getGroupByName(name)) for name in ops._store.groups])
+
+ class FeatureData(CacheData):
+ def get_values(self, attr, ops):
+ ops._set_cache(attr,
+ [(name, ops._store.getFeature(name)) for name in ops._store.features])
+
+ class TagData(CacheData):
+ def get_values(self, attr, ops):
+ # figure out the tag list and nodes per tag
+ groups = ops.get_data(WBTypes.GROUPS)
+ tags = []
+ nodes_by_tag = dict()
+ for g in groups:
+ if not g.name.startswith("+++") and \
+ ops._store.isLabel(g):
+ tags.append(g)
+ nodes = [] #g.membership()
+ nodes_by_tag[g.name] = nodes
+
+ ops._lock.acquire()
+ try:
+ ops._cache[WBTypes.TAGS].data = ops._to_dict(tags)
+ ops._cache[WBTypes.TAGS].reset_remaining(len(tags) == 0)
+ ops._nodes_by_tag = nodes_by_tag
+ finally:
+ ops._lock.release()
+ log.debug("WallabyOperations: %s list updated (%s items)" \
+ % (WBTypes.TAGS, len(tags)))
+
else:
class WallabyOperations(object):
'''
11 years, 2 months
r5682 - in branches/elephant/cumin: model python/cumin/gridhadoop python/cumin/user/hadoop
by croberts@fedoraproject.org
Author: croberts
Date: 2013-02-11 21:10:39 +0000 (Mon, 11 Feb 2013)
New Revision: 5682
Modified:
branches/elephant/cumin/model/cumin.xml
branches/elephant/cumin/model/rosemary.xml
branches/elephant/cumin/python/cumin/gridhadoop/datanode.py
branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py
branches/elephant/cumin/python/cumin/gridhadoop/hadoopobjectframe.py
branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py
branches/elephant/cumin/python/cumin/gridhadoop/main.py
branches/elephant/cumin/python/cumin/gridhadoop/namenode.py
branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py
branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py
branches/elephant/cumin/python/cumin/user/hadoop/main.py
Log:
Starting to use the actual API data where available. Also, now passing the "module" into the frames on creation so that they will have that information available when creating links to other frames (this solves the admin vs user module issue).
Modified: branches/elephant/cumin/model/cumin.xml
===================================================================
--- branches/elephant/cumin/model/cumin.xml 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/model/cumin.xml 2013-02-11 21:10:39 UTC (rev 5682)
@@ -53,7 +53,7 @@
<property name="State" type="sstr"/>
<property name="Uptime" type="sstr"/>
<property name="Owner" type="sstr"/>
- <property name="Location" type="sstr"/>
+ <property name="Http" type="sstr"/>
</class>
<class name="DataNode" storage="none">
<property name="Id" type="sstr"/>
@@ -80,7 +80,7 @@
<property name="State" type="sstr"/>
<property name="Uptime" type="sstr"/>
<property name="Owner" type="sstr"/>
- <property name="Location" type="sstr"/>
+ <property name="Http" type="sstr"/>
</class>
<class name="Scheduler" storage="none">
<property name="Id" type="sstr"/>
Modified: branches/elephant/cumin/model/rosemary.xml
===================================================================
--- branches/elephant/cumin/model/rosemary.xml 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/model/rosemary.xml 2013-02-11 21:10:39 UTC (rev 5682)
@@ -55,8 +55,8 @@
<property name="Owner">
<title>Owner</title>
</property>
- <property name="Location">
- <title>Location</title>
+ <property name="Http">
+ <title>Interface location</title>
</property>
</class>
@@ -121,8 +121,8 @@
<property name="Owner">
<title>Owner</title>
</property>
- <property name="Location">
- <title>Location</title>
+ <property name="Http">
+ <title>Interface location</title>
</property>
</class>
<class name="Scheduler">
Modified: branches/elephant/cumin/python/cumin/gridhadoop/datanode.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/datanode.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/gridhadoop/datanode.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -23,15 +23,17 @@
return results
class DataNodeSelector(ObjectSelector):
- def __init__(self, app, name, check_owner=False):
+ def __init__(self, app, name, module, check_owner=False):
cls = app.model.com_redhat_cumin_grid_hadoop.DataNode
super(DataNodeSelector, self).__init__(app, name, cls)
+
+ self.module = module
self.add_search_filter(self.table.id_col)
self.table.adapter = DataNodeAdapter(app, cls, check_owner)
- task = DataNodeCreate(app, name + ".create")
+ task = DataNodeCreate(app, name + ".create", self.module)
link = TaskLink(app, "dataNode_create", task)
self.links.add_child(link)
@@ -64,10 +66,7 @@
class DataNodeColumn(ObjectLinkColumn):
def render_cell_href(self, session, record):
id = unescape_entity(record[self.id_field.index])
- #need to make this work for user/grid-hadoop path too somehow
- # this is way less than ideal
- frame = self.frame.children_by_name[self.frame_path]
-
+ frame = self.frame.children_by_name[self.frame_path]
return frame.get_href(session, id, frame.parent.host.get(session))
class DataNodeDelete(ObjectSelectorTask):
@@ -107,10 +106,12 @@
return "Remove data nodes"
class DataNodeCreate(Task):
- def __init__(self, app, name):
+ def __init__(self, app, name, module):
super(DataNodeCreate, self).__init__(app, name)
cls = app.model.com_redhat_cumin_grid_hadoop.DataNode
+ self.module = module
+
self.form = DataNodeCreateForm(app, self.name + ".createform", self, cls)
self.invoc = None
@@ -144,13 +145,15 @@
self.form.id.set(form_session, obj._id)
## TODO find a better way to do this
- self.form.nn_id.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].children_by_name["gridhadoop.ghschedframe.namenodeframe"].id.get(session))
+ self.form.hadoophost.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"].host.get(session))
+ self.form.nn_id.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"]\
+ .children_by_name[self.module.frame.name + ".ghschedframe.namenodeframe"].id.get(session))
self.form.return_url.set(form_session, session.marshal())
self.form.show(form_session)
self.do_enter(session, obj, form_session)
return form_session
-
+
def get_href(self, session, obj):
return self.enter(session, obj).marshal()
@@ -166,8 +169,8 @@
self.count = self.CountField(app, name + ".count")
self.add_field(self.count)
- self.hadoophost = HadoopHostField(app, name + ".hadoophost")
- self.add_field(self.hadoophost)
+ self.hadoophost = Parameter(app, "hadoophost")
+ self.add_parameter(self.hadoophost)
self.nn_id = Parameter(app, "nn_id")
self.add_parameter(self.nn_id)
@@ -204,8 +207,8 @@
return "Number of data nodes to start"
class DataNodeFrame(HadoopObjectFrame):
- def __init__(self, app, name):
- super(DataNodeFrame, self).__init__(app, name)
+ def __init__(self, app, name, module):
+ super(DataNodeFrame, self).__init__(app, name, module)
self.view = DataNodeObjectView(app, "view", self.object)
self.replace_child(self.view)
@@ -214,21 +217,23 @@
self.view.add_tab(self.overview)
def get_title(self, session):
- retval = ""
- try:
- retval = "Data node:'%s' (%s)" % (xml_escape(self.object.get(session).name), xml_escape(self.object.get(session).id))
- except Exception, e:
- pass
+ retval = "Data node:'%s' (%s)" % (xml_escape(self.object.get(session).Name), xml_escape(self.object.get(session).Id))
return retval
def get_object(self, session, id):
class Object(object):
pass
fake = Object()
- fake.name = id
- fake.id = id
+ fake.Name = id
+ fake.Id = id
return fake
+# def get_object(self, session, id):
+# host = self.frame.host.get(session)
+# response = self.app.remote.get_data_node(self.frame.host.get(session), [id])
+# obj = HadoopObject(response, self.app.model.com_redhat_cumin_grid_hadoop.DataNode)
+# return obj
+
class DataNodeObjectView(HadoopObjectView):
def add_details_tab(self):
self.add_tab(DataNodeDetails(self.app, self.name + ".details", self.object))
@@ -236,7 +241,7 @@
def render_title(self, session):
obj = self.object.get(session)
if obj is not None:
- return "Data node: %s" % xml_escape(obj.id)
+ return "Data node: %s" % xml_escape(obj.Id)
else:
return ""
@@ -258,4 +263,4 @@
def render_name(self, session):
'''sample....would only be called if DataNodeDetails display was overridden to say {name}'''
obj = self.object.get(session)
- return obj.name
\ No newline at end of file
+ return obj.Name
\ No newline at end of file
Modified: branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/gridhadoop/hadoop.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -7,12 +7,16 @@
from cumin.qmfadapter import ObjectQmfAdapter
from cumin.task import Task, TaskLink
+from datetime import datetime
+
from sage.util import MethodResult
from wooly import Session, Widget
from wooly.datatable import *
from wooly.util import StringCatalog, xml_escape
+from rosemary.model import RosemaryObject
+
strings = StringCatalog(__file__)
def compareHadoopIds(x, y):
@@ -239,3 +243,24 @@
hours = (value / 3600) - (days * 24)
minutes = (value / 60) - (days * 1440) - (hours * 60)
return '%02d:%02d:%02d' % (days, hours, minutes)
+
+class HadoopObject(RosemaryObject):
+ def __init__(self, aviary_response, cls):
+ self._class = cls
+ self._save_time = datetime.now()
+ properties = [prop for prop in cls._properties]
+ for prop in properties:
+ try:
+ if prop.name.lower() in ["id", "ipc"]:
+ setattr(self, prop.name, aviary_response[1][0]["ref"][prop.name.lower()])
+ else:
+ setattr(self, prop.name, aviary_response[1][0][prop.name.lower()])
+ except AttributeError, e:
+ print "Attribute error %s" % e
+ self.Name = self.Ipc
+ self._id = self.Id
+
+ def __repr__(self):
+ name = self.__class__.__name__
+ args = (name, self._class, self._id, self._save_time)
+ return "%s(%s,%s,%s)" % args
\ No newline at end of file
Modified: branches/elephant/cumin/python/cumin/gridhadoop/hadoopobjectframe.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/hadoopobjectframe.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/gridhadoop/hadoopobjectframe.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -1,14 +1,17 @@
from cumin.widgets import *
from cumin.parameters import *
from cumin.objectframe import ObjectView
+from hadoop import HadoopObject
class HadoopObjectFrame(CuminFrame):
'''
This frame houses the view page for a hadoop object
'''
- def __init__(self, app, name):
+ def __init__(self, app, name, module):
super(HadoopObjectFrame, self).__init__(app, name)
+ self.module = module
+
# This will hold the id of the object that should be assigned
# to self.object. The id and the object are typically determined during
# the "process" pass before a page is rendered.
@@ -47,12 +50,11 @@
self.object.set(session, obj)
super(HadoopObjectFrame, self).do_process(session)
-
class HadoopObjectView(ObjectView):
def render_title(self, session):
obj = self.object.get(session)
if obj is not None:
- return "%s" % xml_escape(obj.id)
+ return "%s" % xml_escape(obj.Id)
else:
return ""
Modified: branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -25,7 +25,7 @@
return results
class JobTrackerSelector(ObjectSelector):
- def __init__(self, app, name, check_owner=False):
+ def __init__(self, app, name, module, check_owner=False):
cls = app.model.com_redhat_cumin_grid_hadoop.JobTracker
super(JobTrackerSelector, self).__init__(app, name, cls)
@@ -136,7 +136,21 @@
def get_title(self, session, x):
return "Create a job tracker"
+ def enter(self, session, obj):
+ form_session = wooly.Session(self.app.form_page)
+ if obj:
+ self.form.id.set(form_session, obj._id)
+
+ ## TODO find a better way to do this
+ self.form.hadoophost.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].host.get(session))
+ self.form.return_url.set(form_session, session.marshal())
+ self.form.show(form_session)
+
+ self.do_enter(session, obj, form_session)
+ return form_session
+
+
class JobTrackerCreateForm(HadoopNodeCreateForm):
def __init__(self, app, name, task, cls):
super(JobTrackerCreateForm, self).__init__(app, name, task, cls)
@@ -147,8 +161,8 @@
self.binfile = BinFileLoc(app, "binfile")
self.add_field(self.binfile)
- self.hadoophost = HadoopHostField(app, "hadoophost")
- self.add_field(self.hadoophost)
+ self.hadoophost = Parameter(app, "hadoophost")
+ self.add_parameter(self.hadoophost)
def process_display(self, session):
self.scheduler.validate(session)
@@ -172,13 +186,13 @@
return "Create a job tracker"
class JobTrackerFrame(HadoopObjectFrame):
- def __init__(self, app, name):
- super(JobTrackerFrame, self).__init__(app, name)
-
+ def __init__(self, app, name, module):
+ super(JobTrackerFrame, self).__init__(app, name, module)
+
self.view = JobTrackerObjectView(app, "view", self.object)
self.replace_child(self.view)
- self.task_tracker_frame = TaskTrackerFrame(app, "tasktrackerframe")
+ self.task_tracker_frame = TaskTrackerFrame(app, "tasktrackerframe", module)
self.add_mode(self.task_tracker_frame)
self.overview = JobTrackerOverview(app, self.name + ".jtoverview", self.object)
@@ -187,7 +201,7 @@
def get_title(self, session):
retval = ""
try:
- retval = "Job tracker:'%s' (%s)" % (xml_escape(self.object.get(session).name), xml_escape(self.object.get(session).id))
+ retval = "Job tracker:'%s' (%s)" % (xml_escape(self.object.get(session).Name), xml_escape(self.object.get(session).Id))
except Exception, e:
pass
return retval
@@ -196,8 +210,8 @@
class Object(object):
pass
fake = Object()
- fake.name = id
- fake.id = id
+ fake.Name = id
+ fake.Id = id
return fake
class JobTrackerObjectView(HadoopObjectView):
Modified: branches/elephant/cumin/python/cumin/gridhadoop/main.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/main.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/gridhadoop/main.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -25,7 +25,7 @@
def init(self):
super(Module, self).init()
- self.frame = GridHadoopFrame(self.app, "gridhadoop")
+ self.frame = GridHadoopFrame(self.app, "gridhadoop", self)
self.frame.cumin_module = self.name
self.app.main_page.main.gridhadoop = self.frame
@@ -33,13 +33,15 @@
class GridHadoopFrame(CuminFrame):
- def __init__(self, app, name, check_owner=False):
+ def __init__(self, app, name, module, check_owner=False):
super(GridHadoopFrame, self).__init__(app, name)
+ self.module = module
+
self.view = GridHadoopView(app, name + ".view", check_owner)
self.add_mode(self.view)
- self.scheduler_frame = HadoopSchedulerFrame(app, name + ".ghschedframe", check_owner)
+ self.scheduler_frame = HadoopSchedulerFrame(app, name + ".ghschedframe", check_owner, self.module)
self.add_mode(self.scheduler_frame)
# self.name_node_frame = NameNodeFrame(app, name + ".namenodeframe")
Modified: branches/elephant/cumin/python/cumin/gridhadoop/namenode.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/namenode.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/gridhadoop/namenode.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -23,7 +23,7 @@
return results
class NameNodeSelector(ObjectSelector):
- def __init__(self, app, name, check_owner=False):
+ def __init__(self, app, name, module, check_owner=False):
cls = app.model.com_redhat_cumin_grid_hadoop.NameNode
super(NameNodeSelector, self).__init__(app, name, cls)
@@ -31,7 +31,7 @@
self.add_search_filter(self.table.id_col)
self.table.adapter = NameNodeAdapter(app, cls, check_owner)
- task = NameNodeCreate(app, name + ".create")
+ task = NameNodeCreate(app, name + ".create", module)
link = TaskLink(app, "nameNode_create", task)
self.links.add_child(link)
@@ -72,9 +72,11 @@
return frame.get_href(session, id, frame.parent.host.get(session))
class NameNodeCreate(Task):
- def __init__(self, app, name):
+ def __init__(self, app, name, module):
super(NameNodeCreate, self).__init__(app, name)
cls = app.model.com_redhat_cumin_grid_hadoop.NameNode
+
+ self.module = module
self.form = NameNodeCreateForm(app, self.name, self, cls)
self.invoc = None
@@ -92,7 +94,20 @@
def get_title(self, session, x):
return "Create a name node"
+ def enter(self, session, obj):
+ form_session = wooly.Session(self.app.form_page)
+ if obj:
+ self.form.id.set(form_session, obj._id)
+
+ self.form.hadoophost.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"].host.get(session))
+ self.form.return_url.set(form_session, session.marshal())
+ self.form.show(form_session)
+
+ self.do_enter(session, obj, form_session)
+ return form_session
+
+
class NameNodeCreateForm(HadoopNodeCreateForm):
def __init__(self, app, name, task, cls):
super(NameNodeCreateForm, self).__init__(app, name, task, cls)
@@ -100,8 +115,8 @@
self.binfile = BinFileLoc(app, "binfile")
self.add_field(self.binfile)
- self.hadoophost = HadoopHostField(app, "hadoophost")
- self.add_field(self.hadoophost)
+ self.hadoophost = Parameter(app, "hadoophost")
+ self.add_parameter(self.hadoophost)
def process_display(self, session):
self.scheduler.validate(session)
@@ -157,42 +172,36 @@
return "Remove name nodes"
class NameNodeFrame(HadoopObjectFrame):
- def __init__(self, app, name):
- super(NameNodeFrame, self).__init__(app, name)
+ def __init__(self, app, name, module):
+ super(NameNodeFrame, self).__init__(app, name, module)
self.view = NameNodeObjectView(app, "view", self.object)
self.replace_child(self.view)
- self.data_node_frame = DataNodeFrame(app, "datanodeframe")
+ self.data_node_frame = DataNodeFrame(app, "datanodeframe", module)
self.add_mode(self.data_node_frame)
- self.overview = NameNodeOverview(app, self.name + ".nnoverview", self.object)
+ self.overview = NameNodeOverview(app, self.name + ".nnoverview", self.object, self.module)
self.view.add_tab(self.overview)
def get_title(self, session):
- retval = ""
- try:
- retval = "Name node:'%s' (%s)" % (xml_escape(self.object.get(session).name), xml_escape(self.object.get(session).id))
- except Exception, e:
- pass
+ retval = "Name node:'%s' (%s)" % (xml_escape(self.object.get(session).Name), xml_escape(self.object.get(session).Id))
return retval
def get_object(self, session, id):
- class Object(object):
- pass
- fake = Object()
- fake.name = id
- fake.id = id
- return fake
+ host = self.frame.host.get(session)
+ response = self.app.remote.get_name_node(self.frame.host.get(session), [id])
+ obj = HadoopObject(response, self.app.model.com_redhat_cumin_grid_hadoop.NameNode)
+ return obj
class NameNodeObjectView(HadoopObjectView):
def add_details_tab(self):
self.add_tab(NameNodeDetails(self.app, self.name + ".details", self.object))
class NameNodeOverview(Widget):
- def __init__(self, app, name, node):
+ def __init__(self, app, name, node, module):
super(NameNodeOverview, self).__init__(app, name)
- self.add_child(DataNodeSelector(app, self.name + ".dnfornn", False))
+ self.add_child(DataNodeSelector(app, self.name + ".dnfornn", module, False))
def render_title(self, session):
return "Data nodes"
Modified: branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -71,30 +71,30 @@
class HadoopSchedulerFrame(HadoopObjectFrame):
- def __init__(self, app, name, check_owner):
- super(HadoopSchedulerFrame, self).__init__(app, name)
+ def __init__(self, app, name, check_owner, module):
+ super(HadoopSchedulerFrame, self).__init__(app, name, module)
- self.name_node_frame = NameNodeFrame(app, name + ".namenodeframe")
+ self.name_node_frame = NameNodeFrame(app, name + ".namenodeframe", self.module)
self.add_mode(self.name_node_frame)
- self.job_tracker_frame = JobTrackerFrame(app, name + ".jobtrackerframe")
+ self.job_tracker_frame = JobTrackerFrame(app, name + ".jobtrackerframe", self.module)
self.add_mode(self.job_tracker_frame)
- self.view.add_tab(NameNodeSelector(app, name + ".namenodes", check_owner))
- self.view.add_tab(JobTrackerSelector(app, name + ".jobtrackers", check_owner))
+ self.view.add_tab(NameNodeSelector(app, name + ".namenodes", self.module, check_owner))
+ self.view.add_tab(JobTrackerSelector(app, name + ".jobtrackers", self.module, check_owner))
def get_object(self, session, id):
class Object(object):
pass
sched = Object()
- sched.name = id
- sched.id = id
+ sched.Name = id
+ sched.Id = id
return sched
def get_title(self, session):
retval = ""
try:
- retval = "Hadoop scheduler:'%s'" % (xml_escape(self.object.get(session).name))
+ retval = "Hadoop scheduler:'%s'" % (xml_escape(self.object.get(session).Name))
except Exception, e:
pass
return retval
Modified: branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py
===================================================================
--- branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -135,6 +135,7 @@
self.form.id.set(form_session, obj._id)
## TODO find a better way to do this
+ self.form.hadoophost.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].host.get(session))
self.form.jt_id.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].children_by_name["gridhadoop.ghschedframe.jobtrackerframe"].id.get(session))
self.form.return_url.set(form_session, session.marshal())
self.form.show(form_session)
@@ -156,8 +157,8 @@
self.count = self.CountField(app, "count")
self.add_field(self.count)
- self.hadoophost = HadoopHostField(app, "hadoophost")
- self.add_field(self.hadoophost)
+ self.hadoophost = Parameter(app, "hadoophost")
+ self.add_parameter(self.hadoophost)
self.jt_id = Parameter(app, "jt_id")
self.add_parameter(self.jt_id)
@@ -262,8 +263,8 @@
class TaskTrackerFrame(HadoopObjectFrame):
- def __init__(self, app, name):
- super(TaskTrackerFrame, self).__init__(app, name)
+ def __init__(self, app, name, module):
+ super(TaskTrackerFrame, self).__init__(app, name, module)
self.view = TaskTrackerObjectView(app, "view", self.object)
self.replace_child(self.view)
@@ -274,7 +275,7 @@
def get_title(self, session):
retval = ""
try:
- retval = "Task tracker:'%s' (%s)" % (xml_escape(self.object.get(session).name), xml_escape(self.object.get(session).id))
+ retval = "Task tracker:'%s' (%s)" % (xml_escape(self.object.get(session).Name), xml_escape(self.object.get(session).Id))
except Exception, e:
pass
return retval
@@ -283,8 +284,8 @@
class Object(object):
pass
fake = Object()
- fake.name = id
- fake.id = id
+ fake.Name = id
+ fake.Id = id
return fake
class TaskTrackerObjectView(HadoopObjectView):
@@ -294,7 +295,7 @@
def render_title(self, session):
obj = self.object.get(session)
if obj is not None:
- return "Task tracker: %s" % xml_escape(obj.id)
+ return "Task tracker: %s" % xml_escape(obj.Id)
else:
return ""
@@ -316,4 +317,4 @@
def render_name(self, session):
'''sample....would only be called if TaskTrackerDetails display was overridden to say {name}'''
obj = self.object.get(session)
- return obj.name
\ No newline at end of file
+ return obj.Name
\ No newline at end of file
Modified: branches/elephant/cumin/python/cumin/user/hadoop/main.py
===================================================================
--- branches/elephant/cumin/python/cumin/user/hadoop/main.py 2013-02-11 18:25:59 UTC (rev 5681)
+++ branches/elephant/cumin/python/cumin/user/hadoop/main.py 2013-02-11 21:10:39 UTC (rev 5682)
@@ -20,7 +20,7 @@
# CuminModule classes will add themselves to "app" as an attribute
# during intialization so we can use that to check...
if hasattr(self.app, "user") and self.app.user:
- self.frame = GridHadoopFrame(self.app, "userhadoop", check_owner=True)
+ self.frame = GridHadoopFrame(self.app, "userhadoop", self, check_owner=True)
self.frame.cumin_module = self.name
self.app.user.add_to_view(self.frame)
else:
11 years, 2 months
r5681 - trunk/sage/python/sage/aviary
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-02-11 18:25:59 +0000 (Mon, 11 Feb 2013)
New Revision: 5681
Modified:
trunk/sage/python/sage/aviary/aviaryoperations.py
Log:
Exception under Python 2.4 does not have a ".message" attribute
BZ909952
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2013-02-08 16:48:42 UTC (rev 5680)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2013-02-11 18:25:59 UTC (rev 5681)
@@ -129,7 +129,7 @@
result = _AviaryCommon._pretty_result(e,
self.locator.locator_uri)
log.debug("AviaryOperations: failed to get endpoints, " \
- "exception message '%s'" % result.message)
+ "exception message '%s'" % str(result))
raise result
urls = []
11 years, 2 months