r5755 - trunk/sage/python/sage/aviary
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-06-19 18:38:42 +0000 (Wed, 19 Jun 2013)
New Revision: 5755
Modified:
trunk/sage/python/sage/aviary/aviaryoperations.py
Log:
Don't allow duplicates in external name node and job tracker lists
Allow sync or aysnc
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2013-06-19 18:13:58 UTC (rev 5754)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2013-06-19 18:38:42 UTC (rev 5755)
@@ -1109,27 +1109,43 @@
def get_task_tracker_list(self, owner=None, callback=None):
return self._get_node_list(self.get_task_tracker, owner, callback)
- def add_external_name_node(self, ipc, url):
+ def add_external_name_node(self, ipc, url, callback=None):
self._hadoop_lock.acquire()
- self._external_name_nodes.append((ipc, url))
+ if not (ipc, url) in self._external_name_nodes:
+ self._external_name_nodes.append((ipc, url))
self._hadoop_lock.release()
-
- def get_external_name_nodes(self):
+ if callback:
+ callback("OK")
+ else:
+ return "OK"
+
+ def get_external_name_nodes(self, callback=None):
self._hadoop_lock.acquire()
res = copy.deepcopy(self._external_name_nodes)
self._hadoop_lock.release()
- return res
+ if callback:
+ callback("OK", res)
+ else:
+ return ("OK", res)
- def add_external_job_tracker(self, ipc, url):
+ def add_external_job_tracker(self, ipc, url, callback=None):
self._hadoop_lock.acquire()
- self._external_job_trackers.append((ipc, url))
+ if not (ipc, url) in self._external_job_trackers:
+ self._external_job_trackers.append((ipc, url))
self._hadoop_lock.release()
+ if callback:
+ callback("OK")
+ else:
+ return "OK"
- def get_external_job_trackers(self):
+ def get_external_job_trackers(self, callback=None):
self._hadoop_lock.acquire()
res = copy.deepcopy(self._external_job_trackers)
self._hadoop_lock.release()
- return res
+ if callback:
+ callback("OK", res)
+ else:
+ return ("OK", res)
def _make_id(self, client, val):
10 years, 10 months
r5754 - trunk/sage/python/sage/aviary
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-06-19 18:13:58 +0000 (Wed, 19 Jun 2013)
New Revision: 5754
Modified:
trunk/sage/python/sage/aviary/aviaryoperations.py
Log:
Stub implementation to allow addition of external name nodes and job trackers
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2013-06-19 17:38:03 UTC (rev 5753)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2013-06-19 18:13:58 UTC (rev 5754)
@@ -7,6 +7,7 @@
import time
import sage
import socket
+import copy
from datetime import datetime
from threading import Lock
@@ -76,7 +77,6 @@
# Nice, friendly strings for error messages on lookup
_nice = {("SCHEDULER","JOB"): "job service",
("CUSTOM","QUERY_SERVER"): "query service",
- ("SCHEDULER","HADOOP"): "hadoop service",
("COLLECTOR","POOL"): "collector service",
("SCHEDULER","HADOOP"): "hadoop service"}
@@ -1027,6 +1027,11 @@
# Equivalence? We don't want no stinking QMF structural equivalence!
self.use_MethodResult_for_sync_calls = False
+ # This lock protects external name_node and job_tracker lists
+ self._hadoop_lock = Lock()
+ self._external_name_nodes = []
+ self._external_job_trackers = []
+
def start_name_node(self, host, bin_file, owner, description, callback):
assert callable(callback)
@@ -1104,6 +1109,28 @@
def get_task_tracker_list(self, owner=None, callback=None):
return self._get_node_list(self.get_task_tracker, owner, callback)
+ def add_external_name_node(self, ipc, url):
+ self._hadoop_lock.acquire()
+ self._external_name_nodes.append((ipc, url))
+ self._hadoop_lock.release()
+
+ def get_external_name_nodes(self):
+ self._hadoop_lock.acquire()
+ res = copy.deepcopy(self._external_name_nodes)
+ self._hadoop_lock.release()
+ return res
+
+ def add_external_job_tracker(self, ipc, url):
+ self._hadoop_lock.acquire()
+ self._external_job_trackers.append((ipc, url))
+ self._hadoop_lock.release()
+
+ def get_external_job_trackers(self):
+ self._hadoop_lock.acquire()
+ res = copy.deepcopy(self._external_job_trackers)
+ self._hadoop_lock.release()
+ return res
+
def _make_id(self, client, val):
def url(v):
10 years, 10 months
r5753 - trunk/cumin/etc
by tmckay@fedoraproject.org
Author: tmckay
Date: 2013-06-19 17:38:03 +0000 (Wed, 19 Jun 2013)
New Revision: 5753
Modified:
trunk/cumin/etc/cumin.conf
Log:
Add a convenience data setting for gridoop with "hadoop" persona (hadoop only)
Modified: trunk/cumin/etc/cumin.conf
===================================================================
--- trunk/cumin/etc/cumin.conf 2013-06-18 18:09:45 UTC (rev 5752)
+++ trunk/cumin/etc/cumin.conf 2013-06-19 17:38:03 UTC (rev 5753)
@@ -82,6 +82,10 @@
# section. See additional comments in [data] section.
#datas: data
+# For gridoop, with persona set to "hadoop", use this
+# It looks only for the collector and schedulers
+# datas: data.grid-hadoop
+
# Reporting is off by default.
# To enable reporting features, uncomment the following line.
# reports: report
@@ -132,6 +136,15 @@
exclude-classes: com.redhat.grid:Slot, com.redhat.grid:Submission,
com.redhat.grid:JobServer, com.redhat.grid:Submitter, com.redhat.grid:Scheduler
+[data.grid-hadoop]
+# log-file: /usr/share/cumin/log/data.grid.log
+# expire-enabled: True
+expire-interval: 900
+# expire-threshold: 86400
+# vacuum-enabled: True
+vacuum-interval: 900
+include-classes: com.redhat.grid:Collector, com.redhat.grid:Scheduler
+
[data.grid-slots]
# log-file: /usr/share/cumin/log/data.grid-slots.log
expire-enabled: False
10 years, 10 months
r5752 - trunk/cumin/python/cumin/gridhadoop
by croberts@fedoraproject.org
Author: croberts
Date: 2013-06-18 18:09:45 +0000 (Tue, 18 Jun 2013)
New Revision: 5752
Modified:
trunk/cumin/python/cumin/gridhadoop/jobtracker.py
trunk/cumin/python/cumin/gridhadoop/namenode.py
Log:
Renaming some of the hadoop tabs. Also removing the Ipc column from the name nodes and job tracker tables (it sill appears in the details tab of those objects).
Modified: trunk/cumin/python/cumin/gridhadoop/jobtracker.py
===================================================================
--- trunk/cumin/python/cumin/gridhadoop/jobtracker.py 2013-06-18 17:57:47 UTC (rev 5751)
+++ trunk/cumin/python/cumin/gridhadoop/jobtracker.py 2013-06-18 18:09:45 UTC (rev 5752)
@@ -48,7 +48,7 @@
return JobTrackerTable(app, name, cls)
def render_title(self, session):
- return "Hadoop job trackers"
+ return "Job trackers"
class JobTrackerTable(ObjectQmfSelectorTable):
def __init__(self, app, name, cls):
@@ -56,7 +56,6 @@
self.id_col = self.JobTrackerColumn(app, "idcol", cls.Id, cls.Id, ".jobtrackerframe")
self.id_col.width = "10%"
- self.ipc_col = HadoopIpcColumn(app, "ipc", cls.Ipc, "20%")
self.http_col = HadoopHttpColumn(app, "http", cls.Http, cls.Http, None)
self.state_col = ObjectTableColumn(app, "statecol", cls.State)
self.state_col.width = "10%"
@@ -65,7 +64,6 @@
self.owner_col = ObjectTableColumn(app, "ownercol", cls.Owner)
self.add_column(self.id_col)
- self.add_column(self.ipc_col)
self.add_column(self.http_col)
self.add_column(self.state_col)
self.add_column(self.uptime_col)
Modified: trunk/cumin/python/cumin/gridhadoop/namenode.py
===================================================================
--- trunk/cumin/python/cumin/gridhadoop/namenode.py 2013-06-18 17:57:47 UTC (rev 5751)
+++ trunk/cumin/python/cumin/gridhadoop/namenode.py 2013-06-18 18:09:45 UTC (rev 5752)
@@ -44,7 +44,7 @@
return NameNodeTable(app, name, cls)
def render_title(self, session):
- return "HDFS Name nodes"
+ return "Name nodes"
class NameNodeTable(ObjectQmfSelectorTable):
def __init__(self, app, name, cls):
@@ -52,7 +52,6 @@
self.id_col = self.NameNodeColumn(app, "idcol", cls.Id, cls.Id, ".namenodeframe")
self.id_col.width = "10%"
- self.ipc_col = HadoopIpcColumn(app, "ipc", cls.Ipc, "20%")
self.http_col = HadoopHttpColumn(app, "http", cls.Http, cls.Http, None)
self.state_col = ObjectTableColumn(app, "statecol", cls.State)
self.state_col.width = "10%"
@@ -61,7 +60,6 @@
self.owner_col = ObjectTableColumn(app, "ownercol", cls.Owner)
self.add_column(self.id_col)
- self.add_column(self.ipc_col)
self.add_column(self.http_col)
self.add_column(self.state_col)
self.add_column(self.uptime_col)
10 years, 10 months
r5751 - trunk/cumin/python/cumin/gridhadoop
by croberts@fedoraproject.org
Author: croberts
Date: 2013-06-18 17:57:47 +0000 (Tue, 18 Jun 2013)
New Revision: 5751
Modified:
trunk/cumin/python/cumin/gridhadoop/main.py
Log:
Grid-hadoop is now just "Hadoop" (tab label).
Modified: trunk/cumin/python/cumin/gridhadoop/main.py
===================================================================
--- trunk/cumin/python/cumin/gridhadoop/main.py 2013-06-11 16:49:05 UTC (rev 5750)
+++ trunk/cumin/python/cumin/gridhadoop/main.py 2013-06-18 17:57:47 UTC (rev 5751)
@@ -48,7 +48,7 @@
self.add_mode(self.job_tracker_frame)
def render_title(self, session):
- return "Grid-hadoop"
+ return "Hadoop"
class GridHadoopView(Widget):
def __init__(self, app, name, check_owner):
10 years, 10 months
r5750 - trunk/cumin/python/cumin/gridhadoop
by croberts@fedoraproject.org
Author: croberts
Date: 2013-06-11 16:49:05 +0000 (Tue, 11 Jun 2013)
New Revision: 5750
Modified:
trunk/cumin/python/cumin/gridhadoop/datanode.py
trunk/cumin/python/cumin/gridhadoop/jobtracker.py
trunk/cumin/python/cumin/gridhadoop/main.py
trunk/cumin/python/cumin/gridhadoop/namenode.py
trunk/cumin/python/cumin/gridhadoop/tasktracker.py
Log:
Gridoop tweaks: 1) Removing the scheduler selection page -- now always assuming that the 0th schedd in the response is the one we want. 2) Fixing filter by owner bug for name node/job tracker lists.
Modified: trunk/cumin/python/cumin/gridhadoop/datanode.py
===================================================================
--- trunk/cumin/python/cumin/gridhadoop/datanode.py 2013-03-21 15:01:16 UTC (rev 5749)
+++ trunk/cumin/python/cumin/gridhadoop/datanode.py 2013-06-11 16:49:05 UTC (rev 5750)
@@ -149,10 +149,9 @@
if obj:
self.form.id.set(form_session, obj._id)
-
- self.form.hadoophost.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"].host.get(session))
- self.form.nn_id.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"]\
- .children_by_name[self.module.frame.name + ".ghschedframe.namenodeframe"].id.get(session))
+ hadoop_schedulers = self.app.remote.get_hosts("SCHEDULER", "HADOOP")
+ self.form.hadoophost.set(form_session, hadoop_schedulers[0])
+ self.form.nn_id.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".namenodeframe"].id.get(session))
self.form.return_url.set(form_session, session.marshal())
self.form.show(form_session)
Modified: trunk/cumin/python/cumin/gridhadoop/jobtracker.py
===================================================================
--- trunk/cumin/python/cumin/gridhadoop/jobtracker.py 2013-03-21 15:01:16 UTC (rev 5749)
+++ trunk/cumin/python/cumin/gridhadoop/jobtracker.py 2013-06-11 16:49:05 UTC (rev 5750)
@@ -75,7 +75,7 @@
def render_cell_href(self, session, record):
id = unescape_entity(record[self.id_field.index])
frame = self.frame.children_by_name[self.frame.name + self.frame_path]
- return frame.get_href(session, id, frame.parent.host.get(session))
+ return frame.get_href(session, id, self.app.remote.get_hosts("SCHEDULER", "HADOOP")[0])
class JobTrackerDelete(HadoopNodeDeleteTask):
def __init__(self, app, selector, name, module):
@@ -148,7 +148,7 @@
if obj:
self.form.id.set(form_session, obj._id)
- self.form.hadoophost.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].host.get(session))
+ self.form.hadoophost.set(form_session, self.app.remote.get_hosts("SCHEDULER", "HADOOP")[0])
self.form.return_url.set(form_session, session.marshal())
self.form.show(form_session)
@@ -215,8 +215,8 @@
return retval
def get_object(self, session, id):
- host = self.frame.host.get(session)
- response = self.app.remote.get_job_tracker(self.frame.host.get(session), [id])
+ host = self.app.remote.get_hosts("SCHEDULER", "HADOOP")[0]
+ response = self.app.remote.get_job_tracker(host, [id])
obj = HadoopObject(response, self.app.model.com_redhat_cumin_grid_hadoop.JobTracker)
return obj
Modified: trunk/cumin/python/cumin/gridhadoop/main.py
===================================================================
--- trunk/cumin/python/cumin/gridhadoop/main.py 2013-03-21 15:01:16 UTC (rev 5749)
+++ trunk/cumin/python/cumin/gridhadoop/main.py 2013-06-11 16:49:05 UTC (rev 5750)
@@ -41,8 +41,11 @@
self.view = GridHadoopView(app, name + ".view", check_owner)
self.add_mode(self.view)
- self.scheduler_frame = HadoopSchedulerFrame(app, name + ".ghschedframe", check_owner, self.module)
- self.add_mode(self.scheduler_frame)
+ self.name_node_frame = NameNodeFrame(app, name + ".namenodeframe", self.module)
+ self.add_mode(self.name_node_frame)
+
+ self.job_tracker_frame = JobTrackerFrame(app, name + ".jobtrackerframe", self.module)
+ self.add_mode(self.job_tracker_frame)
def render_title(self, session):
return "Grid-hadoop"
@@ -54,4 +57,5 @@
self.tabs = TabbedModeSet(app, "childtabs")
self.add_child(self.tabs)
- self.tabs.add_tab(HadoopSchedulerSelector(app, name + ".hscheds"))
+ self.tabs.add_tab(NameNodeSelector(app, name + ".namenodes", self, check_owner))
+ self.tabs.add_tab(JobTrackerSelector(app, name + ".jobtrackers", self, check_owner))
Modified: trunk/cumin/python/cumin/gridhadoop/namenode.py
===================================================================
--- trunk/cumin/python/cumin/gridhadoop/namenode.py 2013-03-21 15:01:16 UTC (rev 5749)
+++ trunk/cumin/python/cumin/gridhadoop/namenode.py 2013-06-11 16:49:05 UTC (rev 5750)
@@ -71,7 +71,8 @@
def render_cell_href(self, session, record):
id = unescape_entity(record[self.id_field.index])
frame = self.frame.children_by_name[self.frame.name + self.frame_path]
- return frame.get_href(session, id, frame.parent.host.get(session))
+ hadoop_schedulers = self.app.remote.get_hosts("SCHEDULER", "HADOOP")
+ return frame.get_href(session, id, hadoop_schedulers[0])
class NameNodeCreate(Task):
def __init__(self, app, name, module):
@@ -102,7 +103,8 @@
if obj:
self.form.id.set(form_session, obj._id)
- self.form.hadoophost.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".ghschedframe"].host.get(session))
+ hadoop_schedulers = self.app.remote.get_hosts("SCHEDULER", "HADOOP")
+ self.form.hadoophost.set(form_session, hadoop_schedulers[0])
self.form.return_url.set(form_session, session.marshal())
self.form.show(form_session)
@@ -194,8 +196,9 @@
return retval
def get_object(self, session, id):
- host = self.frame.host.get(session)
- response = self.app.remote.get_name_node(self.frame.host.get(session), [id])
+ hadoop_schedulers = self.app.remote.get_hosts("SCHEDULER", "HADOOP")
+ host = hadoop_schedulers[0]
+ response = self.app.remote.get_name_node(host, [id])
obj = HadoopObject(response, self.app.model.com_redhat_cumin_grid_hadoop.NameNode)
return obj
Modified: trunk/cumin/python/cumin/gridhadoop/tasktracker.py
===================================================================
--- trunk/cumin/python/cumin/gridhadoop/tasktracker.py 2013-03-21 15:01:16 UTC (rev 5749)
+++ trunk/cumin/python/cumin/gridhadoop/tasktracker.py 2013-06-11 16:49:05 UTC (rev 5750)
@@ -76,7 +76,7 @@
def render_cell_href(self, session, record):
id = unescape_entity(record[self.id_field.index])
frame = self.frame.children_by_name[self.frame_path]
- return frame.get_href(session, id, frame.parent.host.get(session))
+ return frame.get_href(session, id, self.app.remote.get_hosts("SCHEDULER", "HADOOP")[0])
class TaskTrackerDelete(HadoopNodeDeleteTask):
def __init__(self, app, selector, name, module):
@@ -146,8 +146,8 @@
if obj:
self.form.id.set(form_session, obj._id)
- self.form.hadoophost.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].host.get(session))
- self.form.jt_id.set(form_session, self.app.gridhadoop.frame.children_by_name["gridhadoop.ghschedframe"].children_by_name["gridhadoop.ghschedframe.jobtrackerframe"].id.get(session))
+ self.form.hadoophost.set(form_session, self.app.remote.get_hosts("SCHEDULER", "HADOOP")[0])
+ self.form.jt_id.set(form_session, self.module.frame.children_by_name[self.module.frame.name + ".jobtrackerframe"].id.get(session))
self.form.return_url.set(form_session, session.marshal())
self.form.show(form_session)
10 years, 10 months