[PATCH] Shell: Create a Matahari API
by Zane Bitter
The API takes the form of a store for the session state, wrapped around the
core API. Currently, acessing objects is done synchronously, so the cache
must be deleted (and the objects re-fetched) to see updated parameter
values. In the future, I expect to change this to get parameter updates
asynchronously.
---
src/python/CMakeLists.txt | 1
src/python/matahari/__init__.py | 1
src/python/matahari/api.py | 110 ++++++++++++++++++++++++++++++++++++++
src/python/matahari/core.py | 20 ++++++-
src/python/matahari/shell.py | 6 +-
src/python/matahari/shellmode.py | 71 ++++++++++++-------------
6 files changed, 169 insertions(+), 40 deletions(-)
create mode 100644 src/python/matahari/api.py
diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt
index f4e9b4a..2468ced 100644
--- a/src/python/CMakeLists.txt
+++ b/src/python/CMakeLists.txt
@@ -25,6 +25,7 @@ if(WITH-QMF)
py_module(__init__ matahari)
py_module(rpc_server matahari)
py_module(core matahari)
+ py_module(api matahari)
py_module(shell matahari)
py_module(shellmode matahari)
py_module(shelltypes matahari)
diff --git a/src/python/matahari/__init__.py b/src/python/matahari/__init__.py
index e69de29..096d22e 100644
--- a/src/python/matahari/__init__.py
+++ b/src/python/matahari/__init__.py
@@ -0,0 +1 @@
+from api import Matahari, BrokerConnection
diff --git a/src/python/matahari/api.py b/src/python/matahari/api.py
new file mode 100644
index 0000000..718d814
--- /dev/null
+++ b/src/python/matahari/api.py
@@ -0,0 +1,110 @@
+import core
+
+class BrokerConnection(object):
+ def __init__(self, host='localhost', port=49000, ssl=False):
+ self.host = host
+ self.port = 49000
+ self.ssl = ssl
+
+class Proxy(object):
+ def __init__(self, state):
+ self._state = state
+ agents = state.manager.agents(state._hosts)
+ self._objects = state.manager.get(state._class,
+ package=state._package,
+ agents=agents)
+ self._schemata = set(o.getSchema() for o in self._objects)
+ self._methodnames = set(m.name for m in self._methods())
+
+ def __getattr__(self, attr):
+ if attr in self._methodnames:
+ def call_method(*args):
+ return self._state.manager.invoke_method(self._objects,
+ attr,
+ *args)
+ return call_method
+ else:
+ return [getattr(o, attr) for o in self._objects]
+
+ def __setattr__(self, attr, val):
+ if attr.startswith('_'):
+ self.__dict__[attr] = val
+ return
+
+ for o in self._objects:
+ setattr(o, attr, val)
+
+ def __str__(self):
+ return str(self._objects)
+
+ def __repr__(self):
+ return repr(self._objects)
+
+ def _properties(self):
+ return set.union(*[set(s.getProperties()) for s in self._schemata])
+
+ def _statistics(self):
+ return set.union(*[set(s.getStatistics()) for s in self._schemata])
+
+ def _methods(self):
+ return set.union(*[set(s.getMethods()) for s in self._schemata])
+
+class CacheDescriptor(object):
+ def __init__(self, constructor):
+ self.cache = {}
+ self.constructor = constructor
+
+ def __get__(self, instance, owner):
+ if instance is None:
+ return None
+ if instance not in self.cache:
+ if instance._class is None:
+ proxy = None
+ else:
+ proxy = self.constructor(instance)
+ self.cache[instance] = proxy
+ return self.cache[instance]
+
+ def __delete__(self, instance):
+ if instance in self.cache:
+ del self.cache[instance]
+
+class Matahari(object):
+
+ objects = CacheDescriptor(Proxy)
+
+ def __init__(self, broker=BrokerConnection()):
+ self._qmf = core.AsyncHandler()
+ self.manager = core.Manager(broker.host, broker.port, broker.ssl,
+ self._qmf)
+ self._hosts = None
+ self._package = None
+ self._class = None
+
+ def set_hosts(self, hosts=None):
+ self._hosts = hosts
+ del self.objects
+
+ def set_class(self, klass, package='org.matahariproject'):
+ self._class = klass
+ self._package = package
+ del self.objects
+
+ def selected_classname(self):
+ return '%s:%s' % (self._package, self._class)
+
+ def list_hosts(self):
+ return set(self.manager.hosts())
+
+ def list_packages(self):
+ return set(self._qmf.packages)
+
+ def list_classnames(self, package=None):
+ classes = list(self._qmf.classes)
+ pkgmatch = (package is None and
+ (lambda k: True) or
+ (lambda k: k.getPackageName() == package))
+ return set([k.getClassName() for k in classes if pkgmatch(k)])
+
+ def agents(self):
+ return set(self.manager.agents(self._hosts))
diff --git a/src/python/matahari/core.py b/src/python/matahari/core.py
index 6ae7b3d..0dd8ecb 100644
--- a/src/python/matahari/core.py
+++ b/src/python/matahari/core.py
@@ -4,6 +4,7 @@ import qmf.console as qc
(TIMEOUT,) = (10,)
+exclude_packages = frozenset(['org.apache.qpid.broker'])
_chain = itertools.chain.from_iterable
@@ -37,6 +38,7 @@ class MHAgent(object):
"""Get the host on which the agent in running"""
return self._host
+
class Host(object):
"""A Host on which Matahari agents are running"""
def __init__(self, agentobj):
@@ -63,10 +65,13 @@ class Host(object):
"""Return the host (filesystem) UUID"""
return self._uuid
+
class AsyncHandler(qc.Console):
"""A handler for Asynchronous QMF events"""
def __init__(self):
self.method_response_handler = None
+ self.packages = []
+ self.classes = []
def handle_method_responses(self, callback):
"""Register a callback for QMF method responses"""
@@ -76,13 +81,24 @@ class AsyncHandler(qc.Console):
if self.method_response_handler:
self.method_response_handler(seq, response)
+ def newClass(self, kind, classKey):
+ if (kind == qc.SchemaClass.CLASS_KIND_TABLE and
+ classKey.getPackageName() not in exclude_packages):
+ self.classes.append(classKey)
+
+ def newPackage(self, name):
+ if name not in exclude_packages:
+ self.packages.append(name)
+
+
class Manager(object):
- def __init__(self, broker='localhost', port=49000, ssl=False):
+ def __init__(self, broker='localhost', port=49000, ssl=False,
+ async_handler=AsyncHandler()):
addr = '%(prot)s://%(host)s:%(port)u' % {'host': broker,
'port': port,
'prot': ssl and 'amqps'
or 'amqp'}
- self._async = AsyncHandler()
+ self._async = async_handler
self._session = qc.Session(self._async)
self._broker = self._session.addBroker(addr)
diff --git a/src/python/matahari/shell.py b/src/python/matahari/shell.py
index ed52de5..6635320 100644
--- a/src/python/matahari/shell.py
+++ b/src/python/matahari/shell.py
@@ -1,11 +1,11 @@
-import core
+import api
import shellmode
import interpreter
class MatahariShell(object):
def __init__(self, prompt='mhsh'):
- self.manager = core.Manager()
- initial_mode = shellmode.RootMode(self.manager)
+ self.state = api.Matahari()
+ initial_mode = shellmode.RootMode(self.state)
self.interpreter = interpreter.Interpreter(prompt, initial_mode, debug=True)
def __call__(self):
diff --git a/src/python/matahari/shellmode.py b/src/python/matahari/shellmode.py
index 785f8bf..96fca82 100644
--- a/src/python/matahari/shellmode.py
+++ b/src/python/matahari/shellmode.py
@@ -9,64 +9,66 @@ import shelltypes
class RootMode(Mode):
"""The default, root, shell mode."""
- def __init__(self, manager):
+ def __init__(self, state):
Mode.__init__(self)
- self.manager = manager
+ self.state = state
select_wrapper = Command('select', 'host',
- [shelltypes.Host(self.manager)])
+ [shelltypes.Host(self.state.manager)])
self += select_wrapper(self.select)
- def getFilteredMode(self, hosts):
- return FilteredMode(self.manager, hosts)
+ def getFilteredMode(self):
+ return FilteredMode(self.state)
- def getClassMode(self, package, klass):
- return ClassMode(self.manager, package, klass)
+ def getClassMode(self):
+ return ClassMode(self.state)
@Command('show', 'hosts')
def show_hosts(self, kw_show, kw_hosts):
"""Show the list of connected hosts."""
- self.write(*[str(h) for h in self.manager.hosts()])
+ self.write(*[str(h) for h in self.state.list_hosts()])
+
+ @Command('show', 'agents')
+ def show_agents(self, *kws):
+ """Show the list of connected agents."""
+ self.write(*[str(a) for a in self.state.agents()])
def select(self, kw_select, kw_host, *arg_hosts):
""" Select a set of hosts to restrict commands to."""
- self.shell.set_mode(self.getFilteredMode(arg_hosts))
+ self.state.set_hosts(arg_hosts)
+ self.shell.set_mode(self.getFilteredMode())
@Command('class', ('package', 'PACKAGE'), 'CLASS')
def package_class(self, kw_class, (kw_package, package), klass):
""" Select an object class to operate on."""
if kw_package is None:
package = 'org.matahariproject'
- self.shell.set_mode(self.getClassMode(package, klass))
+ self.state.set_class(klass, package)
+ self.shell.set_mode(self.getClassMode())
class FilteredMode(RootMode):
"""Shell mode filtered by host"""
- def __init__(self, manager, hosts):
- self.hosts = hosts
- RootMode.__init__(self, manager)
+ def __init__(self, state):
+ RootMode.__init__(self, state)
def getUnfilteredMode(self):
- return RootMode(self.manager)
+ return RootMode(self.state)
- def getClassMode(self, package, klass):
- return FilteredClassMode(self.manager, self.hosts, package, klass)
+ def getClassMode(self):
+ return FilteredClassMode(self.state)
@Command('clear', 'selection')
def clear_selection(self, kw_clear, kw_selection):
"""Clear the current host selection."""
+ self.state.set_hosts(None)
self.shell.set_mode(self.getUnfilteredMode())
@Command('show', 'selection')
def show_selection(self, *kws):
"""Show the current host selection."""
- self.write(*[str(h) for h in self.hosts])
-
- @Command('show', 'agents')
- def show_agents(self, *kws):
- """Show the list of connected agents."""
- self.write(*[str(a) for a in self.manager.agents(self.hosts)])
+ self.write(*[str(h) for h in self.state.hosts])
def prompt(self):
return '(host)'
@@ -75,19 +77,18 @@ class FilteredMode(RootMode):
class ClassMode(RootMode):
"""Shell mode filtered by QMF class."""
- def __init__(self, manager, package, klass):
- RootMode.__init__(self, manager)
- self.package = package
- self.klass = klass
+ def __init__(self, state):
+ RootMode.__init__(self, state)
- def getFilteredMode(self, hosts):
- return FilteredClassMode(self.manager, hosts, self.package, self.klass)
+ def getFilteredMode(self):
+ return FilteredClassMode(self.state)
def getUnclassifiedMode(self):
- return RootMode(self.manager)
+ return RootMode(self.state)
@Command('clear', 'class')
def clear_class(self, kw_clear, kw_class):
+ self.state.set_class(None, None)
self.shell.set_mode(self.getUnclassifiedMode())
@Command('select', 'property', 'PROPERTY', 'VALUE')
@@ -95,21 +96,21 @@ class ClassMode(RootMode):
pass
def prompt(self):
- return '[%s.%s]' % (self.package, self.klass)
+ return '[%s]' % (self.state.selected_classname(),)
class FilteredClassMode(FilteredMode, ClassMode):
"""Shell mode filtered by host and QMF class."""
- def __init__(self, manager, hosts, package, klass):
- FilteredMode.__init__(self, manager, hosts)
- ClassMode.__init__(self, manager, package, klass)
+ def __init__(self, state):
+ FilteredMode.__init__(self, state)
+ ClassMode.__init__(self, state)
def getUnfilteredMode(self):
- return ClassMode(self.manager, self.package, self.klass)
+ return ClassMode(self.state)
def getUnclassifiedMode(self):
- return FilteredMode(self.manager, self.hosts)
+ return FilteredMode(self.state)
def prompt(self):
return FilteredMode.prompt(self) + ClassMode.prompt(self)
12 years, 5 months
[PATCH] RPC: Install plugin directory
by Zane Bitter
---
matahari.spec.in | 1 +
src/python/matahari/rpc_server.py | 7 +++++--
src/rpc/CMakeLists.txt | 1 +
3 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/matahari.spec.in b/matahari.spec.in
index 81ce81e..827c9e4 100644
--- a/matahari.spec.in
+++ b/matahari.spec.in
@@ -581,6 +581,7 @@ test "x%{buildroot}" != "x" && rm -rf %{buildroot}
%files rpc
%defattr(644, root, root, 755)
%doc AUTHORS COPYING
+%attr(0755, -, -) %dir /usr/lib/matahari/plugins
%if %{systemd}
%{_unitdir}/matahari-rpc.service
diff --git a/src/python/matahari/rpc_server.py b/src/python/matahari/rpc_server.py
index 6cf2eea..eddcd4c 100644
--- a/src/python/matahari/rpc_server.py
+++ b/src/python/matahari/rpc_server.py
@@ -2,7 +2,7 @@ import sys, os
import json
import traceback
-(PLUGIN_PATH,) = ('.',) # TODO: choose proper install path
+(PLUGIN_PATH,) = ('/usr/lib/matahari/plugins',)
class Plugin(object):
def __init__(self, module):
@@ -28,7 +28,10 @@ def plugins(path=PLUGIN_PATH):
old_path = sys.path
try:
sys.path = [path] + old_path
- filenames = [os.path.splitext(f) for f in os.listdir(path)]
+ try:
+ filenames = [os.path.splitext(f) for f in os.listdir(path)]
+ except OSError:
+ return []
modnames = [n for n, e in filenames if e == '.py']
def getPlugin(name):
try:
diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt
index a43be57..a6794c3 100644
--- a/src/rpc/CMakeLists.txt
+++ b/src/rpc/CMakeLists.txt
@@ -16,5 +16,6 @@ if(WITH-QMF)
install(TARGETS ${QMF_AGENT} DESTINATION sbin)
+ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/usr/lib/matahari/plugins)")
endif(WITH-QMF)
12 years, 5 months
[PATCH 1/2] RPC: Move Python code to src/python directory
by Zane Bitter
This will allow us to install the rpc Python code and (later) the mhsh
shell as a single 'matahari' Python package.
---
matahari.spec.in | 19 ++++++++++++++++
src/CMakeLists.txt | 1 +
src/lib/rpc-python.c | 5 +++-
src/python/.gitignore | 2 ++
src/python/CMakeLists.txt | 30 +++++++++++++++++++++++++
src/python/matahari/__init__.py | 0
src/python/matahari/rpc_server.py | 45 +++++++++++++++++++++++++++++++++++++
src/rpc/.gitignore | 2 --
src/rpc/rpc.py | 45 -------------------------------------
9 files changed, 101 insertions(+), 48 deletions(-)
create mode 100644 src/python/.gitignore
create mode 100644 src/python/CMakeLists.txt
create mode 100644 src/python/matahari/__init__.py
create mode 100644 src/python/matahari/rpc_server.py
delete mode 100644 src/rpc/.gitignore
delete mode 100644 src/rpc/rpc.py
diff --git a/matahari.spec.in b/matahari.spec.in
index ada689e..81ce81e 100644
--- a/matahari.spec.in
+++ b/matahari.spec.in
@@ -57,6 +57,7 @@ BuildRequires: libcurl-devel
BuildRequires: nss-devel
BuildRequires: help2man
BuildRequires: augeas-devel >= 0.9.0
+BuildRequires: python2-devel
%if %{with qmf}
BuildRequires: qpid-cpp-client-devel > 0.7
@@ -157,6 +158,15 @@ Requires: kstart
%description agent-lib
C++ library containing the base class for Matahari agents
+%package python
+License: GPLv2+
+Summary: Matahari Python API
+Group: Applications/System
+Requires: python-qpid-qmf
+
+%description python
+Python libraries for Matahari
+
%package host
License: GPLv2+
Summary: QMF agent for remote hosts
@@ -195,6 +205,7 @@ Summary: QMF agent for RPC plugins
Group: Applications/System
Requires: %{name}-lib = %{version}-%{release}
Requires: %{name}-agent-lib = %{version}-%{release}
+Requires: %{name}-python = %{version}-%{release}
Requires(post): chkconfig
Requires(preun):chkconfig
Requires(preun):initscripts
@@ -511,6 +522,14 @@ test "x%{buildroot}" != "x" && rm -rf %{buildroot}
%dir %attr(0770, root, qpidd) %{_localstatedir}/lib/%{name}
%endif
+%files python
+%defattr(644, root, root, 755)
+%doc AUTHORS COPYING
+%dir %{python_sitelib}/matahari/
+%{python_sitelib}/matahari/*.py
+%{python_sitelib}/matahari/*.pyc
+%{python_sitelib}/matahari/*.pyo
+
%files network
%defattr(644, root, root, 755)
%doc AUTHORS COPYING
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 1aa486a..63f3ae8 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -185,6 +185,7 @@ endif(WITH-QMF)
### Subdirectories
add_subdirectory(broker)
+add_subdirectory(python)
add_subdirectory(lib)
add_subdirectory(host)
add_subdirectory(network)
diff --git a/src/lib/rpc-python.c b/src/lib/rpc-python.c
index d2388e9..73c65be 100644
--- a/src/lib/rpc-python.c
+++ b/src/lib/rpc-python.c
@@ -25,6 +25,9 @@
#include "matahari/errors.h"
+#define RPC_SERVER_MODULE "matahari.rpc_server"
+
+
struct mh_rpc_plugin_impl {
PyObject *plugin;
};
@@ -62,7 +65,7 @@ call_rpc_server_func(const char *funcname, PyObject *args)
{
PyObject *rpc_mod, *func, *result;
- rpc_mod = PyImport_ImportModule("rpc"); // TODO: move to matahari package
+ rpc_mod = PyImport_ImportModule(RPC_SERVER_MODULE);
if (!rpc_mod) {
mh_err("Error importing Matahari RPC Python module");
PyErr_Print();
diff --git a/src/python/.gitignore b/src/python/.gitignore
new file mode 100644
index 0000000..52e4e61
--- /dev/null
+++ b/src/python/.gitignore
@@ -0,0 +1,2 @@
+*.pyc
+*.pyo
diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt
new file mode 100644
index 0000000..3dc1050
--- /dev/null
+++ b/src/python/CMakeLists.txt
@@ -0,0 +1,30 @@
+FIND_PACKAGE(PythonInterp)
+
+macro(py_module MODULE DIRECTORY)
+ # Copy the Python file to the Binary directory
+ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/${MODULE}.py
+ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/${DIRECTORY})
+
+ # Compile .pyc and .pyo files
+ execute_process(COMMAND echo "PYTHON_EXECUTABLE = ${PYTHON_EXECUTABLE}")
+ execute_process(COMMAND ${PYTHON_EXECUTABLE} -m py_compile ${MODULE}.py
+ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${DIRECTORY})
+ execute_process(COMMAND ${PYTHON_EXECUTABLE} -O -m py_compile ${MODULE}.py
+ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${DIRECTORY})
+
+ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${DIRECTORY}/${MODULE}.py
+ ${CMAKE_CURRENT_BINARY_DIR}/${DIRECTORY}/${MODULE}.pyc
+ ${CMAKE_CURRENT_BINARY_DIR}/${DIRECTORY}/${MODULE}.pyo
+ DESTINATION ${PYTHON_SITE_PACKAGES}/${DIRECTORY})
+endmacro(py_module)
+
+if(WITH-QMF)
+ execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"
+ OUTPUT_VARIABLE PYTHON_SITE_PACKAGES
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+ py_module(__init__ matahari)
+ py_module(rpc_server matahari)
+
+endif(WITH-QMF)
+
diff --git a/src/python/matahari/__init__.py b/src/python/matahari/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/python/matahari/rpc_server.py b/src/python/matahari/rpc_server.py
new file mode 100644
index 0000000..6cf2eea
--- /dev/null
+++ b/src/python/matahari/rpc_server.py
@@ -0,0 +1,45 @@
+import sys, os
+import json
+import traceback
+
+(PLUGIN_PATH,) = ('.',) # TODO: choose proper install path
+
+class Plugin(object):
+ def __init__(self, module):
+ self.module = module
+ self.name = self.module.__name__
+
+ def procedures(self):
+ isProcedure = lambda a: callable(getattr(self.module, a))
+ return [o for o in dir(self.module) if isProcedure(o) and
+ not o.startswith('_')]
+
+ def call(self, procedure, args_json=[], kwargs_json=[]):
+ func = getattr(self.module, procedure)
+ args = [json.loads(a) for a in args_json]
+ kwargs = dict((k, json.loads(v)) for k,v in kwargs_json)
+ result = func(*args, **kwargs)
+ return json.dumps(result)
+
+def formatException(exc_type, exc_val, exc_tb):
+ return exc_type.__name__, str(exc_val), ''.join(traceback.format_tb(exc_tb))
+
+def plugins(path=PLUGIN_PATH):
+ old_path = sys.path
+ try:
+ sys.path = [path] + old_path
+ filenames = [os.path.splitext(f) for f in os.listdir(path)]
+ modnames = [n for n, e in filenames if e == '.py']
+ def getPlugin(name):
+ try:
+ mod = __import__(name, {}, {}, [], -1)
+ return Plugin(mod)
+ except ImportError:
+ return None
+ return filter(lambda p: p is not None, map(getPlugin, modnames))
+ finally:
+ sys.path = old_path
+
+
+if __name__ == '__main__':
+ print [p.name for p in plugins('src/rpc/plugins')]
diff --git a/src/rpc/.gitignore b/src/rpc/.gitignore
deleted file mode 100644
index 52e4e61..0000000
--- a/src/rpc/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*.pyc
-*.pyo
diff --git a/src/rpc/rpc.py b/src/rpc/rpc.py
deleted file mode 100644
index 6cf2eea..0000000
--- a/src/rpc/rpc.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import sys, os
-import json
-import traceback
-
-(PLUGIN_PATH,) = ('.',) # TODO: choose proper install path
-
-class Plugin(object):
- def __init__(self, module):
- self.module = module
- self.name = self.module.__name__
-
- def procedures(self):
- isProcedure = lambda a: callable(getattr(self.module, a))
- return [o for o in dir(self.module) if isProcedure(o) and
- not o.startswith('_')]
-
- def call(self, procedure, args_json=[], kwargs_json=[]):
- func = getattr(self.module, procedure)
- args = [json.loads(a) for a in args_json]
- kwargs = dict((k, json.loads(v)) for k,v in kwargs_json)
- result = func(*args, **kwargs)
- return json.dumps(result)
-
-def formatException(exc_type, exc_val, exc_tb):
- return exc_type.__name__, str(exc_val), ''.join(traceback.format_tb(exc_tb))
-
-def plugins(path=PLUGIN_PATH):
- old_path = sys.path
- try:
- sys.path = [path] + old_path
- filenames = [os.path.splitext(f) for f in os.listdir(path)]
- modnames = [n for n, e in filenames if e == '.py']
- def getPlugin(name):
- try:
- mod = __import__(name, {}, {}, [], -1)
- return Plugin(mod)
- except ImportError:
- return None
- return filter(lambda p: p is not None, map(getPlugin, modnames))
- finally:
- sys.path = old_path
-
-
-if __name__ == '__main__':
- print [p.name for p in plugins('src/rpc/plugins')]
12 years, 5 months
[PATCH] like-gofer-but-not-gofer agent (WIP)
by Zane Bitter
Here is a first cut at an agent to do a much-simplified version of what
Gofer does: RPC with JSON-encoded parameters to a set of Python plugins.
I'm calling it the RPC agent because that's what it does (more to the
point, that's *all* it does), but I'm extremely open to better ideas on
that front in particular.
At present, each plugin has a single "invoke" method, which is something of
a holdover from my previous Gofer plugin prototype (Gofer plugins can
contain both top-level functions and methods). In theory we can dynamically
generate the schema for each plugin and add the actual methods to it. I'm
not sure how this would handle naming conflicts among different schemata
though. It might be better to just add a method to get the list of RPC
functions and leave it at that.
This is still in progress, but now is a good time for comments.
cheers,
Zane.
---
Zane Bitter (1):
RPC: Add a prototype of the new Python RPC agent
matahari.spec.in | 56 ++++++
src/CMakeLists.txt | 2
src/include/matahari/rpc.h | 122 ++++++++++++++
src/lib/CMakeLists.txt | 6 +
src/lib/rpc-python.c | 390 ++++++++++++++++++++++++++++++++++++++++++++
src/lib/rpc.c | 197 ++++++++++++++++++++++
src/lib/rpc_private.h | 56 ++++++
src/rpc/.gitignore | 2
src/rpc/CMakeLists.txt | 20 ++
src/rpc/plugins/example.py | 12 +
src/rpc/rpc-qmf.cpp | 210 ++++++++++++++++++++++++
src/rpc/rpc.py | 43 +++++
src/rpc/schema.xml | 19 ++
13 files changed, 1134 insertions(+), 1 deletions(-)
create mode 100644 src/include/matahari/rpc.h
create mode 100644 src/lib/rpc-python.c
create mode 100644 src/lib/rpc.c
create mode 100644 src/lib/rpc_private.h
create mode 100644 src/rpc/.gitignore
create mode 100644 src/rpc/CMakeLists.txt
create mode 100644 src/rpc/plugins/example.py
create mode 100644 src/rpc/rpc-qmf.cpp
create mode 100644 src/rpc/rpc.py
create mode 100644 src/rpc/schema.xml
12 years, 5 months
[PATCH] RPC: Publish a list of commands for each Plugin
by Zane Bitter
Also check that a given procedure exists before attempting to invoke it.
---
src/include/matahari/rpc.h | 10 ++++
src/lib/rpc-python.c | 104 +++++++++++++++++++++++++++++++-------------
src/lib/rpc.c | 10 ++++
src/lib/rpc_private.h | 1
src/rpc/rpc-qmf.cpp | 25 ++++++++++-
src/rpc/rpc.py | 3 +
src/rpc/schema.xml | 1
7 files changed, 120 insertions(+), 34 deletions(-)
diff --git a/src/include/matahari/rpc.h b/src/include/matahari/rpc.h
index 15c0696..59898ed 100644
--- a/src/include/matahari/rpc.h
+++ b/src/include/matahari/rpc.h
@@ -68,6 +68,16 @@ int
mh_rpc_load_plugins(int *count, mh_rpc_plugin_t **plugins);
/**
+ * Get a list of procedures supplied by the plugin.
+ * \param plugin pointer to the plugin.
+ * \return pointer to an array of strings containing the method names. The last
+ * value in the array is NULL. Both the array and the strings it contains
+ * should be released with a call to free().
+ */
+char **
+mh_rpc_get_procedures(const mh_rpc_plugin_t *plugin);
+
+/**
* Create a Remote Method Invocation.
* This should be freed with mh_rpc_rmi_destroy() when done.
* \param pcall pointer in which to store the RMI data.
diff --git a/src/lib/rpc-python.c b/src/lib/rpc-python.c
index b9dac7d..c54ee1d 100644
--- a/src/lib/rpc-python.c
+++ b/src/lib/rpc-python.c
@@ -200,6 +200,78 @@ cleanup:
}
/**
+ * Convert the supplied Python string to a newly-allocated C string. This
+ * string must later be released with a call to free().
+ *
+ * This function consumes a reference to the input Python string.
+ *
+ * \param object the Python string object.
+ * \return pointer to a new C string containing the Python object's string.
+ */
+static char *
+py_to_c_string(PyObject *object)
+{
+ char *str;
+
+ if (!object) {
+ return NULL;
+ }
+
+ str = PyString_AsString(object);
+ if (!str) {
+ mh_err("Error getting string");
+ PyErr_Print();
+ Py_DECREF(object);
+ return NULL;
+ }
+ str = strdup(str);
+
+ Py_DECREF(object);
+ return str;
+}
+
+char **get_procedures(const struct mh_rpc_plugin_impl *plugin)
+{
+ PyObject *proclist;
+ Py_ssize_t len;
+ size_t i;
+ char **procs = NULL;
+
+ proclist = PyObject_CallMethod(plugin->plugin, "procedures", NULL);
+ if (!proclist) {
+ mh_err("Failed to get procedure list");
+ PyErr_Print();
+ return NULL;
+ }
+
+ len = PySequence_Length(proclist);
+ if (len < 0) {
+ mh_err("Failed to get length of procedure list");
+ Py_DECREF(proclist);
+ return NULL;
+ }
+
+ procs = malloc(sizeof(char*) * (len + 1));
+ if (!procs) {
+ mh_err("Out of memory");
+ return NULL;
+ }
+
+ for (i = 0; i < len; i++) {
+ procs[i] = py_to_c_string(PySequence_GetItem(proclist, i));
+ if (!procs[i]) {
+ break;
+ }
+ mh_debug("Found procedure \"%s\"", procs[i]);
+ }
+ procs[len] = NULL;
+
+ Py_DECREF(proclist);
+
+ return procs;
+}
+
+/**
* Return a Python string for an argument value.
*
* This function creates a new reference to the returned object.
@@ -256,37 +328,6 @@ get_arg_list(const struct rpc_arg_list *args,
}
/**
- * Convert the supplied Python string to a newly-allocated C string. This
- * string must later be released with a call to free().
- *
- * This function consumes a reference to the input Python string.
- *
- * \param object the Python string object.
- * \return pointer to a new C string containing the Python object's string.
- */
-static char *
-py_to_c_string(PyObject *object)
-{
- char *str;
-
- if (!object) {
- return NULL;
- }
-
- str = PyString_AsString(object);
- if (!str) {
- mh_err("Error getting string");
- PyErr_Print();
- Py_DECREF(object);
- return NULL;
- }
- str = strdup(str);
-
- Py_DECREF(object);
- return str;
-}
-
-/**
* Fetch the latest Python exception and format it as strings suitable for
* returning in the RPC output.
* \param output the RPC result output.
@@ -386,5 +427,6 @@ struct mh_rpc_api mh_rpc_python_api = {
init,
deinit,
load_plugins,
+ get_procedures,
invoke,
};
diff --git a/src/lib/rpc.c b/src/lib/rpc.c
index f7add33..3dde2aa 100644
--- a/src/lib/rpc.c
+++ b/src/lib/rpc.c
@@ -73,6 +73,16 @@ mh_rpc_load_plugins(int *count, mh_rpc_plugin_t **plugins)
return 0;
}
+char **
+mh_rpc_get_procedures(const mh_rpc_plugin_t *plugin)
+{
+ if (!API_DEFINED(plugin->type)) {
+ return NULL;
+ }
+
+ return apis[plugin->type]->get_procedures(plugin->impl);
+}
+
int
mh_rpc_call_create(mh_rpc_t *pcall,
const char *procedure)
diff --git a/src/lib/rpc_private.h b/src/lib/rpc_private.h
index 6321ed9..81d9c08 100644
--- a/src/lib/rpc_private.h
+++ b/src/lib/rpc_private.h
@@ -30,6 +30,7 @@ struct mh_rpc_api {
void (*init)(int, char**);
void (*deinit)(void);
int (*load_plugins)(int*, mh_rpc_plugin_t**);
+ char **(*get_procedures)(const struct mh_rpc_plugin_impl*);
enum mh_result (*invoke)(const struct mh_rpc_plugin_impl*,
const struct mh_rpc*,
mh_rpc_result_t *output);
diff --git a/src/rpc/rpc-qmf.cpp b/src/rpc/rpc-qmf.cpp
index e61a466..a0936a6 100644
--- a/src/rpc/rpc-qmf.cpp
+++ b/src/rpc/rpc-qmf.cpp
@@ -45,9 +45,12 @@ public:
bool invoke(qmf::AgentSession& session, qmf::AgentEvent& event);
private:
+ typedef std::set<std::string> CommandList;
+
RPCAgent& _agent;
std::string _name;
mh_rpc_plugin_t _plugin;
+ CommandList _commands;
};
@@ -136,7 +139,20 @@ RPCPlugin::RPCPlugin(RPCAgent& agent, mh_rpc_plugin_t& plugin):
{
mh_info("Creating Plugin %s", _name.c_str());
+ char **procs = mh_rpc_get_procedures(&_plugin);
+ qpid::types::Variant::List cmd_list;
+ if (procs) {
+ for (char **p = procs; *p; p++) {
+ _commands.insert(*p);
+ cmd_list.push_back(qpid::types::Variant(*p));
+ free(*p);
+ }
+ free(procs);
+ }
+
_instance.setProperty("name", _name);
+ _instance.setProperty("commands", cmd_list);
+
*_agent._objectManager += this;
}
@@ -157,8 +173,13 @@ RPCPlugin::invoke(qmf::AgentSession& session, qmf::AgentEvent& event)
if (methodName == "invoke") {
mh_rpc_t call;
- mh_rpc_call_create(&call,
- args["procedure"].asString().c_str());
+ std::string proc(args["procedure"].asString());
+
+ if (_commands.find(proc) == _commands.end()) {
+ session.raiseException(event, mh_result_to_str(MH_RES_INVALID_ARGS));
+ }
+
+ mh_rpc_call_create(&call, proc.c_str());
qpid::types::Variant::List arglist(args["args"].asList());
for (qpid::types::Variant::List::iterator iter = arglist.begin();
diff --git a/src/rpc/rpc.py b/src/rpc/rpc.py
index 9fb56df..7269bdc 100644
--- a/src/rpc/rpc.py
+++ b/src/rpc/rpc.py
@@ -10,7 +10,8 @@ class Plugin(object):
self.name = self.module.__name__
def procedures(self):
- return [p.__name__ for p in self.module if callable(p)]
+ isProcedure = lambda a: callable(getattr(self.module, a))
+ return [o for o in dir(self.module) if isProcedure(o)]
def call(self, procedure, args_json=[], kwargs_json=[]):
func = getattr(self.module, procedure)
diff --git a/src/rpc/schema.xml b/src/rpc/schema.xml
index e8ddcd4..a5155ce 100644
--- a/src/rpc/schema.xml
+++ b/src/rpc/schema.xml
@@ -2,6 +2,7 @@
<class name="Plugin">
<property name="name" type="sstr" access="RO" />
+ <property name="commands" type="list" access="RO" />
<method name="invoke" desc="Make a remote procedure call">
<arg name="procedure" dir="I" type="sstr" />
12 years, 5 months
DBus-QMF bridge
by Radek Novacek
Hello,
as I promised on the meeting yesterday, I have pushed what I have on DBus
bridge to github. You can see the changeset here:
https://github.com/rnovacek/matahari/compare/master...dbus-bridge
It has now only one method "call", which will call any DBus method on system
bus, for given bus_name, object_path and interface. Argument "args" is list of
input arguments, that will be converted to arguments to DBus method. The
"results" is list with output arguments.
<method name="call">
<arg name="bus_name" type="sstr" dir="I" />
<arg name="object_path" type="sstr" dir="I" />
<arg name="interface" type="sstr" dir="I" />
<arg name="method_name" type="sstr" dir="I" />
<arg name="args" type="list" dir="I" />
<arg name="results" type="list" dir="O" />
</method>
There are also few tests to exercise various types of input and output
arguments.
The code is still work in progress, but any comments are welcomed.
Radek Novacek
12 years, 5 months
Gofer integration update
by Zane Bitter
Y'all will have noticed that I posted a patch a couple of days ago with
a prototype of a Gofer agent for Matahari. As it stands, here is a rough
outline of what the patch does:
* Imports existing Gofer plugins and creates a QMF object (of class
org.matahariproject.gofer.Plugin) for each.
* Accepts RPC calls in the form of QMF calls to the "invoke" method of
the plugin object.
* Invokes RPC calls on the specified class and method in the plugin,
passing positional and keyword arguments which are supplied as a list
and map (respectively) with the parameter values in JSON format.
* Returns the result simply as a JSON string.
And what it does *not* do:
* Pass any out-of-band data with the RPC call - e.g. maintenance
windows, class constructors, shared secrets. In principle, all of these
could be added as parameters to the QMF call
* Distinguish between normal return values and exceptions - it is
currently up to the client to deduce this from the JSON string.
* Provide any way for a client to call methods RPC-style (i.e. by just
calling them, rather than going through the QMF API).
For a number of reasons I have put this functionality in a separate
agent rather than adding it to, say, the host or sysconfig agents. In
general we want to avoid a proliferation of agents, but the following
are valid reasons for creating a separate agent instead of adding
functionality to an existing one:
* The SELinux permissions required can be narrowed by putting the
functionality in a separate agent.
* The user could conceivably want the functionality executed with a
different user id.
* The user could conceivably want to enable or disable the functionality
independently of the functionality of existing agents.
All but the first apply to the Gofer agent.
Now that I am somewhat familiar with the internals of how Gofer works,
and thanks to the discussion with Jeff that has been happening here on
the mailing list, it seems that there are essentially two paths we could
go down to fulfil the requirements.
To recap: the requirement is that we provide a way for sysadmins/devops
to easily write custom APIs in a high-level language that integrates
with their existing code (i.e. Ruby & Python), and expose them over
Matahari using the same infrastructure as other Matahari agents. This
means only one lot of brokers, configuration, certificates, &c. to deal
with.
The big unknown is around whether we need to provide to clients an
RPC-style API (i.e. where calling remote code looks not much different
to calling local code) as well as a native Matahari API. This is
fundamentally what Gofer is about (in Python) and also what MCollective
does (in Ruby - http://docs.puppetlabs.com/mcollective/simplerpc/). On
the other hand, I'm not sure if this is really needed, or if it even
makes sense to talk about it in an environment where the client can be
written in one of several languages (Gofer supports both Python and Ruby
client interfaces to the Python plugins).
Option 1: Continue with the Gofer port
--------------------------------------
Essentially, attempt to get all of Gofer working over Matahari. Existing
Gofer plugins would continue to work; existing clients would need only
to switch over to the new backend (which should be as simple as "from
matahari import gofer" at the top of the file.
Work required:
* Parse the method output JSON and either return ret_val or raise a QMF
exception with the exception data.
* Implement Gofer-specific code in the Matahari Python API to simplify
the interface.
* Implement a client-side library to dispatch RMI calls to the
QMF/Matahari backend. (Needs to be done for both Python and Ruby.)
* Add authorisation parameter to invoke method.
* Add support for delayed/recurring actions.
* Add support for class constructors.
Advantages:
* Compatibility with existing Gofer plugins and clients.
* Reuses existing Gofer configuration &c. (already defined location for
plugins, config files, config format...)
Disadvantages:
* QMF API becomes complex
* We may have to do a lot of work to support features we are not
necessarily that concerned about.
Option 2: Start from scratch
----------------------------
Forget about Gofer and implement the simplest possible agent that will
allow us to call out to Python (and later Ruby) code. Basically what
Jeff suggested in his last email.
Work required:
* Define a plugin format, location, settings, SELinux config &c.
* Implement plugin loading and method call dispatching.
* Implement Gofer-specific code in the Matahari Python API to simplify
the interface.
Advantages:
* We only have to support the features we need and that make sense with
a QMF backend.
* Possibly a simpler interface to also write a Ruby agent against?
Disadvantages:
* No RPC API initially
* If we were to end up implementing a lot of Gofer's features, we would
have reinvented the wheel.
* No support for existing(/future) Gofer plugins/clients
Before writing this email I had no real clue which path to pursue. After
writing it, I am leaning toward option 2. Here are the issues I think we
need to consider:
Questions:
* Do we need an RPC API?
* How important is it that we maintain compatibility with existing Gofer
plugins?
* Do we need method-level authorisation using shared secrets, or could
we effect authorisation by other means? I believe we should have some
kind of authorisation at at least the per-plugin level. (What is our
plan for authorisation with existing Matahari agents today?)
* Are delayed/repeated actions important, or should we leave this to the
client?
* Do we need to (eventually) support Ruby plugins?
This concludes the most epic email in history. Thanks for reading.
regards,
Zane.
12 years, 5 months