[lnst] NetTestCommand, NetTestController, NetTestSlave: log changes
by Jiří Pírko
commit 63cf30fb5e524b35e500780f8dc3cf8d23bf1646
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Fri Mar 8 16:24:42 2013 +0100
NetTestCommand, NetTestController, NetTestSlave: log changes
This commit removes the old Logs code. The class Logs will be replaced
with an instance of class LoggingCtl so I also prepared the class
constructors to recieve it as a parameter.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
lnst/Common/NetTestCommand.py | 23 +++++------------------
lnst/Controller/NetTestController.py | 23 ++++++++---------------
lnst/Slave/NetTestSlave.py | 20 +++++++-------------
3 files changed, 20 insertions(+), 46 deletions(-)
---
diff --git a/lnst/Common/NetTestCommand.py b/lnst/Common/NetTestCommand.py
index c8c807d..49feeae 100644
--- a/lnst/Common/NetTestCommand.py
+++ b/lnst/Common/NetTestCommand.py
@@ -16,7 +16,6 @@ import sys
import signal
import imp
import pickle, traceback
-from lnst.Common.Logs import Logs
from lnst.Common.ExecCmd import exec_cmd, ExecCmdFail
def str_command(command):
@@ -47,12 +46,13 @@ class BgCommandException(Exception):
return "BgCommandError: " + self._str
class BgCommand:
- def __init__(self, bg_id, cmd_cls):
+ def __init__(self, bg_id, cmd_cls, log_ctl):
self._bg_id = bg_id
self._cmd_cls = cmd_cls
self._pid = None
self._read_pipe = None
self._killed = False
+ self._log_ctl = log_ctl
def get_bg_id(self):
return self._bg_id
@@ -66,7 +66,6 @@ class BgCommand:
" id \"%s\", pid \"%d\"" % (self._bg_id, self._pid))
self._read_pipe = read_pipe
return {"passed": True}
- Logs.get_buffer().flush()
os.close(read_pipe)
os.setpgrp()
self._cmd_cls.set_handle_intr()
@@ -78,8 +77,6 @@ class BgCommand:
except:
type, value, tb = sys.exc_info()
result = {"Exception": ''.join(traceback.format_exception(type, value, tb))}
- buf = Logs.get_buffer()
- result["logs"] = buf.flush()
tmp = pickle.dumps(result)
os.write(write_pipe, tmp)
os.close(write_pipe)
@@ -267,10 +264,6 @@ class NetTestCommandWait(NetTestCommandControl):
bg_cmd.wait_for()
result = bg_cmd.get_result()
self._command_context.del_bg_cmd(bg_cmd)
- buf = Logs.get_buffer()
- logs = result["logs"]
- buf.add_buffer(logs)
- del result["logs"]
self.set_result(result)
class NetTestCommandIntr(NetTestCommandControl):
@@ -280,10 +273,6 @@ class NetTestCommandIntr(NetTestCommandControl):
bg_cmd.interrupt()
result = bg_cmd.get_result()
self._command_context.del_bg_cmd(bg_cmd)
- buf = Logs.get_buffer()
- logs = result["logs"]
- buf.add_buffer(logs)
- del result["logs"]
self.set_result(result)
class NetTestCommandKill(NetTestCommandControl):
@@ -293,9 +282,6 @@ class NetTestCommandKill(NetTestCommandControl):
bg_cmd.kill()
result = bg_cmd.get_result()
self._command_context.del_bg_cmd(bg_cmd)
- buf = Logs.get_buffer()
- logs = result["logs"]
- buf.add_buffer(logs)
self.set_result({"passed": True})
def get_command_class(command_context, command, resource_table):
@@ -320,7 +306,8 @@ def get_command_class(command_context, command, resource_table):
return cmd_cls
class NetTestCommand:
- def __init__(self, command_context, command, resource_table):
+ def __init__(self, command_context, command, resource_table, log_ctl):
+ self._log_ctl = log_ctl
self._command_class = get_command_class(command_context, command,
resource_table)
self._command_context = command_context
@@ -330,7 +317,7 @@ class NetTestCommand:
cmd_cls = self._command_class
if "bg_id" in self._command:
bg_id = self._command["bg_id"]
- bg_cmd = BgCommand(bg_id, cmd_cls)
+ bg_cmd = BgCommand(bg_id, cmd_cls, self._log_ctl)
self._command_context.add_bg_cmd(bg_cmd)
return bg_cmd.run()
else:
diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py
index 80a132d..c034f33 100644
--- a/lnst/Controller/NetTestController.py
+++ b/lnst/Controller/NetTestController.py
@@ -20,7 +20,7 @@ import tempfile
from time import sleep
from xmlrpclib import Binary
from pprint import pprint, pformat
-from lnst.Common.Logs import Logs, log_exc_traceback
+from lnst.Common.Logs import log_exc_traceback
from lnst.Common.XmlRpc import ServerProxy, ServerException
from lnst.Common.NetUtils import MacPool
from lnst.Common.VirtUtils import VirtNetCtl, VirtDomainCtl, BridgeCtl
@@ -38,13 +38,13 @@ def ignore_event(**kwarg):
pass
class NetTestController:
- def __init__(self, recipe_path, cleanup=False,
+ def __init__(self, recipe_path, log_ctl, cleanup=False,
res_serializer=None, config=None):
self._docleanup = cleanup
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._config = config
- self._log_root_path = Logs.get_logging_root_path()
+ self._log_ctl = log_ctl
self._recipe_path = recipe_path
sp = SlavePool(config.get_option('environment', 'pool_dirs'),
@@ -323,18 +323,7 @@ class NetTestController:
info = self._get_machineinfo(machine_id)
address = socket.gethostbyname(info["hostname"])
- slave_root_path = os.path.join(self._log_root_path, address)
- try:
- os.mkdir(slave_root_path)
- except OSError as e:
- if e.errno != 17:
- raise
-
- logger = logging.getLogger(address)
- Logs(Logs.debug, False, logger, log_folder=slave_root_path,
- to_display=False, date="")
-
- info['logger'] = logger
+ info['logger'] = self._log_ctl.add_slave(address)
def _deconfigure_slaves(self):
if 'machines' not in self._recipe:
@@ -358,6 +347,10 @@ class NetTestController:
domain_ctl = info["virt_domain_ctl"]
domain_ctl.detach_interface(dev["hwaddr"])
+ #clean-up slave logger
+ address = socket.gethostbyname(info["hostname"])
+ self._log_ctl.remove_slave(address)
+
# remove dynamically created bridges
networks = self._recipe["networks"]
for net in networks.itervalues():
diff --git a/lnst/Slave/NetTestSlave.py b/lnst/Slave/NetTestSlave.py
index 229aabf..bdbf4b7 100644
--- a/lnst/Slave/NetTestSlave.py
+++ b/lnst/Slave/NetTestSlave.py
@@ -18,7 +18,7 @@ from time import sleep
from xmlrpclib import Binary
from tempfile import NamedTemporaryFile
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
-from lnst.Common.Logs import Logs, log_exc_traceback
+from lnst.Common.Logs import log_exc_traceback
from lnst.Common.PacketCapture import PacketCapture
from lnst.Common.XmlRpc import Server
from lnst.Common.Utils import die_when_parent_die
@@ -37,12 +37,13 @@ class NetTestSlaveXMLRPC:
'''
Exported xmlrpc methods
'''
- def __init__(self, command_context, config):
+ def __init__(self, command_context, config, log_ctl):
self._netconfig = None
self._packet_captures = {}
self._netconfig = NetConfig()
self._command_context = command_context
self._config = config
+ self._log_ctl = log_ctl
self._capture_files = {}
self._copy_targets = {}
@@ -58,10 +59,8 @@ class NetTestSlaveXMLRPC:
self._cache.del_old_entries()
self.reset_file_transfers()
- log_dir = self._config.get_option('environment', 'log_dir')
- recipe_name = os.path.splitext(os.path.split(recipe_path)[1])[0]
- Logs.relocate_log_folder(date=None, log_folder=log_dir,
- nameExtend=recipe_name)
+ date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+ self._log_ctl.set_recipe(recipe_path, expand=date)
sleep(1)
if check_process_running("NetworkManager"):
@@ -78,11 +77,6 @@ class NetTestSlaveXMLRPC:
self._remove_capture_files()
return "bye"
- def get_new_logs(self):
- buffer = Logs.get_buffer()
- logs = buffer.flush()
- return logs
-
def get_devices_by_hwaddr(self, hwaddr):
name_scan = scan_netdevs()
netdevs = []
@@ -164,7 +158,7 @@ class NetTestSlaveXMLRPC:
def run_command(self, command):
try:
cmd_cls = NetTestCommand(self._command_context, command,
- self._resource_table)
+ self._resource_table, self._log_ctl)
return cmd_cls.run()
except:
log_exc_traceback()
@@ -290,7 +284,7 @@ class MySimpleXMLRPCServer(Server):
pass
class NetTestSlave:
- def __init__(self, config, port = DefaultRPCPort):
+ def __init__(self, config, log_ctl, port = DefaultRPCPort):
die_when_parent_die()
command_context = NetTestCommandContext()
10 years, 9 months
[lnst] lnst-{ctl, slave}: changes to reflect the Logs change
by Jiří Pírko
commit 0e0900ea014ce48f7fad1a4c8f84e2443526c62b
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Fri Mar 8 16:24:41 2013 +0100
lnst-{ctl, slave}: changes to reflect the Logs change
This commit updates the lnst-ctl and lnst-slave applications to reflect
the change in how we use logging. The created LoggingCtl instance will
be propagated to other modules as a parameter.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
lnst-ctl | 29 ++++++++++++++---------------
lnst-slave | 9 +++++----
2 files changed, 19 insertions(+), 19 deletions(-)
---
diff --git a/lnst-ctl b/lnst-ctl
index c15e9af..17ce6cd 100755
--- a/lnst-ctl
+++ b/lnst-ctl
@@ -16,7 +16,8 @@ import sys
import logging
import os
import re
-from lnst.Common.Logs import Logs
+import datetime
+from lnst.Common.Logs import LoggingCtl
from lnst.Common.Config import Config
from lnst.Controller.NetTestController import NetTestController, NetTestError
from lnst.Controller.NetTestResultSerializer import NetTestResultSerializer
@@ -41,8 +42,8 @@ def usage():
sys.exit()
def process_recipe(action, file_path, cleanup, res_serializer,
- packet_capture, config):
- nettestctl = NetTestController(file_path, cleanup=cleanup,
+ packet_capture, config, log_ctl):
+ nettestctl = NetTestController(file_path, log_ctl, cleanup=cleanup,
res_serializer=res_serializer, config=config)
if action == "run":
return nettestctl.run_recipe(packet_capture)
@@ -65,14 +66,14 @@ def print_summary(summary):
logging.info("=====================================================")
def get_recipe_result(args, file_path, cleanup,
- res_serializer, packet_capture, config):
+ res_serializer, packet_capture, config, log_ctl):
res_serializer.add_recipe(file_path)
- Logs.set_logging_root_path(file_path)
+ log_ctl.set_recipe(file_path)
res = None
try:
res = process_recipe(args, file_path, cleanup,
- res_serializer, packet_capture, config)
+ res_serializer, packet_capture, config, log_ctl)
except NetTestError as err:
logging.error(err)
@@ -130,7 +131,10 @@ def main():
packet_capture = True
- Logs(debug, log_folder=config.get_option('environment', 'log_dir'))
+ date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+ log_ctl = LoggingCtl(debug,
+ log_dir=config.get_option('environment', 'log_dir'),
+ log_subdir=date)
if not args:
logging.error("No action command passed")
@@ -141,7 +145,6 @@ def main():
summary = []
res_serializer = NetTestResultSerializer()
- Logs.save_state()
for recipe_path in args:
if os.path.isdir(recipe_path):
all_files = []
@@ -153,23 +156,19 @@ def main():
for f in all_files:
recipe_file = os.path.join(recipe_path, f)
if re.match(r'^.*\.xml$', recipe_file):
- Logs.reset_state()
logging.info("Processing recipe file \"%s\"" % recipe_file)
summary.append(get_recipe_result(action, recipe_file,
cleanup,
res_serializer,
packet_capture,
- config))
- Logs.set_logging_root_path(clean=False)
+ config, log_ctl))
else:
- Logs.reset_state()
summary.append(get_recipe_result(action, recipe_path,
cleanup, res_serializer,
packet_capture,
- config))
+ config, log_ctl))
- Logs.reset_state()
- Logs.set_logging_root_path(clean=False)
+ log_ctl.set_recipe("", clean=False)
print_summary(summary)
diff --git a/lnst-slave b/lnst-slave
index 2181e42..91f257c 100755
--- a/lnst-slave
+++ b/lnst-slave
@@ -16,7 +16,7 @@ import sys
import os
import logging
from lnst.Common.Daemon import Daemon
-from lnst.Common.Logs import Logs
+from lnst.Common.Logs import LoggingCtl
from lnst.Common.Config import Config
from lnst.Slave.NetTestSlave import NetTestSlave
@@ -72,13 +72,14 @@ def main():
elif opt in ("-p", "--port"):
port = int(arg)
- Logs(debug, True, log_folder=config.get_option('environment', 'log_dir'))
+ log_ctl = LoggingCtl(debug,
+ log_dir=config.get_option('environment', 'log_dir'))
logging.info("Started")
if port:
- nettestslave = NetTestSlave(config, port=port)
+ nettestslave = NetTestSlave(config, log_ctl, port=port)
else:
- nettestslave = NetTestSlave(config)
+ nettestslave = NetTestSlave(config, log_ctl)
if daemon:
daemon = Daemon(pidfile)
10 years, 9 months
[lnst] Logs: add class LoggingCtl
by Jiří Pírko
commit a80a2a63d6af074c9c016d5d4a31e81bca052a12
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Fri Mar 8 16:24:40 2013 +0100
Logs: add class LoggingCtl
This commit adds a new class LoggingCtl. This class replaces the
functionality of the old static class Logs. This class however is not
static. Instead we will create one instance of this class that will
carry all the important state information needed to properly manipulate
with the logging module.
The class implements a simple interface that reflects our needs when
running test recipes:
* {set,unset}_recipe- when we loop over a list of specified recipes
* {add,remove}_slave- when connecting to new slaves
* {set,unset}_connection- for the slave application to redirect where to
send log records
When used correctly the class will output logs on the display and can
create a directory structure like this:
<log_folder>/ (for summary of recipe results)
<log_folder>/<recipe>/ (for all the recipe specific logs)
<log_folder>/<recipe>/<slave>/ (for slave specific logs)
This patch also changes the output format of log messages to what we
have agreed on.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
lnst/Common/Logs.py | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 125 insertions(+), 0 deletions(-)
---
diff --git a/lnst/Common/Logs.py b/lnst/Common/Logs.py
index 15a11a6..f00082d 100644
--- a/lnst/Common/Logs.py
+++ b/lnst/Common/Logs.py
@@ -74,3 +74,128 @@ class MultilineFormater(Formatter):
# See issue 8924
s = s + record.exc_text.decode(sys.getfilesystemencoding())
return s
+
+class LoggingCtl:
+ log_folder = ""
+ formatter = None
+ recipe_handlers = (None,None)
+ recipe_log_path = ""
+ slaves = {}
+ transmit_handler = None
+
+ def __init__(self, debug=False, log_dir=None, log_subdir=""):
+ if log_dir != None:
+ self.log_folder = os.path.abspath(os.path.join(log_dir, log_subdir))
+ else:
+ self.log_folder = os.path.abspath(os.path.join(
+ os.path.dirname(sys.argv[0]),
+ './Logs',
+ log_subdir))
+ if not os.path.isdir(self.log_folder):
+ self._clean_folder(self.log_folder)
+
+
+ self.formatter = MultilineFormater(
+ '%(asctime)s| %(address)17.17s| %(levelname)5.5s: '
+ '%(message)s', '%Y-%m-%d %H:%M:%S', " "*4)
+
+
+ #the display_handler will display logs in the terminal
+ display_handler = logging.StreamHandler(sys.stdout)
+ display_handler.setFormatter(self.formatter)
+ if not debug:
+ display_handler.setLevel(logging.INFO)
+ else:
+ if debug == 1:
+ display_handler.setLevel(logging.DEBUG)
+ else:
+ display_handler.setLevel(logging.NOTSET)
+
+ logger = logging.getLogger()
+ logger.setLevel(logging.NOTSET)
+ logger.addHandler(display_handler)
+
+ def set_recipe(self, recipe_path, clean=True, expand=""):
+ recipe_name = os.path.splitext(os.path.split(recipe_path)[1])[0]
+ if expand != "":
+ recipe_name += "_" + expand
+ self.recipe_log_path = os.path.join(self.log_folder, recipe_name)
+ if clean:
+ self._clean_folder(self.recipe_log_path)
+
+ (recipe_info, recipe_debug) = self._create_file_handler(
+ self.recipe_log_path)
+ logger = logging.getLogger()
+ #remove handlers of the previous recipe
+ logger.removeHandler(self.recipe_handlers[0])
+ logger.removeHandler(self.recipe_handlers[1])
+
+ self.recipe_handlers = (recipe_info, recipe_debug)
+ logger.addHandler(recipe_info)
+ logger.addHandler(recipe_debug)
+
+ def unset_recipe(self):
+ logger = logging.getLogger()
+ logger.removeHandler(self.recipe_handlers[0])
+ logger.removeHandler(self.recipe_handlers[1])
+ self.recipe_handlers = (None, None)
+
+ def add_slave(self, name):
+ slave_log_path = os.path.join(self.recipe_log_path, name)
+ self._clean_folder(slave_log_path)
+
+ logger = logging.getLogger(name)
+ logger.setLevel(logging.DEBUG)
+ logger.propagate = True
+
+ (slave_info, slave_debug) = self._create_file_handler(slave_log_path)
+ logger.addHandler(slave_info)
+ logger.addHandler(slave_debug)
+
+ self.slaves[name] = (slave_info, slave_debug)
+ return logger
+
+ def remove_slave(self, name):
+ logger = logging.getLogger(name)
+ logger.propagate = False
+
+ logger.removeHandler(self.slaves[name][0])
+ logger.removeHandler(self.slaves[name][1])
+
+ del slaves[name]
+
+ def set_connection(self, target):
+ if self.transmit_handler != None:
+ self.cancel_connection()
+ self.transmit_handler = TransmitHandler(target)
+
+ logger = logging.getLogger()
+ logger.addHandler(self.transmit_handler)
+
+ for k in self.slaves.keys():
+ self.remove_slave(k)
+
+ def cancel_connection(self):
+ if self.transmit_handler != None:
+ logger = logging.getLogger()
+ logger.removeHandler(self.transmit_handler)
+ del self.transmit_handler
+
+ def _clean_folder(self, path):
+ try:
+ shutil.rmtree(path)
+ except OSError as e:
+ if e.errno != 2:
+ raise
+ os.makedirs(path)
+
+ def _create_file_handler(self, folder_path):
+ file_debug = logging.FileHandler(os.path.join(folder_path, 'debug'))
+ file_debug.setFormatter(self.formatter)
+ file_debug.setLevel(logging.DEBUG)
+
+ file_info = logging.FileHandler(os.path.join(folder_path, 'info'))
+ file_info.setFormatter(self.formatter)
+ file_info.setLevel(logging.INFO)
+
+ return (file_debug, file_info)
10 years, 9 months
[PATCH 00/10] Logs and Communication patchset
by Ondrej Lichtner
From: Ondrej Lichtner <olichtne(a)redhat.com>
The following patchset reimplements both the Logging subsystem and the
communication protocol used between controller and slaves.
Our new Logging subsystem is now using the class LoggingCtl that provides a
simpler interface than the previous implementation and also it's not a static
class anymore.
The communication protocol is also very simple, it works by sending dictionaries
in pickled form through TCP sockets. Both the controller and the slave are
capable of communicating with more end-points via the use of select, which
allows us to transport messages even if there are "blocking" operations
running. The protocol is not fully complete yet but this is a good basis
that implements everything necessary for it to work. In the future I will
add the ability to transfer exceptions and control messages for the protocol.
Ondrej Lichtner (10):
add module ConnectionHandler
LoggingHandler: add TransmitHandler
Logs: code removal
Logs: add class LoggingCtl
lnst-{ctl, slave}: changes to reflect the Logs change
NetTestCommand, NetTestController, NetTestSlave: log changes
NetTestSlave: remove current Slave communication implementation
NetTestCommand: add pipe, set logs
NetTestSlave: new Slave implementation
NetTestController: reimplement communication
lnst-ctl | 29 ++-
lnst-slave | 9 +-
lnst/Common/ConnectionHandler.py | 116 ++++++++++++
lnst/Common/LoggingHandler.py | 19 ++
lnst/Common/Logs.py | 329 ++++++++++++-----------------------
lnst/Common/NetTestCommand.py | 45 ++---
lnst/Controller/NetTestController.py | 127 ++++++++------
lnst/Slave/NetTestSlave.py | 170 +++++++++++++-----
8 files changed, 487 insertions(+), 357 deletions(-)
create mode 100644 lnst/Common/ConnectionHandler.py
--
1.7.11.7
10 years, 9 months
[lnst] Logs: code removal
by Jiří Pírko
commit 748093a15a7e221c0210b43fa17b87e58e29578a
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Fri Mar 8 16:24:39 2013 +0100
Logs: code removal
This commit removes old deprecated code, in preparation for the
reimplementation of how we work with the logging module.
I completely removed the class stdLogger because its sole purpose was to
introduce a side-effect when importing the file. We decided that we
don't want this functionality or any side-effects anymore.
I also removed the static class Logs because the code was too complex to
use, understand and maintain. The reimplementation is focusing on this
class, the new implementation will be present in the following commit.
Finally one smaller change- I removed the global constant LOCAL_IP and
placed its value into a variable of the MultilineFormatter class as this
was the only place it was ever used and I failed to see a reason to keep
it as a global constant.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
lnst/Common/Logs.py | 240 +--------------------------------------------------
1 files changed, 2 insertions(+), 238 deletions(-)
---
diff --git a/lnst/Common/Logs.py b/lnst/Common/Logs.py
index 4932c85..15a11a6 100644
--- a/lnst/Common/Logs.py
+++ b/lnst/Common/Logs.py
@@ -14,8 +14,7 @@ from logging import Formatter
import logging.handlers
import traceback
from lnst.Common.LoggingHandler import LogBuffer
-
-LOCAL_IP = "(127.0.0.1)"
+from lnst.Common.LoggingHandler import TransmitHandler
def log_exc_traceback():
cmd_type, value, tb = sys.exc_info()
@@ -49,7 +48,7 @@ class MultilineFormater(Formatter):
"""
record.message = record.getMessage()
if not "address" in record.__dict__:
- record.address = LOCAL_IP
+ record.address = "(127.0.0.1)"
if self._fmt.find("%(asctime)") >= 0:
record.asctime = self.formatTime(record, self.datefmt)
lines = record.__dict__["message"].split("\n")
@@ -75,238 +74,3 @@ class MultilineFormater(Formatter):
# See issue 8924
s = s + record.exc_text.decode(sys.getfilesystemencoding())
return s
-
-
-class stdLogger(logging.Logger):
- def __init__(self, name, level=logging.NOTSET):
- logging.Logger.__init__(self, name, level)
-
- def findCaller(self):
- """
- Find the stack frame of the caller so that we can note the source
- file name, line number and function name.
- """
- rv = "stdio", 0, "(unknown function)"
- return rv
-
-logging._acquireLock()
-try:
- logging.setLoggerClass(stdLogger)
- logging.getLogger("root.stdLogger")
- logging.setLoggerClass(logging.Logger)
-finally:
- logging._releaseLock()
-
-
-class LoggingFile(object):
- """
- File-like object that will receive messages pass them to the logging
- infrastructure in an appropriate way.
- """
- def __init__(self, prefix='', level=logging.DEBUG):
- """
- @param prefix - The prefix for each line logged by this object.
- """
- self._prefix = prefix
- self._level = level
- self._buffer = []
- self._stdLogger = logging.getLogger("root.stdLogger")
-
-
- def write(self, data):
- """"
- Writes data only if it constitutes a whole line. If it's not the case,
- store it in a buffer and wait until we have a complete line.
- @param data - Raw data (a string) that will be processed.
- """
- # splitlines() discards a trailing blank line, so use split() instead
- data_lines = data.split('\n')
- if len(data_lines) > 1:
- self._buffer.append(data_lines[0])
- self._flush_buffer()
- for line in data_lines[1:-1]:
- self._log_line(line)
- if data_lines[-1]:
- self._buffer.append(data_lines[-1])
-
-
- def _log_line(self, line):
- """
- Passes lines of output to the logging module.
- """
- self._stdLogger.log(self._level, self._prefix + line)
-
-
- def _flush_buffer(self):
- if self._buffer:
- self._log_line(''.join(self._buffer))
- self._buffer = []
-
-
- def flush(self):
- self._flush_buffer()
-
-
-class Logs:
- formatter = None
- logFolder = None
- loggers = []
- root_path = None
- debug = None
- date = None
- nameExtend = None
- state = None
- @classmethod
- def __init__(cls,debug=0, waitForNet=False, logger=logging.getLogger(),
- recipe_path=None, to_display=True, date=None,
- nameExtend=None, log_folder=None):
- logging.addLevelName(5, "DEBUG2")
- logging.DEBUG2 = 5
- if nameExtend is None:
- nameExtend = ""
- else:
- nameExtend = "_" + nameExtend
- cls.formatter = MultilineFormater(
- '%(asctime)s| %(address)17.17s%(module)15.15s'
- ':%(lineno)4.4d| %(levelname)s: '
- '%(message)s', '%d/%m %H:%M:%S', " "*4)
- if log_folder != None:
- cls.logFolder = log_folder
- else:
- cls.logFolder = os.path.join(os.path.dirname(sys.argv[0]), './Logs')
- cls.loggers.append(logger)
- cls.debug = debug
- if date is None:
- cls.date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
- else:
- cls.date = date
- cls.nameExtend = nameExtend
- cls.root_path = cls.prepare_logging(debug, waitForNet,
- recipe_path, to_display)
-
- @classmethod
- def relocate_log_folder(cls, date=None, log_folder=None, nameExtend=None):
- root_logger = logging.getLogger()
- if log_folder != None:
- cls.logFolder = log_folder
- else:
- cls.logFolder = os.path.join(os.path.dirname(sys.argv[0]), './Logs')
-
- if date is None:
- cls.date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
- else:
- cls.date = date
-
- if nameExtend != None:
- cls.nameExtend = "_" + nameExtend
- else:
- cls.nameExtend = ""
- cls.log_root_folder = cls.set_logging_root_path(None, True)
-
- @classmethod
- def save_state(cls):
- cls.state = {"logFolder": cls.logFolder, "date": cls.date}
-
- @classmethod
- def reset_state(cls):
- cls.logFolder = cls.state["logFolder"]
- cls.date = cls.state["date"]
-
- for logger in cls.loggers:
- handlers = list(logger.handlers)
- for handler in handlers:
- if isinstance(handler, logging.FileHandler):
- handler.close()
- logger.removeHandler(handler)
-
- cls.loggers = cls.loggers[:1]
-
- @classmethod
- def clean_root_log_folder(cls, logRootPath):
- try:
- shutil.rmtree(logRootPath)
- except OSError as e:
- if e.errno != 2:
- raise
- os.makedirs(logRootPath)
-
- @classmethod
- def _create_file_handler(cls, path):
- file_debug = logging.FileHandler(os.path.join(path, 'debug'))
- file_debug.setFormatter(cls.formatter)
- file_debug.setLevel(logging.NOTSET)
-
- file_info = logging.FileHandler(os.path.join(path, 'info'))
- file_info.setFormatter(cls.formatter)
- file_info.setLevel(logging.INFO)
-
- return (file_debug, file_info)
-
-
- @classmethod
- def set_logging_root_path(cls, recipe_path=None, clean=True):
- """
- Change file handlers path.
- """
-
- if recipe_path is None:
- recipe_path = ""
- root_logger = cls.loggers[-1]
- recipe_name = os.path.splitext(os.path.split(recipe_path)[1])[0]
- cls.root_path = os.path.join(cls.logFolder, cls.date+cls.nameExtend,
- recipe_name)
- if (clean):
- cls.clean_root_log_folder(cls.root_path)
-
- handlers = list(root_logger.handlers)
- for handler in handlers:
- if isinstance(handler, logging.FileHandler):
- handler.close()
- root_logger.removeHandler(handler)
-
- (file_debug, file_info) = cls._create_file_handler(cls.root_path)
- root_logger.addHandler(file_debug)
- root_logger.addHandler(file_info)
- return cls.root_path
-
-
- @classmethod
- def get_logging_root_path(cls):
- return cls.root_path
-
- @classmethod
- def get_buffer(cls):
- return cls.buffer
-
- @classmethod
- def prepare_logging(cls, debug=0, waitForNet=False,
- recipe_path=None, to_display=True):
- """
- Configure logging.
-
- @param debug: If True print to terminal debug level of logging messages.
- """
- root_logger = cls.loggers[-1]
- if to_display:
- display = logging.StreamHandler()
- display.setFormatter(cls.formatter)
- if not debug:
- display.setLevel(logging.INFO)
- else:
- if debug == 1:
- display.setLevel(logging.DEBUG)
- else:
- display.setLevel(logging.NOTSET)
- root_logger.addHandler(display)
-
- if waitForNet:
- handler = LogBuffer()
- cls.buffer = handler
- root_logger.addHandler(handler)
-
- log_root_folder = cls.set_logging_root_path(recipe_path)
-
- root_logger.setLevel(logging.NOTSET)
- sys.stdout = LoggingFile(level=logging.INFO)
- sys.stderr = LoggingFile(level=logging.ERROR)
- return log_root_folder
10 years, 9 months
[lnst] LoggingHandler: add TransmitHandler
by Jiří Pírko
commit 82dff415fd168552c7f862fcafc193fe0ed07a7c
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Fri Mar 8 16:24:38 2013 +0100
LoggingHandler: add TransmitHandler
This handler will transmit the emmited logging records to a selected
target. The record transmission is via the use of the send_data
function so the target can be a socket or a Connection type object.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
lnst/Common/LoggingHandler.py | 19 +++++++++++++++++++
1 files changed, 19 insertions(+), 0 deletions(-)
---
diff --git a/lnst/Common/LoggingHandler.py b/lnst/Common/LoggingHandler.py
index cea0529..3fd8919 100644
--- a/lnst/Common/LoggingHandler.py
+++ b/lnst/Common/LoggingHandler.py
@@ -17,6 +17,7 @@ olichtne(a)redhat.com (Ondrej Lichtner)
import socket, struct, pickle
import logging
import xmlrpclib
+from lnst.Common.ConnectionHandler import send_data
class LogBuffer(logging.Handler):
"""
@@ -60,3 +61,21 @@ class LogBuffer(logging.Handler):
def close(self):
self.flush()
logging.Handler.close(self)
+
+class TransmitHandler(logging.Handler):
+ def __init__(self, target):
+ logging.Handler.__init__(self)
+ self.target = target
+
+ def emit(self, record):
+ r = dict(record.__dict__)
+ r['msg'] = record.getMessage()
+ r['args'] = None
+ r['exc_info'] = None
+
+ data = {"type": "log", "record": r}
+
+ send_data(self.target, data)
+
+ def close(self):
+ logging.Handler.close(self)
10 years, 9 months
[lnst] add module ConnectionHandler
by Jiří Pírko
commit 46d4fdd27e7004d41127bb28e7088a47ecbdefc1
Author: Ondrej Lichtner <olichtne(a)redhat.com>
Date: Fri Mar 8 16:24:37 2013 +0100
add module ConnectionHandler
This module implements the base class ConnectionHandler that is used to
store and handle multiple connections that are communicating during
recipe execution.
The module also contains the functions send_data and recv_data which are
used to deliver messages in our communication protocol.
Signed-off-by: Ondrej Lichtner <olichtne(a)redhat.com>
lnst/Common/ConnectionHandler.py | 116 ++++++++++++++++++++++++++++++++++++++
1 files changed, 116 insertions(+), 0 deletions(-)
---
diff --git a/lnst/Common/ConnectionHandler.py b/lnst/Common/ConnectionHandler.py
new file mode 100644
index 0000000..eb80055
--- /dev/null
+++ b/lnst/Common/ConnectionHandler.py
@@ -0,0 +1,116 @@
+"""
+This module defines the base class for connection handling, and helper
+functions used in our communication protocol.
+
+Copyright 2013 Red Hat, Inc.
+Licensed under the GNU General Public License, version 2 as
+published by the Free Software Foundation; see COPYING for details.
+"""
+
+__author__ = """
+olichtne(a)redhat.com (Ondrej Lichtner)
+"""
+
+import select
+import cPickle
+import socket
+from _multiprocessing import Connection
+
+def send_data(s, data):
+ pickled_data = cPickle.dumps(data)
+ length = len(pickled_data)
+
+ data_to_send = str(length) + " " + pickled_data
+
+ try:
+ if isinstance(s, socket.SocketType):
+ s.sendall(data_to_send)
+ elif isinstance(s, Connection):
+ s.send(pickled_data)
+ else:
+ return False
+ except socket.error:
+ return False
+ return True
+
+def recv_data(s):
+ if isinstance(s, socket.SocketType):
+ length = ""
+ while True:
+ c = s.recv(1)
+ if c == ' ':
+ length = int(length)
+ break
+ elif c == "":
+ return ""
+ else:
+ length += c
+ data = ""
+
+ while len(data)<length:
+ c = s.recv(length - len(data))
+ if c == "":
+ return ""
+ else:
+ data += c
+
+ data = cPickle.loads(data)
+ elif isinstance(s, Connection):
+ data = s.recv()
+ data = cPickle.loads(data)
+ else:
+ return None
+ return data
+
+
+class ConnectionHandler(object):
+ def __init__(self):
+ self._connections = {}
+
+ def check_connections(self):
+ requests = []
+ rl, wl, xl = select.select(self._connections.values(), [], [])
+ for f in rl:
+ try:
+ data = recv_data(f)
+
+ if data == "":
+ f.close()
+ self.remove_connection(f)
+ else:
+ id = self.get_connection_id(f)
+ requests.append((id, data))
+
+ except socket.error:
+ f.close()
+ self.remove_connection(f)
+ except EOFError:
+ f.close()
+ self.remove_connection(f)
+ return requests
+
+ def get_connection(self, id):
+ if id in self._connections:
+ return self._connections[id]
+ else:
+ return None
+
+ def get_connection_id(self, connection):
+ for id in self._connections:
+ if self._connections[id] == connection:
+ return id
+ return None
+
+ def add_connection(self, id, connection):
+ if id not in self._connections:
+ self._connections[id] = connection
+
+ def remove_connection(self, connection):
+ d = {}
+ for key, value in self._connections.iteritems():
+ if value != connection:
+ d[key] = value
+ self._connections = d
+
+ def clear_connections(self):
+ self._connections = {}
10 years, 9 months
[PATCH] XmlTemplates: Typo in _devname_func
by Radek Pazdera
There was a typo in the _devname_func() in the prevous patch. Missing
's' in 'machines'. This commit fixes it.
I'm sorry about that.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
---
lnst/Common/XmlTemplates.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/lnst/Common/XmlTemplates.py b/lnst/Common/XmlTemplates.py
index 218031d..26db78a 100644
--- a/lnst/Common/XmlTemplates.py
+++ b/lnst/Common/XmlTemplates.py
@@ -271,7 +271,7 @@ class XmlTemplates:
msg = "First parameter of function devname() is invalid: "\
"Machine %s does not exist." % m_id
raise XmlTemplateError(msg)
- machine = machine[m_id]
+ machine = machines[m_id]
if if_id not in machine['netconfig']:
msg = "Second parameter of function devname() is invalid: "\
--
1.7.7.6
10 years, 9 months
[lnst] XmlTemplates: Typo in _devname_func
by Jiří Pírko
commit 941362b868b4469534b3a0d573bffca44445bff2
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Fri Mar 8 15:54:44 2013 +0100
XmlTemplates: Typo in _devname_func
There was a typo in the _devname_func() in the prevous patch. Missing
's' in 'machines'. This commit fixes it.
I'm sorry about that.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
lnst/Common/XmlTemplates.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
---
diff --git a/lnst/Common/XmlTemplates.py b/lnst/Common/XmlTemplates.py
index 218031d..26db78a 100644
--- a/lnst/Common/XmlTemplates.py
+++ b/lnst/Common/XmlTemplates.py
@@ -271,7 +271,7 @@ class XmlTemplates:
msg = "First parameter of function devname() is invalid: "\
"Machine %s does not exist." % m_id
raise XmlTemplateError(msg)
- machine = machine[m_id]
+ machine = machines[m_id]
if if_id not in machine['netconfig']:
msg = "Second parameter of function devname() is invalid: "\
10 years, 9 months