From: Ondrej Lichtner olichtne@redhat.com
nettestctl now reads the users configuration file ~/.lnst/lnst.conf
As we agreed, the loaded configs will be distributed as function parameters, therefore I made the necessary changes to functions process_recipe and get_recipe_result.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- Common/Config.py | 5 ----- nettestctl.py | 16 +++++++++++----- 2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/Common/Config.py b/Common/Config.py index 8a1ad56..8ef2452 100644 --- a/Common/Config.py +++ b/Common/Config.py @@ -24,12 +24,7 @@ class Config():
def __init__(self): self._parser = ConfigParser(dict_type=dict) - - # defaults.conf should contain all possible sections and options - # sections and options not listed there will be undefined which - # can cause problems self.options = dict() - self.load_config("default.conf")
def get_config(self): return self.options diff --git a/nettestctl.py b/nettestctl.py index de07911..2fab067 100755 --- a/nettestctl.py +++ b/nettestctl.py @@ -21,6 +21,7 @@ from NetTest.NetTestResultSerializer import NetTestResultSerializer from Common.Logs import Logs from Common.LoggingServer import LoggingServer import Common.ProcessManager +from Common.Config import Config
def usage(): """ @@ -45,7 +46,7 @@ def usage(): sys.exit()
def process_recipe(action, file_path, remoteexec, cleanup, - res_serializer, packet_capture): + res_serializer, packet_capture, config): nettestctl = NetTestController(os.path.realpath(file_path), remoteexec=remoteexec, cleanup=cleanup, res_serializer=res_serializer) @@ -70,14 +71,14 @@ def print_summary(summary): logging.info("=====================================================")
def get_recipe_result(args, file_path, remoteexec, cleanup, - res_serializer, packet_capture): + res_serializer, packet_capture, config): res_serializer.add_recipe(file_path) Logs.set_logging_root_path(file_path) loggingServer = LoggingServer(LoggingServer.DEFAULT_PORT, Logs.root_path, Logs.debug) loggingServer.start() res = process_recipe(args, file_path, remoteexec, cleanup, - res_serializer, packet_capture) + res_serializer, packet_capture, config) loggingServer.stop() return ((file_path, res))
@@ -97,6 +98,9 @@ def main(): usage() sys.exit()
+ config = Config() + config.load_config('~/.lnst/lnst.conf') + debug = 0 recipe_path = None remoteexec = False @@ -144,12 +148,14 @@ def main(): summary.append(get_recipe_result(action, recipe_file, remoteexec, cleanup, res_serializer, - packet_capture)) + packet_capture, + config)) Logs.set_logging_root_path(clean=False) else: summary.append(get_recipe_result(action, recipe_path, remoteexec, cleanup, res_serializer, - packet_capture)) + packet_capture, + config))
Logs.set_logging_root_path(clean=False)
From: Ondrej Lichtner olichtne@redhat.com
I made a small mistake in verification of mac range. This commit fixes that.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- Common/Config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/Common/Config.py b/Common/Config.py index 8ef2452..4b02a54 100644 --- a/Common/Config.py +++ b/Common/Config.py @@ -104,7 +104,10 @@ class Config(): msg = "Option mac_pool_range expects 2"\ " values sepparated by whitespaces." raise ConfigError(msg) - if not verify_mac_address(option): - msg = "Invalid MAC address: %s" % option + if not verify_mac_address(vals[0]): + msg = "Invalid MAC address: %s" % vals[0] + raise ConfigError(msg) + if not verify_mac_address(vals[1]): + msg = "Invalid MAC address: %s" % vals[1] raise ConfigError(msg) return vals
From: Ondrej Lichtner olichtne@redhat.com
Logs are now saved to location specified in the config files by option 'path' in section 'log'. The port on which the logging server is listening on can now be changed by option 'port' in the same section.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- Common/Logs.py | 7 +++++-- NetTest/NetTestController.py | 5 +++-- nettestctl.py | 7 ++++--- 3 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/Common/Logs.py b/Common/Logs.py index 0772f1f..bf20e80 100644 --- a/Common/Logs.py +++ b/Common/Logs.py @@ -187,7 +187,7 @@ class Logs: @classmethod def __init__(cls,debug=0, waitForNet=False, logger=logging.getLogger(), recipe_path=None, log_root="Logs", to_display=True, date=None, - nameExtend=None): + nameExtend=None, log_folder=None): logging.addLevelName(5, "DEBUG2") logging.DEBUG2 = 5 if nameExtend is None: @@ -200,7 +200,10 @@ class Logs: ':%(lineno)4.4d| %(levelname)s: ' '%(message)s', '%d/%m %H:%M:%S', " "*4) cls.log_root = log_root - cls.logFolder = os.path.dirname(sys.argv[0]) + if log_folder != None: + cls.logFolder = log_folder + else: + cls.logFolder = os.path.dirname(sys.argv[0]) cls.logger = logger cls.debug = debug if date is None: diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py index 2a6bbd6..cf94a05 100644 --- a/NetTest/NetTestController.py +++ b/NetTest/NetTestController.py @@ -38,11 +38,12 @@ def ignore_event(**kwarg):
class NetTestController: def __init__(self, recipe_path, remoteexec=False, cleanup=False, - res_serializer=None): + res_serializer=None, config=None): self._remoteexec = remoteexec self._docleanup = cleanup self._res_serializer = res_serializer self._remote_capture_files = {} + self._config = config
self._machine_pool = MachinePool([])
@@ -259,7 +260,7 @@ class NetTestController: logging.info("Setting logging server on machine %s", hostname) rpc = self._get_machinerpc(machine_id) ip_addr = get_corespond_local_ip(hostname) - rpc.set_logging(ip_addr, LoggingServer.DEFAULT_PORT) + rpc.set_logging(ip_addr, self._config.get_option('log', 'port'))
def _deconfigure_slaves(self): for machine_id in self._recipe["machines"]: diff --git a/nettestctl.py b/nettestctl.py index 2fab067..bc79398 100755 --- a/nettestctl.py +++ b/nettestctl.py @@ -49,7 +49,8 @@ def process_recipe(action, file_path, remoteexec, cleanup, res_serializer, packet_capture, config): nettestctl = NetTestController(os.path.realpath(file_path), remoteexec=remoteexec, cleanup=cleanup, - res_serializer=res_serializer) + res_serializer=res_serializer, + config=config) if action == "run": return nettestctl.run_recipe(packet_capture) elif action == "dump": @@ -74,7 +75,7 @@ def get_recipe_result(args, file_path, remoteexec, cleanup, res_serializer, packet_capture, config): res_serializer.add_recipe(file_path) Logs.set_logging_root_path(file_path) - loggingServer = LoggingServer(LoggingServer.DEFAULT_PORT, + loggingServer = LoggingServer(config.get_option('log', 'port'), Logs.root_path, Logs.debug) loggingServer.start() res = process_recipe(args, file_path, remoteexec, cleanup, @@ -122,7 +123,7 @@ def main(): packet_capture = True
- Logs(debug) + Logs(debug, log_folder=config.get_option('log', 'path'))
if not args: logging.error("No action command passed")
From: Ondrej Lichtner olichtne@redhat.com
Ranges of mac pool are now defined in config files, by option 'mac_pool_range' in section 'environment'. Format of this option is mac_pool_range = <starting_mac_address> <ending_mac_address>
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- NetTest/NetTestController.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py index cf94a05..f191b4d 100644 --- a/NetTest/NetTestController.py +++ b/NetTest/NetTestController.py @@ -28,8 +28,6 @@ from Common.VirtUtils import VirtNetCtl, VirtDomainCtl, BridgeCtl from Common.Utils import wait_for from NetTest.MachinePool import MachinePool
-MAC_POOL_RANGE = {"start": "52:54:01:00:00:01", "end": "52:54:01:FF:FF:FF"} - class NetTestError(Exception): pass
@@ -51,8 +49,10 @@ class NetTestController: definitions = {"recipe": self._recipe}
self._recipe["networks"] = {} - self._mac_pool = MacPool(MAC_POOL_RANGE["start"], - MAC_POOL_RANGE["end"]) + + mac_pool_range = config.get_option('environment', 'mac_pool_range') + self._mac_pool = MacPool(mac_pool_range[0], + mac_pool_range[1])
ntparse = NetTestParse(recipe_path) ntparse.set_recipe(self._recipe)
From: Ondrej Lichtner olichtne@redhat.com
This patch adds support for option 'machine_pool_dirs' in section 'environment' of config files. This option is a list of directories separated by whitespace. Whitespaces in names need to be escaped with ''. Behaviour for this option is also a bit different, config files with higher priority don't overwrite the previous value but instead are added to the list.
This list is later passed to class MachinePool. Currently this class doesn't check for the existence of these directories. Nonexistent directories will cause a crash of the application so be sure to create them before trying to use them.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- Common/Config.py | 10 ++++++++++ NetTest/NetTestController.py | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/Common/Config.py b/Common/Config.py index 4b02a54..6e68918 100644 --- a/Common/Config.py +++ b/Common/Config.py @@ -12,6 +12,7 @@ olichtne@redhat.com (Ondrej Lichtner)
import os import logging +import re from ConfigParser import ConfigParser from NetUtils import verify_ip_address, verify_mac_address
@@ -77,6 +78,8 @@ class Config(): section['mac_pool_range'] = self.optionMacRange(config[option]) elif option == 'rpcport': section['rpcport'] = self.optionPort(config[option]) + elif option == 'machine_pool_dirs': + section['pool_dirs'] = self.optionPoolDirs(config[option]) else: msg = "Unknown option: %s in section environment" % option raise ConfigError(msg) @@ -111,3 +114,10 @@ class Config(): msg = "Invalid MAC address: %s" % vals[1] raise ConfigError(msg) return vals + + def optionPoolDirs(self, option): + env = self.get_section('environment') + if 'pool_dirs' not in env: + env['pool_dirs'] = [] + opts = re.split(r'(?<!\)\s', option) + return env['pool_dirs'] + opts diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py index f191b4d..059a65e 100644 --- a/NetTest/NetTestController.py +++ b/NetTest/NetTestController.py @@ -43,7 +43,8 @@ class NetTestController: self._remote_capture_files = {} self._config = config
- self._machine_pool = MachinePool([]) + self._machine_pool = MachinePool(config.get_option('environment', + 'pool_dirs'))
self._recipe = {} definitions = {"recipe": self._recipe}
From: Ondrej Lichtner olichtne@redhat.com
When remote execution is used NetTestController now uses the rpc port specified in configuration files. This port is also used for connecting to the rpc servers when no custom port is specified in the recipe.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- NetTest/NetTestController.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py index 059a65e..d9b2124 100644 --- a/NetTest/NetTestController.py +++ b/NetTest/NetTestController.py @@ -21,7 +21,6 @@ from Common.XmlRpc import ServerProxy from NetTest.NetTestParse import NetTestParse from Common.SlaveUtils import prepare_client_session from Common.NetUtils import get_corespond_local_ip, MacPool -from NetTest.NetTestSlave import DefaultRPCPort from NetTest.NetTestCommand import NetTestCommand, str_command from Common.LoggingServer import LoggingServer from Common.VirtUtils import VirtNetCtl, VirtDomainCtl, BridgeCtl @@ -233,8 +232,9 @@ class NetTestController:
port = "22" login = "root" + rpc_port = self._config.get_option('environment', 'rpcport') session = prepare_client_session(hostname, port, login, passwd, - "nettestslave.py") + "nettestslave.py -p %s" % rpc_port) session.add_kill_handler(self._session_die) info["session"] = session
@@ -244,7 +244,7 @@ class NetTestController: if "rpcport" in info: port = info["rpcport"] else: - port = DefaultRPCPort + port = self._config.get_option('environment', 'rpcport') logging.info("Connecting to RPC on machine %s", hostname)
url = "http://%s:%d" % (hostname, port)
From: Ondrej Lichtner olichtne@redhat.com
In this patch I removed the support for option 'local_ip' ip section 'log' because this value is only used in one place as part of an output string.
I also added proper initialization of the data structure that stores loaded configuration values. The default values used here are values that were previously used as global variables. Functionality therefore shouldn't change.
The parser is now a local object of the function load_config, previously it would cause problems by using old values when nonexistant files were to be parsed.
I also added exception handling to functions get_section and get_option.
Finally I modified the way paths are handled- relative paths should be relative to the location of the config file they are specified in. And paths using ~ should be properly expanded to the current users home.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- Common/Config.py | 83 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 31 deletions(-)
diff --git a/Common/Config.py b/Common/Config.py index 6e68918..cf79c6c 100644 --- a/Common/Config.py +++ b/Common/Config.py @@ -11,9 +11,11 @@ olichtne@redhat.com (Ondrej Lichtner) """
import os +import sys import logging import re from ConfigParser import ConfigParser +from NetTest.NetTestSlave import DefaultRPCPort from NetUtils import verify_ip_address, verify_mac_address
class ConfigError(Exception): @@ -21,55 +23,68 @@ class ConfigError(Exception):
class Config(): options = None - _parser = None
def __init__(self): - self._parser = ConfigParser(dict_type=dict) self.options = dict()
+ self.options['log'] = dict() + self.options['log']['port'] = 9998 + self.options['log']['path'] = os.path.join( + os.path.dirname(sys.argv[0]), './') + + self.options['environment'] = dict() + self.options['environment']['mac_pool_range'] = \ + ['52:54:01:00:00:01', '52:54:01:FF:FF:FF'] + self.options['environment']['rpcport'] = DefaultRPCPort + self.options['environment']['pool_dirs'] = [] + def get_config(self): return self.options
def get_section(self, section): + if section not in self.options: + msg = 'Unknow section: %s' % section + raise ConfigError(msg) return self.options[section]
def get_option(self, section, option): - return self.options[section][option] + sect = self.get_section(section) + if option not in sect: + msg = 'Unknown option: %s in section: %s' % (option, section) + raise ConfigError(msg) + return sect[option]
def load_config(self, path): '''Parse and load the config file''' - self._parser.read(path) + exp_path = os.path.expanduser(path) + abs_path = os.path.abspath(exp_path) + parser = ConfigParser(dict_type=dict) + parser.read(abs_path)
- sections = self._parser._sections + sections = parser._sections for section in sections: if section == "log": - self.sectionLogs(sections[section]) + self.sectionLogs(sections[section], abs_path) elif section == "environment": - self.sectionEnvironment(sections[section]) + self.sectionEnvironment(sections[section], abs_path) else: msg = "Unknown section: %s" % section raise ConfigError(msg)
- def sectionLogs(self, config): - if 'log' not in self.options: - self.options['log'] = dict() + def sectionLogs(self, config, cfg_path): section = self.options['log']
config.pop('__name__', None) for option in config: - if option == 'local_ip': - section['local_ip'] = self.optionLocalIP(config[option]) - elif option == 'port': + if option == 'port': section['port'] = self.optionPort(config[option]) elif option == 'path': - section['path'] = self.optionLogPath(config[option]) + section['path'] = self.optionLogPath(config[option], cfg_path) else: msg = "Unknown option: %s in section log" % option raise ConfigError(msg)
- def sectionEnvironment(self, config): - if 'environment' not in self.options: - self.options['environment'] = dict() + def sectionEnvironment(self, config, cfg_path): section = self.options['environment']
config.pop('__name__', None) @@ -79,17 +94,12 @@ class Config(): elif option == 'rpcport': section['rpcport'] = self.optionPort(config[option]) elif option == 'machine_pool_dirs': - section['pool_dirs'] = self.optionPoolDirs(config[option]) + section['pool_dirs'] = self.optionPoolDirs(config[option], + cfg_path) else: msg = "Unknown option: %s in section environment" % option raise ConfigError(msg)
- def optionLocalIP(self, option): - if not verify_ip_address(option): - msg = "Invalid IP address: %s" % option - raise ConfigError(msg) - return option - def optionPort(self, option): try: int(option) @@ -98,8 +108,11 @@ class Config(): raise ConfigError(msg) return int(option)
- def optionLogPath(self, option): - return option + def optionLogPath(self, option, cfg_path): + exp_path = os.path.expanduser(option) + abs_path = os.path.join(os.path.dirname(cfg_path), exp_path) + norm_path = os.path.normpath(abs_path) + return norm_path
def optionMacRange(self, option): vals = option.split() @@ -115,9 +128,17 @@ class Config(): raise ConfigError(msg) return vals
- def optionPoolDirs(self, option): + def optionPoolDirs(self, option, cfg_path): env = self.get_section('environment') - if 'pool_dirs' not in env: - env['pool_dirs'] = [] - opts = re.split(r'(?<!\)\s', option) - return env['pool_dirs'] + opts + paths = re.split(r'(?<!\)\s', option) + + pool_dirs = env['pool_dirs'] + for path in paths: + if path == '': + continue + exp_path = os.path.expanduser(path) + abs_path = os.path.join(os.path.dirname(cfg_path), exp_path) + norm_path = os.path.normpath(abs_path) + pool_dirs.append(norm_path) + + return pool_dirs
From: Ondrej Lichtner olichtne@redhat.com
This file is being added as an example of how to create custom configuration files. For illustration it contains the default values currently used by lnst. All that needs to be done is changing the values and place the resulting file in a proper location. Currently only file ~/.lnst/lnst.conf will be used.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst.conf.example | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 lnst.conf.example
diff --git a/lnst.conf.example b/lnst.conf.example new file mode 100644 index 0000000..af64aa8 --- /dev/null +++ b/lnst.conf.example @@ -0,0 +1,7 @@ +[environment] +mac_pool_range = 52:54:01:00:00:01 52:54:01:FF:FF:FF +rpcport = 9999 +machine_pool_dirs = +[log] +path = ./ +port = 9998
lnst-developers@lists.fedorahosted.org