Tue, Jul 12, 2016 at 11:00:33AM CEST, jprochaz@redhat.com wrote:
This patch implements PyRecipes and breaks existing XML recipes.
Python recipes, unlike XML recipes, do not need to be parsed before execution. They contain both network topology definition and test commands.
Network topology is defined in the beginning of PyRecipe, example:
import lnst
m1 = lnst.add_host() m2 = lnst.add_host()
m1_eth1 = m1.add_interface(label="tnet") m2_eth1 = m2.add_interface(label="tnet")
In the example, test will use 2 different hosts, each with 1 eth device.
To run match and network provision, use lnst.match() method in while loop, example:
while lnst.match(): m1_eth1.reset(ip="192.168.0.1/24") m2_eth1.reset(ip="192.168.0.2/24")
Why "reset"? For every match, the config should be cleared.
This piece of code will run the match, prepare the setup and sets IP addresses to both eth devices.
More info on how to use Pyrecipes will be in our documentation (github wiki).
After this patch, XML recipes will no longer work. It removes and modifies a lot of code in order to not break git bisect.
Signed-off-by: Jiri Prochazka jprochaz@redhat.com
lnst-ctl | 13 +- lnst/Controller/NetTestController.py | 266 ++++++----------------------------- lnst/Controller/Task.py | 24 ++-- 3 files changed, 62 insertions(+), 241 deletions(-)
diff --git a/lnst-ctl b/lnst-ctl index b75b8a0..93b084b 100755 --- a/lnst-ctl +++ b/lnst-ctl @@ -178,9 +178,13 @@ def get_recipe_result(action, file_path, log_ctl, res_serializer, res = {} if matches == 1: try:
nettestctl.provision_machines()
nettestctl.print_match_description()
# init TaskAPI.Ctl
nettestctl.init_taskapi() res = exec_action(action, nettestctl)
except NoMatchError as err:
no_match = True
log_ctl.unset_recipe()
logging.warning("Match %d not possible." % matches) except Exception as err: no_match = True log_exc_traceback()
@@ -190,12 +194,11 @@ def get_recipe_result(action, file_path, log_ctl, res_serializer, retval = RETVAL_ERR elif matches > 1: try:
nettestctl.provision_machines()
nettestctl.init_taskapi() log_ctl.set_recipe(file_path, prepend=True, expand="match_%d" % matches) log_dir = log_ctl.get_recipe_log_path() recipe_head_log_entry(file_path, log_dir, matches) res_serializer.add_recipe(file_path, matches)
nettestctl.print_match_description() res = exec_action(action, nettestctl) except NoMatchError as err: no_match = True
@@ -400,7 +403,7 @@ def main(): all_files.sort() for f in all_files: recipe_file = os.path.join(recipe_path, f)
if re.match(r'^.*\.xml$', recipe_file):
if re.match(r'^.*\.py$', recipe_file): recipe_files.append(recipe_file) else: recipe_files.append(recipe_path)
diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py index 0115d2f..135bbf6 100644 --- a/lnst/Controller/NetTestController.py +++ b/lnst/Controller/NetTestController.py @@ -61,11 +61,10 @@ class NetTestController: self._res_serializer = res_serializer self._remote_capture_files = {} self._log_ctl = log_ctl
self._recipe_path = Path(None, recipe_path).abs_path()
self._recipe_path = Path(None, recipe_path) self._msg_dispatcher = MessageDispatcher(log_ctl) self._packet_capture = packet_capture self._reduce_sync = reduce_sync
self._parser = RecipeParser(recipe_path) self._defined_aliases = defined_aliases self._multi_match = multi_match
@@ -80,10 +79,6 @@ class NetTestController: mac_pool_range = lnst_config.get_option('environment', 'mac_pool_range') self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
self._parser.set_machines(self._machines)
self._parser.set_aliases(defined_aliases, overriden_aliases)
self._recipe = self._parser.parse()
conf_pools = lnst_config.get_pools() pools = {} if len(restrict_pools) > 0:
@@ -100,9 +95,6 @@ class NetTestController: sp = SlavePool(pools, pool_checks) self._slave_pool = sp
mreq = self._get_machine_requirements()
sp.set_machine_requirements(mreq)
modules_dirs = lnst_config.get_option('environment', 'module_dirs') tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
@@ -196,21 +188,21 @@ class NetTestController: return mreq
def _prepare_network(self, resource_sync=True):
recipe = self._recipe
mreq = Task.get_mreq() machines = self._machines for m_id in machines.keys(): self._prepare_machine(m_id, resource_sync)
for machine_xml_data in recipe["machines"]:
m_id = machine_xml_data["id"]
for machine_id, machine_data in mreq.iteritems():
m_id = machine_id m = machines[m_id] namespaces = set()
for iface_xml_data in machine_xml_data["interfaces"]:
self._prepare_interface(m_id, iface_xml_data)
for if_id, iface_data in machine_data["interfaces"].iteritems():
self._prepare_interface(m_id, if_id, iface_data)
if iface_xml_data["netns"] != None:
namespaces.add(iface_xml_data["netns"])
if iface_data["netns"] != None:
namespaces.add(iface_data["netns"]) if len(namespaces) > 0: m.disable_nm()
@@ -269,148 +261,16 @@ class NetTestController: machine.set_mac_pool(self._mac_pool) machine.set_network_bridges(self._network_bridges)
recipe_name = os.path.basename(self._recipe_path)
recipe_name = os.path.basename(self._recipe_path.abs_path()) machine.init_connection(recipe_name)
sync_table = {'module': {}, 'tools': {}}
if resource_sync:
res_table = copy.deepcopy(self._resource_table)
for task in self._recipe['tasks']:
if 'module_dir' in task:
modules = self._load_test_modules([task['module_dir']])
res_table['module'].update(modules)
if 'tools_dir' in task:
tools = self._load_test_tools([task['tools_dir']])
res_table['tools'].update(tools)
if 'commands' not in task:
if not self._reduce_sync:
sync_table = res_table
break
else:
continue
for cmd in task['commands']:
if 'host' not in cmd or cmd['host'] != m_id:
continue
if cmd['type'] == 'test':
mod = cmd['module']
if mod in res_table['module']:
sync_table['module'][mod] = res_table['module'][mod]
# check if test module uses some test tools
mod_path = res_table['module'][mod]["path"]
mod_tools = get_module_tools(mod_path)
for t in mod_tools:
if t in sync_table['tools']:
continue
logging.debug("Adding '%s' tool as "\
"dependency of %s test module" % (t, mod))
sync_table['tools'][t] = res_table['tools'][t]
else:
msg = "Module '%s' not found on the controller"\
% mod
raise RecipeError(msg, cmd)
if cmd['type'] == 'exec' and 'from' in cmd:
tool = cmd['from']
if tool in res_table['tools']:
sync_table['tools'][tool] = res_table['tools'][tool]
else:
msg = "Tool '%s' not found on the controller" % tool
raise RecipeError(msg, cmd)
machine.sync_resources(sync_table)
- def _prepare_interface(self, m_id, iface_xml_data):
- def _prepare_interface(self, m_id, if_id, iface_data): machine = self._machines[m_id]
if_id = iface_xml_data["id"]
if_type = iface_xml_data["type"]
try:
iface = machine.get_interface(if_id)
except MachineError:
if if_type == 'lo':
iface = machine.new_loopback_interface(if_id)
else:
iface = machine.new_soft_interface(if_id, if_type)
if "slaves" in iface_xml_data:
for slave in iface_xml_data["slaves"]:
slave_id = slave["id"]
iface.add_slave(machine.get_interface(slave_id))
# Some soft devices (such as team) use per-slave options
if "options" in slave:
for opt in slave["options"]:
iface.set_slave_option(slave_id, opt["name"],
opt["value"])
if "addresses" in iface_xml_data:
for addr in iface_xml_data["addresses"]:
iface.add_address(addr)
if "options" in iface_xml_data:
for opt in iface_xml_data["options"]:
iface.set_option(opt["name"], opt["value"])
if "netem" in iface_xml_data:
iface.set_netem(iface_xml_data["netem"].to_dict())
if "ovs_conf" in iface_xml_data:
iface.set_ovs_conf(iface_xml_data["ovs_conf"].to_dict())
if iface_xml_data["netns"] != None:
iface.set_netns(iface_xml_data["netns"])
if "peer" in iface_xml_data:
iface.set_peer(iface_xml_data["peer"])
def _prepare_tasks(self):
self._tasks = []
for task_data in self._recipe["tasks"]:
task = {}
task["quit_on_fail"] = False
if "quit_on_fail" in task_data:
task["quit_on_fail"] = bool_it(task_data["quit_on_fail"])
if "module_dir" in task_data:
task["module_dir"] = task_data["module_dir"]
if "tools_dir" in task_data:
task["tools_dir"] = task_data["tools_dir"]
if "python" in task_data:
root = Path(None, self._recipe_path).get_root()
path = Path(root, task_data["python"])
task["python"] = path
if not path.exists():
msg = "Task file '%s' not found." % path.to_str()
raise RecipeError(msg, task_data)
self._tasks.append(task)
continue
task["commands"] = task_data["commands"]
task["skeleton"] = []
for cmd_data in task["commands"]:
cmd = {"type": cmd_data["type"]}
if "host" in cmd_data:
cmd["host"] = cmd_data["host"]
if cmd["host"] not in self._machines:
msg = "Invalid host id '%s'." % cmd["host"]
raise RecipeError(msg, cmd_data)
if cmd["type"] in ["test", "exec"]:
if "bg_id" in cmd_data:
cmd["bg_id"] = cmd_data["bg_id"]
elif cmd["type"] in ["wait", "intr", "kill"]:
cmd["proc_id"] = cmd_data["bg_id"]
task["skeleton"].append(cmd)
if self._check_task(task):
raise RecipeError("Incorrect command sequence.", task_data)
iface = machine.get_interface(if_id)
self._tasks.append(task)
if iface_data["netns"] != None:
iface.set_netns(iface_data["netns"])
def _prepare_command(self, cmd_data): cmd = {"type": cmd_data["type"]}
@@ -635,36 +495,23 @@ class NetTestController: os.remove("/tmp/.lnst_machine_conf")
def match_setup(self):
self.run_mode = "match_setup"
res = self._run_python_task() return {"passed": True}
def config_only_recipe(self):
self.run_mode = "config_only" try:
self._prepare_network(resource_sync=False)
except (KeyboardInterrupt, Exception) as exc:
msg = "Exception raised during configuration."
logging.error(msg)
self._cleanup_slaves()
res = self._run_recipe()
except Exception as exc:
logging.error("Recipe execution terminated by unexpected exception") raise
self._save_machine_config()
self._cleanup_slaves(deconfigure=False)
return {"passed": True}
- def run_recipe(self):
try:
self._prepare_tasks()
self._prepare_network()
except (KeyboardInterrupt, Exception) as exc:
msg = "Exception raised during configuration."
logging.error(msg)
finally: self._cleanup_slaves()
raise
if self._packet_capture:
self._start_packet_capture()
return res
err = None
- def run_recipe(self): try: res = self._run_recipe() except Exception as exc:
@@ -699,70 +546,44 @@ class NetTestController: def _run_recipe(self): overall_res = {"passed": True}
for task in self._tasks:
try: self._res_serializer.add_task()
try:
res = self._run_task(task)
except CommandException as exc:
logging.debug(exc)
overall_res["passed"] = False
overall_res["err_msg"] = "Command exception raised."
break
for machine in self._machines.itervalues():
machine.restore_system_config()
# task failed, check if we should quit_on_fail
if not res:
overall_res["passed"] = False
overall_res["err_msg"] = "At least one command failed."
if task["quit_on_fail"]:
break
res = self._run_python_task()
except CommandException as exc:
logging.debug(exc)
overall_res["passed"] = False
overall_res["err_msg"] = "Command exception raised."
for machine in self._machines.itervalues():
machine.restore_system_config()
# task failed
if not res:
overall_res["passed"] = False
overall_res["err_msg"] = "At least one command failed." return overall_res
def init_taskapi(self): Task.ctl = Task.ControllerAPI(self)
- def _run_python_task(self, task):
- def _run_python_task(self): #backup of resource table res_table_bkp = copy.deepcopy(self._resource_table)
if 'module_dir' in task:
modules = self._load_test_modules([task['module_dir']])
self._resource_table['module'].update(modules)
if 'tools_dir' in task:
tools = self._load_test_tools([task['tools_dir']])
self._resource_table['tools'].update(tools)
# Initialize the API handle
Task.ctl = Task.ControllerAPI(self, self._machines) cwd = os.getcwd()
task_path = task["python"]
task_path = self._recipe_path name = os.path.basename(task_path.abs_path()).split(".")[0] sys.path.append(os.path.dirname(task_path.resolve())) os.chdir(os.path.dirname(task_path.resolve()))
module = imp.load_source(name, task_path.resolve())
imp.load_source(name, task_path.resolve()) os.chdir(cwd) sys.path.remove(os.path.dirname(task_path.resolve())) #restore resource table self._resource_table = res_table_bkp
return module.ctl._result
- def _run_task(self, task):
if "python" in task:
return self._run_python_task(task)
seq_passed = True
for cmd_data in task["commands"]:
cmd = self._prepare_command(cmd_data)
cmd_res = self._run_command(cmd)
if not cmd_res["passed"]:
seq_passed = False
return seq_passed
return Task.ctl._result
def _run_command(self, command): logging.info("Executing command: [%s]", str_command(command))
@@ -891,12 +712,11 @@ class NetTestController: return packages
def _get_alias(self, alias):
templates = self._parser._template_proc
return templates._find_definition(alias)
if alias in self._defined_aliases:
return self._defined_aliases[alias]
def _get_aliases(self):
templates = self._parser._template_proc
return templates._dump_definitions()
return self._defined_aliases
class MessageDispatcher(ConnectionHandler): def __init__(self, log_ctl): diff --git a/lnst/Controller/Task.py b/lnst/Controller/Task.py index e36b79f..9f40846 100644 --- a/lnst/Controller/Task.py +++ b/lnst/Controller/Task.py @@ -88,7 +88,7 @@ class TaskError(Exception): pass class ControllerAPI(object): """ An API class representing the controller. """
- def __init__(self, ctl, hosts):
- def __init__(self, ctl): self._ctl = ctl self.run_mode = ctl.run_mode self._result = True
@@ -99,8 +99,6 @@ class ControllerAPI(object): self._perf_repo_api = PerfRepoAPI()
self._hosts = {}
for host_id, host in hosts.iteritems():
self._hosts[host_id] = HostAPI(self, host_id, host)
def get_mreq(self): return self.mreq
@@ -188,7 +186,7 @@ class ControllerAPI(object): cmd = {"type": "ctl_wait", "seconds": int(seconds)} return self._ctl._run_command(cmd)
- def get_alias(self, alias):
- def get_alias(self, alias, default=None): """ Get the value of user defined alias.
@@ -199,9 +197,13 @@ class ControllerAPI(object): :rtype: string """ try:
return self._ctl._get_alias(alias)
val = self._ctl._get_alias(alias)
if val is None:
return default
else:
return val except XmlTemplateError:
return None
return default
def get_aliases(self): """
@@ -269,16 +271,12 @@ class ControllerAPI(object): class HostAPI(object): """ An API class representing a host machine. """
- def __init__(self, ctl, host_id, host):
- def __init__(self, ctl, host_id): self._ctl = ctl self._id = host_id
self._m = host
self._m = None self._ifaces = {}
for interface in self._m.get_interfaces():
if interface.get_id() is None:
continue
self._ifaces[interface.get_id()] = InterfaceAPI(interface, self) self._if_id_seq = 0 self._bg_id_seq = 0
@@ -313,7 +311,7 @@ class HostAPI(object): return self._ifaces[if_id]
def get_id(self):
return self._m.get_id()
return self._id
def get_configuration(self): return self._m.get_configuration()
-- 2.4.11 _______________________________________________ LNST-developers mailing list lnst-developers@lists.fedorahosted.org https://lists.fedorahosted.org/admin/lists/lnst-developers@lists.fedorahoste...