Tue, Jul 12, 2016 at 11:00:33AM CEST, jprochaz@redhat.com wrote:
>This patch implements PyRecipes and breaks existing XML recipes.
>
>Python recipes, unlike XML recipes, do not need to be parsed before
>execution. They contain both network topology definition and test
>commands.
>
>Network topology is defined in the beginning of PyRecipe, example:
>
>import lnst
>
>m1 = lnst.add_host()
>m2 = lnst.add_host()
>
>m1_eth1 = m1.add_interface(label="tnet")
>m2_eth1 = m2.add_interface(label="tnet")
>
>In the example, test will use 2 different hosts, each with 1 eth device.
>
>To run match and network provision, use lnst.match() method in while
>loop, example:
>
>while lnst.match():
> m1_eth1.reset(ip="192.168.0.1/24")
> m2_eth1.reset(ip="192.168.0.2/24")
Why "reset"? For every match, the config should be cleared.
>_______________________________________________
>
>This piece of code will run the match, prepare the setup and sets IP
>addresses to both eth devices.
>
>More info on how to use Pyrecipes will be in our documentation (github wiki).
>
>After this patch, XML recipes will no longer work. It removes and
>modifies a lot of code in order to not break git bisect.
>
>Signed-off-by: Jiri Prochazka <jprochaz@redhat.com>
>---
> lnst-ctl | 13 +-
> lnst/Controller/NetTestController.py | 266 ++++++-----------------------------
> lnst/Controller/Task.py | 24 ++--
> 3 files changed, 62 insertions(+), 241 deletions(-)
>
>diff --git a/lnst-ctl b/lnst-ctl
>index b75b8a0..93b084b 100755
>--- a/lnst-ctl
>+++ b/lnst-ctl
>@@ -178,9 +178,13 @@ def get_recipe_result(action, file_path, log_ctl, res_serializer,
> res = {}
> if matches == 1:
> try:
>- nettestctl.provision_machines()
>- nettestctl.print_match_description()
>+ # init TaskAPI.Ctl
>+ nettestctl.init_taskapi()
> res = exec_action(action, nettestctl)
>+ except NoMatchError as err:
>+ no_match = True
>+ log_ctl.unset_recipe()
>+ logging.warning("Match %d not possible." % matches)
> except Exception as err:
> no_match = True
> log_exc_traceback()
>@@ -190,12 +194,11 @@ def get_recipe_result(action, file_path, log_ctl, res_serializer,
> retval = RETVAL_ERR
> elif matches > 1:
> try:
>- nettestctl.provision_machines()
>+ nettestctl.init_taskapi()
> log_ctl.set_recipe(file_path, prepend=True, expand="match_%d" % matches)
> log_dir = log_ctl.get_recipe_log_path()
> recipe_head_log_entry(file_path, log_dir, matches)
> res_serializer.add_recipe(file_path, matches)
>- nettestctl.print_match_description()
> res = exec_action(action, nettestctl)
> except NoMatchError as err:
> no_match = True
>@@ -400,7 +403,7 @@ def main():
> all_files.sort()
> for f in all_files:
> recipe_file = os.path.join(recipe_path, f)
>- if re.match(r'^.*\.xml$', recipe_file):
>+ if re.match(r'^.*\.py$', recipe_file):
> recipe_files.append(recipe_file)
> else:
> recipe_files.append(recipe_path)
>diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py
>index 0115d2f..135bbf6 100644
>--- a/lnst/Controller/NetTestController.py
>+++ b/lnst/Controller/NetTestController.py
>@@ -61,11 +61,10 @@ class NetTestController:
> self._res_serializer = res_serializer
> self._remote_capture_files = {}
> self._log_ctl = log_ctl
>- self._recipe_path = Path(None, recipe_path).abs_path()
>+ self._recipe_path = Path(None, recipe_path)
> self._msg_dispatcher = MessageDispatcher(log_ctl)
> self._packet_capture = packet_capture
> self._reduce_sync = reduce_sync
>- self._parser = RecipeParser(recipe_path)
> self._defined_aliases = defined_aliases
> self._multi_match = multi_match
>
>@@ -80,10 +79,6 @@ class NetTestController:
> mac_pool_range = lnst_config.get_option('environment', 'mac_pool_range')
> self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
>
>- self._parser.set_machines(self._machines)
>- self._parser.set_aliases(defined_aliases, overriden_aliases)
>- self._recipe = self._parser.parse()
>-
> conf_pools = lnst_config.get_pools()
> pools = {}
> if len(restrict_pools) > 0:
>@@ -100,9 +95,6 @@ class NetTestController:
> sp = SlavePool(pools, pool_checks)
> self._slave_pool = sp
>
>- mreq = self._get_machine_requirements()
>- sp.set_machine_requirements(mreq)
>-
> modules_dirs = lnst_config.get_option('environment', 'module_dirs')
> tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
>
>@@ -196,21 +188,21 @@ class NetTestController:
> return mreq
>
> def _prepare_network(self, resource_sync=True):
>- recipe = self._recipe
>+ mreq = Task.get_mreq()
>
> machines = self._machines
> for m_id in machines.keys():
> self._prepare_machine(m_id, resource_sync)
>
>- for machine_xml_data in recipe["machines"]:
>- m_id = machine_xml_data["id"]
>+ for machine_id, machine_data in mreq.iteritems():
>+ m_id = machine_id
> m = machines[m_id]
> namespaces = set()
>- for iface_xml_data in machine_xml_data["interfaces"]:
>- self._prepare_interface(m_id, iface_xml_data)
>+ for if_id, iface_data in machine_data["interfaces"].iteritems():
>+ self._prepare_interface(m_id, if_id, iface_data)
>
>- if iface_xml_data["netns"] != None:
>- namespaces.add(iface_xml_data["netns"])
>+ if iface_data["netns"] != None:
>+ namespaces.add(iface_data["netns"])
>
> if len(namespaces) > 0:
> m.disable_nm()
>@@ -269,148 +261,16 @@ class NetTestController:
> machine.set_mac_pool(self._mac_pool)
> machine.set_network_bridges(self._network_bridges)
>
>- recipe_name = os.path.basename(self._recipe_path)
>+ recipe_name = os.path.basename(self._recipe_path.abs_path())
> machine.init_connection(recipe_name)
>
>- sync_table = {'module': {}, 'tools': {}}
>- if resource_sync:
>- res_table = copy.deepcopy(self._resource_table)
>- for task in self._recipe['tasks']:
>- if 'module_dir' in task:
>- modules = self._load_test_modules([task['module_dir']])
>- res_table['module'].update(modules)
>- if 'tools_dir' in task:
>- tools = self._load_test_tools([task['tools_dir']])
>- res_table['tools'].update(tools)
>-
>- if 'commands' not in task:
>- if not self._reduce_sync:
>- sync_table = res_table
>- break
>- else:
>- continue
>- for cmd in task['commands']:
>- if 'host' not in cmd or cmd['host'] != m_id:
>- continue
>- if cmd['type'] == 'test':
>- mod = cmd['module']
>- if mod in res_table['module']:
>- sync_table['module'][mod] = res_table['module'][mod]
>- # check if test module uses some test tools
>- mod_path = res_table['module'][mod]["path"]
>- mod_tools = get_module_tools(mod_path)
>- for t in mod_tools:
>- if t in sync_table['tools']:
>- continue
>- logging.debug("Adding '%s' tool as "\
>- "dependency of %s test module" % (t, mod))
>- sync_table['tools'][t] = res_table['tools'][t]
>- else:
>- msg = "Module '%s' not found on the controller"\
>- % mod
>- raise RecipeError(msg, cmd)
>- if cmd['type'] == 'exec' and 'from' in cmd:
>- tool = cmd['from']
>- if tool in res_table['tools']:
>- sync_table['tools'][tool] = res_table['tools'][tool]
>- else:
>- msg = "Tool '%s' not found on the controller" % tool
>- raise RecipeError(msg, cmd)
>- machine.sync_resources(sync_table)
>-
>- def _prepare_interface(self, m_id, iface_xml_data):
>+ def _prepare_interface(self, m_id, if_id, iface_data):
> machine = self._machines[m_id]
>- if_id = iface_xml_data["id"]
>- if_type = iface_xml_data["type"]
>-
>- try:
>- iface = machine.get_interface(if_id)
>- except MachineError:
>- if if_type == 'lo':
>- iface = machine.new_loopback_interface(if_id)
>- else:
>- iface = machine.new_soft_interface(if_id, if_type)
>-
>- if "slaves" in iface_xml_data:
>- for slave in iface_xml_data["slaves"]:
>- slave_id = slave["id"]
>- iface.add_slave(machine.get_interface(slave_id))
>-
>- # Some soft devices (such as team) use per-slave options
>- if "options" in slave:
>- for opt in slave["options"]:
>- iface.set_slave_option(slave_id, opt["name"],
>- opt["value"])
>-
>- if "addresses" in iface_xml_data:
>- for addr in iface_xml_data["addresses"]:
>- iface.add_address(addr)
>-
>- if "options" in iface_xml_data:
>- for opt in iface_xml_data["options"]:
>- iface.set_option(opt["name"], opt["value"])
>-
>- if "netem" in iface_xml_data:
>- iface.set_netem(iface_xml_data["netem"].to_dict())
>-
>- if "ovs_conf" in iface_xml_data:
>- iface.set_ovs_conf(iface_xml_data["ovs_conf"].to_dict())
>-
>- if iface_xml_data["netns"] != None:
>- iface.set_netns(iface_xml_data["netns"])
>-
>- if "peer" in iface_xml_data:
>- iface.set_peer(iface_xml_data["peer"])
>-
>- def _prepare_tasks(self):
>- self._tasks = []
>- for task_data in self._recipe["tasks"]:
>- task = {}
>- task["quit_on_fail"] = False
>- if "quit_on_fail" in task_data:
>- task["quit_on_fail"] = bool_it(task_data["quit_on_fail"])
>-
>- if "module_dir" in task_data:
>- task["module_dir"] = task_data["module_dir"]
>-
>- if "tools_dir" in task_data:
>- task["tools_dir"] = task_data["tools_dir"]
>-
>- if "python" in task_data:
>- root = Path(None, self._recipe_path).get_root()
>- path = Path(root, task_data["python"])
>-
>- task["python"] = path
>- if not path.exists():
>- msg = "Task file '%s' not found." % path.to_str()
>- raise RecipeError(msg, task_data)
>-
>- self._tasks.append(task)
>- continue
>-
>- task["commands"] = task_data["commands"]
>- task["skeleton"] = []
>- for cmd_data in task["commands"]:
>- cmd = {"type": cmd_data["type"]}
>-
>- if "host" in cmd_data:
>- cmd["host"] = cmd_data["host"]
>- if cmd["host"] not in self._machines:
>- msg = "Invalid host id '%s'." % cmd["host"]
>- raise RecipeError(msg, cmd_data)
>-
>- if cmd["type"] in ["test", "exec"]:
>- if "bg_id" in cmd_data:
>- cmd["bg_id"] = cmd_data["bg_id"]
>- elif cmd["type"] in ["wait", "intr", "kill"]:
>- cmd["proc_id"] = cmd_data["bg_id"]
>-
>- task["skeleton"].append(cmd)
>
>- if self._check_task(task):
>- raise RecipeError("Incorrect command sequence.", task_data)
>+ iface = machine.get_interface(if_id)
>
>- self._tasks.append(task)
>+ if iface_data["netns"] != None:
>+ iface.set_netns(iface_data["netns"])
>
> def _prepare_command(self, cmd_data):
> cmd = {"type": cmd_data["type"]}
>@@ -635,36 +495,23 @@ class NetTestController:
> os.remove("/tmp/.lnst_machine_conf")
>
> def match_setup(self):
>+ self.run_mode = "match_setup"
>+ res = self._run_python_task()
> return {"passed": True}
>
> def config_only_recipe(self):
>+ self.run_mode = "config_only"
> try:
>- self._prepare_network(resource_sync=False)
>- except (KeyboardInterrupt, Exception) as exc:
>- msg = "Exception raised during configuration."
>- logging.error(msg)
>- self._cleanup_slaves()
>+ res = self._run_recipe()
>+ except Exception as exc:
>+ logging.error("Recipe execution terminated by unexpected exception")
> raise
>-
>- self._save_machine_config()
>-
>- self._cleanup_slaves(deconfigure=False)
>- return {"passed": True}
>-
>- def run_recipe(self):
>- try:
>- self._prepare_tasks()
>- self._prepare_network()
>- except (KeyboardInterrupt, Exception) as exc:
>- msg = "Exception raised during configuration."
>- logging.error(msg)
>+ finally:
> self._cleanup_slaves()
>- raise
>
>- if self._packet_capture:
>- self._start_packet_capture()
>+ return res
>
>- err = None
>+ def run_recipe(self):
> try:
> res = self._run_recipe()
> except Exception as exc:
>@@ -699,70 +546,44 @@ class NetTestController:
> def _run_recipe(self):
> overall_res = {"passed": True}
>
>- for task in self._tasks:
>+ try:
> self._res_serializer.add_task()
>- try:
>- res = self._run_task(task)
>- except CommandException as exc:
>- logging.debug(exc)
>- overall_res["passed"] = False
>- overall_res["err_msg"] = "Command exception raised."
>- break
>-
>- for machine in self._machines.itervalues():
>- machine.restore_system_config()
>-
>- # task failed, check if we should quit_on_fail
>- if not res:
>- overall_res["passed"] = False
>- overall_res["err_msg"] = "At least one command failed."
>- if task["quit_on_fail"]:
>- break
>+ res = self._run_python_task()
>+ except CommandException as exc:
>+ logging.debug(exc)
>+ overall_res["passed"] = False
>+ overall_res["err_msg"] = "Command exception raised."
>+
>+ for machine in self._machines.itervalues():
>+ machine.restore_system_config()
>+
>+ # task failed
>+ if not res:
>+ overall_res["passed"] = False
>+ overall_res["err_msg"] = "At least one command failed."
>
> return overall_res
>
> def init_taskapi(self):
> Task.ctl = Task.ControllerAPI(self)
>
>- def _run_python_task(self, task):
>+ def _run_python_task(self):
> #backup of resource table
> res_table_bkp = copy.deepcopy(self._resource_table)
>- if 'module_dir' in task:
>- modules = self._load_test_modules([task['module_dir']])
>- self._resource_table['module'].update(modules)
>- if 'tools_dir' in task:
>- tools = self._load_test_tools([task['tools_dir']])
>- self._resource_table['tools'].update(tools)
>-
>- # Initialize the API handle
>- Task.ctl = Task.ControllerAPI(self, self._machines)
>
> cwd = os.getcwd()
>- task_path = task["python"]
>+ task_path = self._recipe_path
> name = os.path.basename(task_path.abs_path()).split(".")[0]
> sys.path.append(os.path.dirname(task_path.resolve()))
> os.chdir(os.path.dirname(task_path.resolve()))
>- module = imp.load_source(name, task_path.resolve())
>+ imp.load_source(name, task_path.resolve())
> os.chdir(cwd)
> sys.path.remove(os.path.dirname(task_path.resolve()))
>
> #restore resource table
> self._resource_table = res_table_bkp
>
>- return module.ctl._result
>-
>- def _run_task(self, task):
>- if "python" in task:
>- return self._run_python_task(task)
>-
>- seq_passed = True
>- for cmd_data in task["commands"]:
>- cmd = self._prepare_command(cmd_data)
>- cmd_res = self._run_command(cmd)
>- if not cmd_res["passed"]:
>- seq_passed = False
>-
>- return seq_passed
>+ return Task.ctl._result
>
> def _run_command(self, command):
> logging.info("Executing command: [%s]", str_command(command))
>@@ -891,12 +712,11 @@ class NetTestController:
> return packages
>
> def _get_alias(self, alias):
>- templates = self._parser._template_proc
>- return templates._find_definition(alias)
>+ if alias in self._defined_aliases:
>+ return self._defined_aliases[alias]
>
> def _get_aliases(self):
>- templates = self._parser._template_proc
>- return templates._dump_definitions()
>+ return self._defined_aliases
>
> class MessageDispatcher(ConnectionHandler):
> def __init__(self, log_ctl):
>diff --git a/lnst/Controller/Task.py b/lnst/Controller/Task.py
>index e36b79f..9f40846 100644
>--- a/lnst/Controller/Task.py
>+++ b/lnst/Controller/Task.py
>@@ -88,7 +88,7 @@ class TaskError(Exception): pass
> class ControllerAPI(object):
> """ An API class representing the controller. """
>
>- def __init__(self, ctl, hosts):
>+ def __init__(self, ctl):
> self._ctl = ctl
> self.run_mode = ctl.run_mode
> self._result = True
>@@ -99,8 +99,6 @@ class ControllerAPI(object):
> self._perf_repo_api = PerfRepoAPI()
>
> self._hosts = {}
>- for host_id, host in hosts.iteritems():
>- self._hosts[host_id] = HostAPI(self, host_id, host)
>
> def get_mreq(self):
> return self.mreq
>@@ -188,7 +186,7 @@ class ControllerAPI(object):
> cmd = {"type": "ctl_wait", "seconds": int(seconds)}
> return self._ctl._run_command(cmd)
>
>- def get_alias(self, alias):
>+ def get_alias(self, alias, default=None):
> """
> Get the value of user defined alias.
>
>@@ -199,9 +197,13 @@ class ControllerAPI(object):
> :rtype: string
> """
> try:
>- return self._ctl._get_alias(alias)
>+ val = self._ctl._get_alias(alias)
>+ if val is None:
>+ return default
>+ else:
>+ return val
> except XmlTemplateError:
>- return None
>+ return default
>
> def get_aliases(self):
> """
>@@ -269,16 +271,12 @@ class ControllerAPI(object):
> class HostAPI(object):
> """ An API class representing a host machine. """
>
>- def __init__(self, ctl, host_id, host):
>+ def __init__(self, ctl, host_id):
> self._ctl = ctl
> self._id = host_id
>- self._m = host
>+ self._m = None
>
> self._ifaces = {}
>- for interface in self._m.get_interfaces():
>- if interface.get_id() is None:
>- continue
>- self._ifaces[interface.get_id()] = InterfaceAPI(interface, self)
> self._if_id_seq = 0
> self._bg_id_seq = 0
>
>@@ -313,7 +311,7 @@ class HostAPI(object):
> return self._ifaces[if_id]
>
> def get_id(self):
>- return self._m.get_id()
>+ return self._id
>
> def get_configuration(self):
> return self._m.get_configuration()
>--
>2.4.11
>LNST-developers mailing list
>lnst-developers@lists.fedorahosted.org
>https://lists.fedorahosted.org/admin/lists/lnst-developers@lists.fedorahosted.org