2016-07-12 14:54 GMT+02:00 Jiri Pirko jiri@resnulli.us:
Tue, Jul 12, 2016 at 11:00:33AM CEST, jprochaz@redhat.com wrote:
This patch implements PyRecipes and breaks existing XML recipes.
Python recipes, unlike XML recipes, do not need to be parsed before execution. They contain both network topology definition and test commands.
Network topology is defined in the beginning of PyRecipe, example:
import lnst
m1 = lnst.add_host() m2 = lnst.add_host()
m1_eth1 = m1.add_interface(label="tnet") m2_eth1 = m2.add_interface(label="tnet")
In the example, test will use 2 different hosts, each with 1 eth device.
To run match and network provision, use lnst.match() method in while loop, example:
while lnst.match(): m1_eth1.reset(ip="192.168.0.1/24") m2_eth1.reset(ip="192.168.0.2/24")
Why "reset"? For every match, the config should be cleared.
This (interface handling) should also be one of the items at our API discussion in August.
This piece of code will run the match, prepare the setup and sets IP addresses to both eth devices.
More info on how to use Pyrecipes will be in our documentation (github
wiki).
After this patch, XML recipes will no longer work. It removes and modifies a lot of code in order to not break git bisect.
Signed-off-by: Jiri Prochazka jprochaz@redhat.com
lnst-ctl | 13 +- lnst/Controller/NetTestController.py | 266
++++++-----------------------------
lnst/Controller/Task.py | 24 ++-- 3 files changed, 62 insertions(+), 241 deletions(-)
diff --git a/lnst-ctl b/lnst-ctl index b75b8a0..93b084b 100755 --- a/lnst-ctl +++ b/lnst-ctl @@ -178,9 +178,13 @@ def get_recipe_result(action, file_path, log_ctl,
res_serializer,
res = {} if matches == 1: try:
nettestctl.provision_machines()nettestctl.print_match_description()
# init TaskAPI.Ctlnettestctl.init_taskapi() res = exec_action(action, nettestctl)except NoMatchError as err:no_match = Truelog_ctl.unset_recipe()logging.warning("Match %d not possible." % matches) except Exception as err: no_match = True log_exc_traceback()@@ -190,12 +194,11 @@ def get_recipe_result(action, file_path, log_ctl,
res_serializer,
retval = RETVAL_ERR elif matches > 1: try:
nettestctl.provision_machines()
nettestctl.init_taskapi() log_ctl.set_recipe(file_path, prepend=True,expand="match_%d" % matches)
log_dir = log_ctl.get_recipe_log_path() recipe_head_log_entry(file_path, log_dir, matches) res_serializer.add_recipe(file_path, matches)
nettestctl.print_match_description() res = exec_action(action, nettestctl) except NoMatchError as err: no_match = True@@ -400,7 +403,7 @@ def main(): all_files.sort() for f in all_files: recipe_file = os.path.join(recipe_path, f)
if re.match(r'^.*\.xml$', recipe_file):
if re.match(r'^.*\.py$', recipe_file): recipe_files.append(recipe_file) else: recipe_files.append(recipe_path)diff --git a/lnst/Controller/NetTestController.py
b/lnst/Controller/NetTestController.py
index 0115d2f..135bbf6 100644 --- a/lnst/Controller/NetTestController.py +++ b/lnst/Controller/NetTestController.py @@ -61,11 +61,10 @@ class NetTestController: self._res_serializer = res_serializer self._remote_capture_files = {} self._log_ctl = log_ctl
self._recipe_path = Path(None, recipe_path).abs_path()
self._recipe_path = Path(None, recipe_path) self._msg_dispatcher = MessageDispatcher(log_ctl) self._packet_capture = packet_capture self._reduce_sync = reduce_sync
self._parser = RecipeParser(recipe_path) self._defined_aliases = defined_aliases self._multi_match = multi_match@@ -80,10 +79,6 @@ class NetTestController: mac_pool_range = lnst_config.get_option('environment',
'mac_pool_range')
self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
self._parser.set_machines(self._machines)self._parser.set_aliases(defined_aliases, overriden_aliases)self._recipe = self._parser.parse()conf_pools = lnst_config.get_pools() pools = {} if len(restrict_pools) > 0:@@ -100,9 +95,6 @@ class NetTestController: sp = SlavePool(pools, pool_checks) self._slave_pool = sp
mreq = self._get_machine_requirements()sp.set_machine_requirements(mreq)modules_dirs = lnst_config.get_option('environment','module_dirs')
tools_dirs = lnst_config.get_option('environment', 'tool_dirs')@@ -196,21 +188,21 @@ class NetTestController: return mreq
def _prepare_network(self, resource_sync=True):
recipe = self._recipe
mreq = Task.get_mreq() machines = self._machines for m_id in machines.keys(): self._prepare_machine(m_id, resource_sync)
for machine_xml_data in recipe["machines"]:m_id = machine_xml_data["id"]
for machine_id, machine_data in mreq.iteritems():m_id = machine_id m = machines[m_id] namespaces = set()
for iface_xml_data in machine_xml_data["interfaces"]:self._prepare_interface(m_id, iface_xml_data)
for if_id, iface_data inmachine_data["interfaces"].iteritems():
self._prepare_interface(m_id, if_id, iface_data)
if iface_xml_data["netns"] != None:namespaces.add(iface_xml_data["netns"])
if iface_data["netns"] != None:namespaces.add(iface_data["netns"]) if len(namespaces) > 0: m.disable_nm()@@ -269,148 +261,16 @@ class NetTestController: machine.set_mac_pool(self._mac_pool) machine.set_network_bridges(self._network_bridges)
recipe_name = os.path.basename(self._recipe_path)
recipe_name = os.path.basename(self._recipe_path.abs_path()) machine.init_connection(recipe_name)
sync_table = {'module': {}, 'tools': {}}if resource_sync:res_table = copy.deepcopy(self._resource_table)for task in self._recipe['tasks']:if 'module_dir' in task:modules =self._load_test_modules([task['module_dir']])
res_table['module'].update(modules)if 'tools_dir' in task:tools = self._load_test_tools([task['tools_dir']])res_table['tools'].update(tools)if 'commands' not in task:if not self._reduce_sync:sync_table = res_tablebreakelse:continuefor cmd in task['commands']:if 'host' not in cmd or cmd['host'] != m_id:continueif cmd['type'] == 'test':mod = cmd['module']if mod in res_table['module']:sync_table['module'][mod] =res_table['module'][mod]
# check if test module uses some test toolsmod_path = res_table['module'][mod]["path"]mod_tools = get_module_tools(mod_path)for t in mod_tools:if t in sync_table['tools']:continuelogging.debug("Adding '%s' tool as "\"dependency of %s test module" % (t,mod))
sync_table['tools'][t] =res_table['tools'][t]
else:msg = "Module '%s' not found on thecontroller"\
% modraise RecipeError(msg, cmd)if cmd['type'] == 'exec' and 'from' in cmd:tool = cmd['from']if tool in res_table['tools']:sync_table['tools'][tool] =res_table['tools'][tool]
else:msg = "Tool '%s' not found on thecontroller" % tool
raise RecipeError(msg, cmd)machine.sync_resources(sync_table)- def _prepare_interface(self, m_id, iface_xml_data):
- def _prepare_interface(self, m_id, if_id, iface_data): machine = self._machines[m_id]
if_id = iface_xml_data["id"]if_type = iface_xml_data["type"]try:iface = machine.get_interface(if_id)except MachineError:if if_type == 'lo':iface = machine.new_loopback_interface(if_id)else:iface = machine.new_soft_interface(if_id, if_type)if "slaves" in iface_xml_data:for slave in iface_xml_data["slaves"]:slave_id = slave["id"]iface.add_slave(machine.get_interface(slave_id))# Some soft devices (such as team) use per-slave optionsif "options" in slave:for opt in slave["options"]:iface.set_slave_option(slave_id, opt["name"],opt["value"])if "addresses" in iface_xml_data:for addr in iface_xml_data["addresses"]:iface.add_address(addr)if "options" in iface_xml_data:for opt in iface_xml_data["options"]:iface.set_option(opt["name"], opt["value"])if "netem" in iface_xml_data:iface.set_netem(iface_xml_data["netem"].to_dict())if "ovs_conf" in iface_xml_data:iface.set_ovs_conf(iface_xml_data["ovs_conf"].to_dict())if iface_xml_data["netns"] != None:iface.set_netns(iface_xml_data["netns"])if "peer" in iface_xml_data:iface.set_peer(iface_xml_data["peer"])def _prepare_tasks(self):
self._tasks = []for task_data in self._recipe["tasks"]:task = {}task["quit_on_fail"] = Falseif "quit_on_fail" in task_data:task["quit_on_fail"] = bool_it(task_data["quit_on_fail"])if "module_dir" in task_data:task["module_dir"] = task_data["module_dir"]if "tools_dir" in task_data:task["tools_dir"] = task_data["tools_dir"]if "python" in task_data:root = Path(None, self._recipe_path).get_root()path = Path(root, task_data["python"])task["python"] = pathif not path.exists():msg = "Task file '%s' not found." % path.to_str()raise RecipeError(msg, task_data)self._tasks.append(task)continuetask["commands"] = task_data["commands"]task["skeleton"] = []for cmd_data in task["commands"]:cmd = {"type": cmd_data["type"]}if "host" in cmd_data:cmd["host"] = cmd_data["host"]if cmd["host"] not in self._machines:msg = "Invalid host id '%s'." % cmd["host"]raise RecipeError(msg, cmd_data)if cmd["type"] in ["test", "exec"]:if "bg_id" in cmd_data:cmd["bg_id"] = cmd_data["bg_id"]elif cmd["type"] in ["wait", "intr", "kill"]:cmd["proc_id"] = cmd_data["bg_id"]task["skeleton"].append(cmd)if self._check_task(task):raise RecipeError("Incorrect command sequence.",task_data)
iface = machine.get_interface(if_id)
self._tasks.append(task)
if iface_data["netns"] != None:iface.set_netns(iface_data["netns"])def _prepare_command(self, cmd_data): cmd = {"type": cmd_data["type"]}
@@ -635,36 +495,23 @@ class NetTestController: os.remove("/tmp/.lnst_machine_conf")
def match_setup(self):
self.run_mode = "match_setup"res = self._run_python_task() return {"passed": True}def config_only_recipe(self):
self.run_mode = "config_only" try:
self._prepare_network(resource_sync=False)except (KeyboardInterrupt, Exception) as exc:msg = "Exception raised during configuration."logging.error(msg)self._cleanup_slaves()
res = self._run_recipe()except Exception as exc:logging.error("Recipe execution terminated by unexpectedexception")
raise
self._save_machine_config()self._cleanup_slaves(deconfigure=False)return {"passed": True}- def run_recipe(self):
try:self._prepare_tasks()self._prepare_network()except (KeyboardInterrupt, Exception) as exc:msg = "Exception raised during configuration."logging.error(msg)
finally: self._cleanup_slaves()
raiseif self._packet_capture:self._start_packet_capture()
return res
err = None
- def run_recipe(self): try: res = self._run_recipe() except Exception as exc:
@@ -699,70 +546,44 @@ class NetTestController: def _run_recipe(self): overall_res = {"passed": True}
for task in self._tasks:
try: self._res_serializer.add_task()
try:res = self._run_task(task)except CommandException as exc:logging.debug(exc)overall_res["passed"] = Falseoverall_res["err_msg"] = "Command exception raised."breakfor machine in self._machines.itervalues():machine.restore_system_config()# task failed, check if we should quit_on_failif not res:overall_res["passed"] = Falseoverall_res["err_msg"] = "At least one command failed."if task["quit_on_fail"]:break
res = self._run_python_task()except CommandException as exc:logging.debug(exc)overall_res["passed"] = Falseoverall_res["err_msg"] = "Command exception raised."for machine in self._machines.itervalues():machine.restore_system_config()# task failedif not res:overall_res["passed"] = Falseoverall_res["err_msg"] = "At least one command failed." return overall_resdef init_taskapi(self): Task.ctl = Task.ControllerAPI(self)
- def _run_python_task(self, task):
- def _run_python_task(self): #backup of resource table res_table_bkp = copy.deepcopy(self._resource_table)
if 'module_dir' in task:modules = self._load_test_modules([task['module_dir']])self._resource_table['module'].update(modules)if 'tools_dir' in task:tools = self._load_test_tools([task['tools_dir']])self._resource_table['tools'].update(tools)# Initialize the API handleTask.ctl = Task.ControllerAPI(self, self._machines) cwd = os.getcwd()task_path = task["python"]
task_path = self._recipe_path name = os.path.basename(task_path.abs_path()).split(".")[0] sys.path.append(os.path.dirname(task_path.resolve())) os.chdir(os.path.dirname(task_path.resolve()))
module = imp.load_source(name, task_path.resolve())
imp.load_source(name, task_path.resolve()) os.chdir(cwd) sys.path.remove(os.path.dirname(task_path.resolve())) #restore resource table self._resource_table = res_table_bkp
return module.ctl._result- def _run_task(self, task):
if "python" in task:return self._run_python_task(task)seq_passed = Truefor cmd_data in task["commands"]:cmd = self._prepare_command(cmd_data)cmd_res = self._run_command(cmd)if not cmd_res["passed"]:seq_passed = Falsereturn seq_passed
return Task.ctl._resultdef _run_command(self, command): logging.info("Executing command: [%s]", str_command(command))
@@ -891,12 +712,11 @@ class NetTestController: return packages
def _get_alias(self, alias):
templates = self._parser._template_procreturn templates._find_definition(alias)
if alias in self._defined_aliases:return self._defined_aliases[alias]def _get_aliases(self):
templates = self._parser._template_procreturn templates._dump_definitions()
return self._defined_aliasesclass MessageDispatcher(ConnectionHandler): def __init__(self, log_ctl): diff --git a/lnst/Controller/Task.py b/lnst/Controller/Task.py index e36b79f..9f40846 100644 --- a/lnst/Controller/Task.py +++ b/lnst/Controller/Task.py @@ -88,7 +88,7 @@ class TaskError(Exception): pass class ControllerAPI(object): """ An API class representing the controller. """
- def __init__(self, ctl, hosts):
- def __init__(self, ctl): self._ctl = ctl self.run_mode = ctl.run_mode self._result = True
@@ -99,8 +99,6 @@ class ControllerAPI(object): self._perf_repo_api = PerfRepoAPI()
self._hosts = {}
for host_id, host in hosts.iteritems():self._hosts[host_id] = HostAPI(self, host_id, host)def get_mreq(self): return self.mreq
@@ -188,7 +186,7 @@ class ControllerAPI(object): cmd = {"type": "ctl_wait", "seconds": int(seconds)} return self._ctl._run_command(cmd)
- def get_alias(self, alias):
- def get_alias(self, alias, default=None): """ Get the value of user defined alias.
@@ -199,9 +197,13 @@ class ControllerAPI(object): :rtype: string """ try:
return self._ctl._get_alias(alias)
val = self._ctl._get_alias(alias)if val is None:return defaultelse:return val except XmlTemplateError:
return None
return defaultdef get_aliases(self): """
@@ -269,16 +271,12 @@ class ControllerAPI(object): class HostAPI(object): """ An API class representing a host machine. """
- def __init__(self, ctl, host_id, host):
- def __init__(self, ctl, host_id): self._ctl = ctl self._id = host_id
self._m = host
self._m = None self._ifaces = {}
for interface in self._m.get_interfaces():if interface.get_id() is None:continueself._ifaces[interface.get_id()] = InterfaceAPI(interface,self)
self._if_id_seq = 0 self._bg_id_seq = 0@@ -313,7 +311,7 @@ class HostAPI(object): return self._ifaces[if_id]
def get_id(self):
return self._m.get_id()
return self._iddef get_configuration(self): return self._m.get_configuration()
-- 2.4.11 _______________________________________________ LNST-developers mailing list lnst-developers@lists.fedorahosted.org
https://lists.fedorahosted.org/admin/lists/lnst-developers@lists.fedorahoste...