2016-07-12 14:54 GMT+02:00 Jiri Pirko <jiri@resnulli.us>:
Tue, Jul 12, 2016 at 11:00:33AM CEST, jprochaz@redhat.com wrote:
>This patch implements PyRecipes and breaks existing XML recipes.
>
>Python recipes, unlike XML recipes, do not need to be parsed before
>execution. They contain both network topology definition and test
>commands.
>
>Network topology is defined in the beginning of PyRecipe, example:
>
>import lnst
>
>m1 = lnst.add_host()
>m2 = lnst.add_host()
>
>m1_eth1  = m1.add_interface(label="tnet")
>m2_eth1  = m2.add_interface(label="tnet")
>
>In the example, test will use 2 different hosts, each with 1 eth device.
>
>To run match and network provision, use lnst.match() method in while
>loop, example:
>
>while lnst.match():
>    m1_eth1.reset(ip="192.168.0.1/24")
>    m2_eth1.reset(ip="192.168.0.2/24")


Why "reset"? For every match, the config should be cleared.

​This (interface handling) should also be one of the items at our API discussion in August.



>
>This piece of code will run the match, prepare the setup and sets IP
>addresses to both eth devices.
>
>More info on how to use Pyrecipes will be in our documentation (github wiki).
>
>After this patch, XML recipes will no longer work. It removes and
>modifies a lot of code in order to not break git bisect.
>
>Signed-off-by: Jiri Prochazka <jprochaz@redhat.com>
>---
> lnst-ctl                             |  13 +-
> lnst/Controller/NetTestController.py | 266 ++++++-----------------------------
> lnst/Controller/Task.py              |  24 ++--
> 3 files changed, 62 insertions(+), 241 deletions(-)
>
>diff --git a/lnst-ctl b/lnst-ctl
>index b75b8a0..93b084b 100755
>--- a/lnst-ctl
>+++ b/lnst-ctl
>@@ -178,9 +178,13 @@ def get_recipe_result(action, file_path, log_ctl, res_serializer,
>         res = {}
>         if matches == 1:
>             try:
>-                nettestctl.provision_machines()
>-                nettestctl.print_match_description()
>+                # init TaskAPI.Ctl
>+                nettestctl.init_taskapi()
>                 res = exec_action(action, nettestctl)
>+            except NoMatchError as err:
>+                no_match = True
>+                log_ctl.unset_recipe()
>+                logging.warning("Match %d not possible." % matches)
>             except Exception as err:
>                 no_match = True
>                 log_exc_traceback()
>@@ -190,12 +194,11 @@ def get_recipe_result(action, file_path, log_ctl, res_serializer,
>                 retval = RETVAL_ERR
>         elif matches > 1:
>             try:
>-                nettestctl.provision_machines()
>+                nettestctl.init_taskapi()
>                 log_ctl.set_recipe(file_path, prepend=True, expand="match_%d" % matches)
>                 log_dir = log_ctl.get_recipe_log_path()
>                 recipe_head_log_entry(file_path, log_dir, matches)
>                 res_serializer.add_recipe(file_path, matches)
>-                nettestctl.print_match_description()
>                 res = exec_action(action, nettestctl)
>             except NoMatchError as err:
>                 no_match = True
>@@ -400,7 +403,7 @@ def main():
>             all_files.sort()
>             for f in all_files:
>                 recipe_file = os.path.join(recipe_path, f)
>-                if re.match(r'^.*\.xml$', recipe_file):
>+                if re.match(r'^.*\.py$', recipe_file):
>                     recipe_files.append(recipe_file)
>         else:
>             recipe_files.append(recipe_path)
>diff --git a/lnst/Controller/NetTestController.py b/lnst/Controller/NetTestController.py
>index 0115d2f..135bbf6 100644
>--- a/lnst/Controller/NetTestController.py
>+++ b/lnst/Controller/NetTestController.py
>@@ -61,11 +61,10 @@ class NetTestController:
>         self._res_serializer = res_serializer
>         self._remote_capture_files = {}
>         self._log_ctl = log_ctl
>-        self._recipe_path = Path(None, recipe_path).abs_path()
>+        self._recipe_path = Path(None, recipe_path)
>         self._msg_dispatcher = MessageDispatcher(log_ctl)
>         self._packet_capture = packet_capture
>         self._reduce_sync = reduce_sync
>-        self._parser = RecipeParser(recipe_path)
>         self._defined_aliases = defined_aliases
>         self._multi_match = multi_match
>
>@@ -80,10 +79,6 @@ class NetTestController:
>         mac_pool_range = lnst_config.get_option('environment', 'mac_pool_range')
>         self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
>
>-        self._parser.set_machines(self._machines)
>-        self._parser.set_aliases(defined_aliases, overriden_aliases)
>-        self._recipe = self._parser.parse()
>-
>         conf_pools = lnst_config.get_pools()
>         pools = {}
>         if len(restrict_pools) > 0:
>@@ -100,9 +95,6 @@ class NetTestController:
>         sp = SlavePool(pools, pool_checks)
>         self._slave_pool = sp
>
>-        mreq = self._get_machine_requirements()
>-        sp.set_machine_requirements(mreq)
>-
>         modules_dirs = lnst_config.get_option('environment', 'module_dirs')
>         tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
>
>@@ -196,21 +188,21 @@ class NetTestController:
>         return mreq
>
>     def _prepare_network(self, resource_sync=True):
>-        recipe = self._recipe
>+        mreq = Task.get_mreq()
>
>         machines = self._machines
>         for m_id in machines.keys():
>             self._prepare_machine(m_id, resource_sync)
>
>-        for machine_xml_data in recipe["machines"]:
>-            m_id = machine_xml_data["id"]
>+        for machine_id, machine_data in mreq.iteritems():
>+            m_id = machine_id
>             m = machines[m_id]
>             namespaces = set()
>-            for iface_xml_data in machine_xml_data["interfaces"]:
>-                self._prepare_interface(m_id, iface_xml_data)
>+            for if_id, iface_data in machine_data["interfaces"].iteritems():
>+                self._prepare_interface(m_id, if_id, iface_data)
>
>-                if iface_xml_data["netns"] != None:
>-                    namespaces.add(iface_xml_data["netns"])
>+                if iface_data["netns"] != None:
>+                    namespaces.add(iface_data["netns"])
>
>             if len(namespaces) > 0:
>                 m.disable_nm()
>@@ -269,148 +261,16 @@ class NetTestController:
>         machine.set_mac_pool(self._mac_pool)
>         machine.set_network_bridges(self._network_bridges)
>
>-        recipe_name = os.path.basename(self._recipe_path)
>+        recipe_name = os.path.basename(self._recipe_path.abs_path())
>         machine.init_connection(recipe_name)
>
>-        sync_table = {'module': {}, 'tools': {}}
>-        if resource_sync:
>-            res_table = copy.deepcopy(self._resource_table)
>-            for task in self._recipe['tasks']:
>-                if 'module_dir' in task:
>-                    modules = self._load_test_modules([task['module_dir']])
>-                    res_table['module'].update(modules)
>-                if 'tools_dir' in task:
>-                    tools = self._load_test_tools([task['tools_dir']])
>-                    res_table['tools'].update(tools)
>-
>-                if 'commands' not in task:
>-                    if not self._reduce_sync:
>-                        sync_table = res_table
>-                        break
>-                    else:
>-                        continue
>-                for cmd in task['commands']:
>-                    if 'host' not in cmd or cmd['host'] != m_id:
>-                        continue
>-                    if cmd['type'] == 'test':
>-                        mod = cmd['module']
>-                        if mod in res_table['module']:
>-                            sync_table['module'][mod] = res_table['module'][mod]
>-                            # check if test module uses some test tools
>-                            mod_path = res_table['module'][mod]["path"]
>-                            mod_tools = get_module_tools(mod_path)
>-                            for t in mod_tools:
>-                                if t in sync_table['tools']:
>-                                    continue
>-                                logging.debug("Adding '%s' tool as "\
>-                                    "dependency of %s test module" % (t, mod))
>-                                sync_table['tools'][t] = res_table['tools'][t]
>-                        else:
>-                            msg = "Module '%s' not found on the controller"\
>-                                    % mod
>-                            raise RecipeError(msg, cmd)
>-                    if cmd['type'] == 'exec' and 'from' in cmd:
>-                        tool = cmd['from']
>-                        if tool in res_table['tools']:
>-                            sync_table['tools'][tool] = res_table['tools'][tool]
>-                        else:
>-                            msg = "Tool '%s' not found on the controller" % tool
>-                            raise RecipeError(msg, cmd)
>-        machine.sync_resources(sync_table)
>-
>-    def _prepare_interface(self, m_id, iface_xml_data):
>+    def _prepare_interface(self, m_id, if_id, iface_data):
>         machine = self._machines[m_id]
>-        if_id = iface_xml_data["id"]
>-        if_type = iface_xml_data["type"]
>-
>-        try:
>-            iface = machine.get_interface(if_id)
>-        except MachineError:
>-            if if_type == 'lo':
>-                iface = machine.new_loopback_interface(if_id)
>-            else:
>-                iface = machine.new_soft_interface(if_id, if_type)
>-
>-        if "slaves" in iface_xml_data:
>-            for slave in iface_xml_data["slaves"]:
>-                slave_id = slave["id"]
>-                iface.add_slave(machine.get_interface(slave_id))
>-
>-                # Some soft devices (such as team) use per-slave options
>-                if "options" in slave:
>-                    for opt in slave["options"]:
>-                        iface.set_slave_option(slave_id, opt["name"],
>-                                               opt["value"])
>-
>-        if "addresses" in iface_xml_data:
>-            for addr in iface_xml_data["addresses"]:
>-                iface.add_address(addr)
>-
>-        if "options" in iface_xml_data:
>-            for opt in iface_xml_data["options"]:
>-                iface.set_option(opt["name"], opt["value"])
>-
>-        if "netem" in iface_xml_data:
>-            iface.set_netem(iface_xml_data["netem"].to_dict())
>-
>-        if "ovs_conf" in iface_xml_data:
>-            iface.set_ovs_conf(iface_xml_data["ovs_conf"].to_dict())
>-
>-        if iface_xml_data["netns"] != None:
>-            iface.set_netns(iface_xml_data["netns"])
>-
>-        if "peer" in iface_xml_data:
>-            iface.set_peer(iface_xml_data["peer"])
>-
>-    def _prepare_tasks(self):
>-        self._tasks = []
>-        for task_data in self._recipe["tasks"]:
>-            task = {}
>-            task["quit_on_fail"] = False
>-            if "quit_on_fail" in task_data:
>-                task["quit_on_fail"] = bool_it(task_data["quit_on_fail"])
>-
>-            if "module_dir" in task_data:
>-                task["module_dir"] = task_data["module_dir"]
>-
>-            if "tools_dir" in task_data:
>-                task["tools_dir"] = task_data["tools_dir"]
>-
>-            if "python" in task_data:
>-                root = Path(None, self._recipe_path).get_root()
>-                path = Path(root, task_data["python"])
>-
>-                task["python"] = path
>-                if not path.exists():
>-                    msg = "Task file '%s' not found." % path.to_str()
>-                    raise RecipeError(msg, task_data)
>-
>-                self._tasks.append(task)
>-                continue
>-
>-            task["commands"] = task_data["commands"]
>-            task["skeleton"] = []
>-            for cmd_data in task["commands"]:
>-                cmd = {"type": cmd_data["type"]}
>-
>-                if "host" in cmd_data:
>-                    cmd["host"] = cmd_data["host"]
>-                    if cmd["host"] not in self._machines:
>-                        msg = "Invalid host id '%s'." % cmd["host"]
>-                        raise RecipeError(msg, cmd_data)
>-
>-                if cmd["type"] in ["test", "exec"]:
>-                    if "bg_id" in cmd_data:
>-                        cmd["bg_id"] = cmd_data["bg_id"]
>-                elif cmd["type"] in ["wait", "intr", "kill"]:
>-                    cmd["proc_id"] = cmd_data["bg_id"]
>-
>-                task["skeleton"].append(cmd)
>
>-            if self._check_task(task):
>-                raise RecipeError("Incorrect command sequence.", task_data)
>+        iface = machine.get_interface(if_id)
>
>-            self._tasks.append(task)
>+        if iface_data["netns"] != None:
>+            iface.set_netns(iface_data["netns"])
>
>     def _prepare_command(self, cmd_data):
>         cmd = {"type": cmd_data["type"]}
>@@ -635,36 +495,23 @@ class NetTestController:
>             os.remove("/tmp/.lnst_machine_conf")
>
>     def match_setup(self):
>+        self.run_mode = "match_setup"
>+        res = self._run_python_task()
>         return {"passed": True}
>
>     def config_only_recipe(self):
>+        self.run_mode = "config_only"
>         try:
>-            self._prepare_network(resource_sync=False)
>-        except (KeyboardInterrupt, Exception) as exc:
>-            msg = "Exception raised during configuration."
>-            logging.error(msg)
>-            self._cleanup_slaves()
>+            res = self._run_recipe()
>+        except Exception as exc:
>+            logging.error("Recipe execution terminated by unexpected exception")
>             raise
>-
>-        self._save_machine_config()
>-
>-        self._cleanup_slaves(deconfigure=False)
>-        return {"passed": True}
>-
>-    def run_recipe(self):
>-        try:
>-            self._prepare_tasks()
>-            self._prepare_network()
>-        except (KeyboardInterrupt, Exception) as exc:
>-            msg = "Exception raised during configuration."
>-            logging.error(msg)
>+        finally:
>             self._cleanup_slaves()
>-            raise
>
>-        if self._packet_capture:
>-            self._start_packet_capture()
>+        return res
>
>-        err = None
>+    def run_recipe(self):
>         try:
>             res = self._run_recipe()
>         except Exception as exc:
>@@ -699,70 +546,44 @@ class NetTestController:
>     def _run_recipe(self):
>         overall_res = {"passed": True}
>
>-        for task in self._tasks:
>+        try:
>             self._res_serializer.add_task()
>-            try:
>-                res = self._run_task(task)
>-            except CommandException as exc:
>-                logging.debug(exc)
>-                overall_res["passed"] = False
>-                overall_res["err_msg"] = "Command exception raised."
>-                break
>-
>-            for machine in self._machines.itervalues():
>-                machine.restore_system_config()
>-
>-            # task failed, check if we should quit_on_fail
>-            if not res:
>-                overall_res["passed"] = False
>-                overall_res["err_msg"] = "At least one command failed."
>-                if task["quit_on_fail"]:
>-                    break
>+            res = self._run_python_task()
>+        except CommandException as exc:
>+            logging.debug(exc)
>+            overall_res["passed"] = False
>+            overall_res["err_msg"] = "Command exception raised."
>+
>+        for machine in self._machines.itervalues():
>+            machine.restore_system_config()
>+
>+        # task failed
>+        if not res:
>+            overall_res["passed"] = False
>+            overall_res["err_msg"] = "At least one command failed."
>
>         return overall_res
>
>     def init_taskapi(self):
>         Task.ctl = Task.ControllerAPI(self)
>
>-    def _run_python_task(self, task):
>+    def _run_python_task(self):
>         #backup of resource table
>         res_table_bkp = copy.deepcopy(self._resource_table)
>-        if 'module_dir' in task:
>-            modules = self._load_test_modules([task['module_dir']])
>-            self._resource_table['module'].update(modules)
>-        if 'tools_dir' in task:
>-            tools = self._load_test_tools([task['tools_dir']])
>-            self._resource_table['tools'].update(tools)
>-
>-        # Initialize the API handle
>-        Task.ctl = Task.ControllerAPI(self, self._machines)
>
>         cwd = os.getcwd()
>-        task_path = task["python"]
>+        task_path = self._recipe_path
>         name = os.path.basename(task_path.abs_path()).split(".")[0]
>         sys.path.append(os.path.dirname(task_path.resolve()))
>         os.chdir(os.path.dirname(task_path.resolve()))
>-        module = imp.load_source(name, task_path.resolve())
>+        imp.load_source(name, task_path.resolve())
>         os.chdir(cwd)
>         sys.path.remove(os.path.dirname(task_path.resolve()))
>
>         #restore resource table
>         self._resource_table = res_table_bkp
>
>-        return module.ctl._result
>-
>-    def _run_task(self, task):
>-        if "python" in task:
>-            return self._run_python_task(task)
>-
>-        seq_passed = True
>-        for cmd_data in task["commands"]:
>-            cmd = self._prepare_command(cmd_data)
>-            cmd_res = self._run_command(cmd)
>-            if not cmd_res["passed"]:
>-                seq_passed = False
>-
>-        return seq_passed
>+        return Task.ctl._result
>
>     def _run_command(self, command):
>         logging.info("Executing command: [%s]", str_command(command))
>@@ -891,12 +712,11 @@ class NetTestController:
>         return packages
>
>     def _get_alias(self, alias):
>-        templates = self._parser._template_proc
>-        return templates._find_definition(alias)
>+        if alias in self._defined_aliases:
>+            return self._defined_aliases[alias]
>
>     def _get_aliases(self):
>-        templates = self._parser._template_proc
>-        return templates._dump_definitions()
>+        return self._defined_aliases
>
> class MessageDispatcher(ConnectionHandler):
>     def __init__(self, log_ctl):
>diff --git a/lnst/Controller/Task.py b/lnst/Controller/Task.py
>index e36b79f..9f40846 100644
>--- a/lnst/Controller/Task.py
>+++ b/lnst/Controller/Task.py
>@@ -88,7 +88,7 @@ class TaskError(Exception): pass
> class ControllerAPI(object):
>     """ An API class representing the controller. """
>
>-    def __init__(self, ctl, hosts):
>+    def __init__(self, ctl):
>         self._ctl = ctl
>         self.run_mode = ctl.run_mode
>         self._result = True
>@@ -99,8 +99,6 @@ class ControllerAPI(object):
>         self._perf_repo_api = PerfRepoAPI()
>
>         self._hosts = {}
>-        for host_id, host in hosts.iteritems():
>-            self._hosts[host_id] = HostAPI(self, host_id, host)
>
>     def get_mreq(self):
>         return self.mreq
>@@ -188,7 +186,7 @@ class ControllerAPI(object):
>         cmd = {"type": "ctl_wait", "seconds": int(seconds)}
>         return self._ctl._run_command(cmd)
>
>-    def get_alias(self, alias):
>+    def get_alias(self, alias, default=None):
>         """
>             Get the value of user defined alias.
>
>@@ -199,9 +197,13 @@ class ControllerAPI(object):
>             :rtype: string
>         """
>         try:
>-            return self._ctl._get_alias(alias)
>+            val =  self._ctl._get_alias(alias)
>+            if val is None:
>+                return default
>+            else:
>+                return val
>         except XmlTemplateError:
>-            return None
>+            return default
>
>     def get_aliases(self):
>         """
>@@ -269,16 +271,12 @@ class ControllerAPI(object):
> class HostAPI(object):
>     """ An API class representing a host machine. """
>
>-    def __init__(self, ctl, host_id, host):
>+    def __init__(self, ctl, host_id):
>         self._ctl = ctl
>         self._id = host_id
>-        self._m = host
>+        self._m = None
>
>         self._ifaces = {}
>-        for interface in self._m.get_interfaces():
>-            if interface.get_id() is None:
>-                continue
>-            self._ifaces[interface.get_id()] = InterfaceAPI(interface, self)
>         self._if_id_seq = 0
>         self._bg_id_seq = 0
>
>@@ -313,7 +311,7 @@ class HostAPI(object):
>         return self._ifaces[if_id]
>
>     def get_id(self):
>-        return self._m.get_id()
>+        return self._id
>
>     def get_configuration(self):
>         return self._m.get_configuration()
>--
>2.4.11
>_______________________________________________
>LNST-developers mailing list
>lnst-developers@lists.fedorahosted.org
>https://lists.fedorahosted.org/admin/lists/lnst-developers@lists.fedorahosted.org