Re: [PATCH 12/15] PyRecipes: add PyRecipes support

2016-07-12 14:54 GMT+02:00 Jiri Pirko jiri@resnulli.us:

Tue, Jul 12, 2016 at 11:00:33AM CEST, jprochaz@redhat.com wrote:

...

This patch implements PyRecipes and breaks existing XML recipes.

Python recipes, unlike XML recipes, do not need to be parsed before execution. They contain both network topology definition and test commands.

Network topology is defined in the beginning of PyRecipe, example:

import lnst

m1 = lnst.add_host() m2 = lnst.add_host()

m1_eth1 = m1.add_interface(label="tnet") m2_eth1 = m2.add_interface(label="tnet")

In the example, test will use 2 different hosts, each with 1 eth device.

To run match and network provision, use lnst.match() method in while loop, example:

while lnst.match(): m1_eth1.reset(ip="192.168.0.1/24") m2_eth1.reset(ip="192.168.0.2/24")

Why "reset"? For every match, the config should be cleared.

​I used reset because it was already created (by you I think) We can either create a new method for controlling the interface or use this one.​

...

This piece of code will run the match, prepare the setup and sets IP addresses to both eth devices.

More info on how to use Pyrecipes will be in our documentation (github

wiki).

...

After this patch, XML recipes will no longer work. It removes and modifies a lot of code in order to not break git bisect.

Signed-off-by: Jiri Prochazka jprochaz@redhat.com

lnst-ctl | 13 +- lnst/Controller/NetTestController.py | 266

++++++-----------------------------

...

lnst/Controller/Task.py | 24 ++-- 3 files changed, 62 insertions(+), 241 deletions(-)

diff --git a/lnst-ctl b/lnst-ctl index b75b8a0..93b084b 100755 --- a/lnst-ctl +++ b/lnst-ctl @@ -178,9 +178,13 @@ def get_recipe_result(action, file_path, log_ctl,

res_serializer,

...

    res = {}
    if matches == 1:
        try:

•            nettestctl.provision_machines()
  

•            nettestctl.print_match_description()
  

•            # init TaskAPI.Ctl
  

•            nettestctl.init_taskapi()
            res = exec_action(action, nettestctl)
  

•        except NoMatchError as err:
  

•            no_match = True
  

•            log_ctl.unset_recipe()
  

•            logging.warning("Match %d not possible." % matches)
        except Exception as err:
            no_match = True
            log_exc_traceback()
  

@@ -190,12 +194,11 @@ def get_recipe_result(action, file_path, log_ctl,

res_serializer,

...

            retval = RETVAL_ERR
    elif matches > 1:
        try:

•            nettestctl.provision_machines()
  

•            nettestctl.init_taskapi()
            log_ctl.set_recipe(file_path, prepend=True,
  

expand="match_%d" % matches)

...

            log_dir = log_ctl.get_recipe_log_path()
            recipe_head_log_entry(file_path, log_dir, matches)
            res_serializer.add_recipe(file_path, matches)

•            nettestctl.print_match_description()
            res = exec_action(action, nettestctl)
        except NoMatchError as err:
            no_match = True
  

@@ -400,7 +403,7 @@ def main(): all_files.sort() for f in all_files: recipe_file = os.path.join(recipe_path, f)

•            if re.match(r'^.*\.xml$', recipe_file):
  

•            if re.match(r'^.*\.py$', recipe_file):
                recipe_files.append(recipe_file)
    else:
        recipe_files.append(recipe_path)
  

diff --git a/lnst/Controller/NetTestController.py

b/lnst/Controller/NetTestController.py

...

index 0115d2f..135bbf6 100644 --- a/lnst/Controller/NetTestController.py +++ b/lnst/Controller/NetTestController.py @@ -61,11 +61,10 @@ class NetTestController: self._res_serializer = res_serializer self._remote_capture_files = {} self._log_ctl = log_ctl

•    self._recipe_path = Path(None, recipe_path).abs_path()
  

•    self._recipe_path = Path(None, recipe_path)
    self._msg_dispatcher = MessageDispatcher(log_ctl)
    self._packet_capture = packet_capture
    self._reduce_sync = reduce_sync
  

•    self._parser = RecipeParser(recipe_path)
    self._defined_aliases = defined_aliases
    self._multi_match = multi_match
  
  

@@ -80,10 +79,6 @@ class NetTestController: mac_pool_range = lnst_config.get_option('environment',

'mac_pool_range')

...

    self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])

•    self._parser.set_machines(self._machines)
  

•    self._parser.set_aliases(defined_aliases, overriden_aliases)
  

•    self._recipe = self._parser.parse()
  

•   conf_pools = lnst_config.get_pools()
    pools = {}
    if len(restrict_pools) > 0:
  

@@ -100,9 +95,6 @@ class NetTestController: sp = SlavePool(pools, pool_checks) self._slave_pool = sp

•    mreq = self._get_machine_requirements()
  

•    sp.set_machine_requirements(mreq)
  

•   modules_dirs = lnst_config.get_option('environment',
  

'module_dirs')

...

    tools_dirs = lnst_config.get_option('environment', 'tool_dirs')

@@ -196,21 +188,21 @@ class NetTestController: return mreq

def _prepare_network(self, resource_sync=True):

•    recipe = self._recipe
  

•    mreq = Task.get_mreq()
  
    machines = self._machines
    for m_id in machines.keys():
        self._prepare_machine(m_id, resource_sync)
  
  

•    for machine_xml_data in recipe["machines"]:
  

•        m_id = machine_xml_data["id"]
  

•    for machine_id, machine_data in mreq.iteritems():
  

•        m_id = machine_id
        m = machines[m_id]
        namespaces = set()
  

•        for iface_xml_data in machine_xml_data["interfaces"]:
  

•            self._prepare_interface(m_id, iface_xml_data)
  

•        for if_id, iface_data in
  

machine_data["interfaces"].iteritems():

...

•            self._prepare_interface(m_id, if_id, iface_data)
  
  

•            if iface_xml_data["netns"] != None:
  

•                namespaces.add(iface_xml_data["netns"])
  

•            if iface_data["netns"] != None:
  

•                namespaces.add(iface_data["netns"])
  
        if len(namespaces) > 0:
            m.disable_nm()
  

@@ -269,148 +261,16 @@ class NetTestController: machine.set_mac_pool(self._mac_pool) machine.set_network_bridges(self._network_bridges)

•    recipe_name = os.path.basename(self._recipe_path)
  

•    recipe_name = os.path.basename(self._recipe_path.abs_path())
    machine.init_connection(recipe_name)
  
  

•    sync_table = {'module': {}, 'tools': {}}
  

•    if resource_sync:
  

•        res_table = copy.deepcopy(self._resource_table)
  

•        for task in self._recipe['tasks']:
  

•            if 'module_dir' in task:
  

•                modules =
  

self._load_test_modules([task['module_dir']])

...

•                res_table['module'].update(modules)
  

•            if 'tools_dir' in task:
  

•                tools = self._load_test_tools([task['tools_dir']])
  

•                res_table['tools'].update(tools)
  

•            if 'commands' not in task:
  

•                if not self._reduce_sync:
  

•                    sync_table = res_table
  

•                    break
  

•                else:
  

•                    continue
  

•            for cmd in task['commands']:
  

•                if 'host' not in cmd or cmd['host'] != m_id:
  

•                    continue
  

•                if cmd['type'] == 'test':
  

•                    mod = cmd['module']
  

•                    if mod in res_table['module']:
  

•                        sync_table['module'][mod] =
  

res_table['module'][mod]

...

•                        # check if test module uses some test tools
  

•                        mod_path = res_table['module'][mod]["path"]
  

•                        mod_tools = get_module_tools(mod_path)
  

•                        for t in mod_tools:
  

•                            if t in sync_table['tools']:
  

•                                continue
  

•                            logging.debug("Adding '%s' tool as "\
  

•                                "dependency of %s test module" % (t,
  

mod))

...

•                            sync_table['tools'][t] =
  

res_table['tools'][t]

...

•                    else:
  

•                        msg = "Module '%s' not found on the
  

controller"\

...

•                                % mod
  

•                        raise RecipeError(msg, cmd)
  

•                if cmd['type'] == 'exec' and 'from' in cmd:
  

•                    tool = cmd['from']
  

•                    if tool in res_table['tools']:
  

•                        sync_table['tools'][tool] =
  

res_table['tools'][tool]

...

•                    else:
  

•                        msg = "Tool '%s' not found on the
  

controller" % tool

...

•                        raise RecipeError(msg, cmd)
  

•    machine.sync_resources(sync_table)
  

• def _prepare_interface(self, m_id, iface_xml_data):

• def _prepare_interface(self, m_id, if_id, iface_data): machine = self._machines[m_id]

•    if_id = iface_xml_data["id"]
  

•    if_type = iface_xml_data["type"]
  

•    try:
  

•        iface = machine.get_interface(if_id)
  

•    except MachineError:
  

•        if if_type == 'lo':
  

•            iface = machine.new_loopback_interface(if_id)
  

•        else:
  

•            iface = machine.new_soft_interface(if_id, if_type)
  

•    if "slaves" in iface_xml_data:
  

•        for slave in iface_xml_data["slaves"]:
  

•            slave_id = slave["id"]
  

•            iface.add_slave(machine.get_interface(slave_id))
  

•            # Some soft devices (such as team) use per-slave options
  

•            if "options" in slave:
  

•                for opt in slave["options"]:
  

•                    iface.set_slave_option(slave_id, opt["name"],
  

•                                           opt["value"])
  

•    if "addresses" in iface_xml_data:
  

•        for addr in iface_xml_data["addresses"]:
  

•            iface.add_address(addr)
  

•    if "options" in iface_xml_data:
  

•        for opt in iface_xml_data["options"]:
  

•            iface.set_option(opt["name"], opt["value"])
  

•    if "netem" in iface_xml_data:
  

•        iface.set_netem(iface_xml_data["netem"].to_dict())
  

•    if "ovs_conf" in iface_xml_data:
  

•        iface.set_ovs_conf(iface_xml_data["ovs_conf"].to_dict())
  

•    if iface_xml_data["netns"] != None:
  

•        iface.set_netns(iface_xml_data["netns"])
  

•    if "peer" in iface_xml_data:
  

•        iface.set_peer(iface_xml_data["peer"])
  

• def _prepare_tasks(self):

•    self._tasks = []
  

•    for task_data in self._recipe["tasks"]:
  

•        task = {}
  

•        task["quit_on_fail"] = False
  

•        if "quit_on_fail" in task_data:
  

•            task["quit_on_fail"] = bool_it(task_data["quit_on_fail"])
  

•        if "module_dir" in task_data:
  

•            task["module_dir"] = task_data["module_dir"]
  

•        if "tools_dir" in task_data:
  

•            task["tools_dir"] = task_data["tools_dir"]
  

•        if "python" in task_data:
  

•            root = Path(None, self._recipe_path).get_root()
  

•            path = Path(root, task_data["python"])
  

•            task["python"] = path
  

•            if not path.exists():
  

•                msg = "Task file '%s' not found." % path.to_str()
  

•                raise RecipeError(msg, task_data)
  

•            self._tasks.append(task)
  

•            continue
  

•        task["commands"] = task_data["commands"]
  

•        task["skeleton"] = []
  

•        for cmd_data in task["commands"]:
  

•            cmd = {"type": cmd_data["type"]}
  

•            if "host" in cmd_data:
  

•                cmd["host"] = cmd_data["host"]
  

•                if cmd["host"] not in self._machines:
  

•                    msg = "Invalid host id '%s'." % cmd["host"]
  

•                    raise RecipeError(msg, cmd_data)
  

•            if cmd["type"] in ["test", "exec"]:
  

•                if "bg_id" in cmd_data:
  

•                    cmd["bg_id"] = cmd_data["bg_id"]
  

•            elif cmd["type"] in ["wait", "intr", "kill"]:
  

•                cmd["proc_id"] = cmd_data["bg_id"]
  

•            task["skeleton"].append(cmd)
  
  

•        if self._check_task(task):
  

•            raise RecipeError("Incorrect command sequence.",
  

task_data)

...

•    iface = machine.get_interface(if_id)
  
  

•        self._tasks.append(task)
  

•    if iface_data["netns"] != None:
  

•        iface.set_netns(iface_data["netns"])
  
  

  def _prepare_command(self, cmd_data): cmd = {"type": cmd_data["type"]}

@@ -635,36 +495,23 @@ class NetTestController: os.remove("/tmp/.lnst_machine_conf")

def match_setup(self):

•    self.run_mode = "match_setup"
  

•    res = self._run_python_task()
    return {"passed": True}
  
  

  def config_only_recipe(self):

•    self.run_mode = "config_only"
    try:
  

•        self._prepare_network(resource_sync=False)
  

•    except (KeyboardInterrupt, Exception) as exc:
  

•        msg = "Exception raised during configuration."
  

•        logging.error(msg)
  

•        self._cleanup_slaves()
  

•        res = self._run_recipe()
  

•    except Exception as exc:
  

•        logging.error("Recipe execution terminated by unexpected
  

exception")

...

        raise

•    self._save_machine_config()
  

•    self._cleanup_slaves(deconfigure=False)
  

•    return {"passed": True}
  

• def run_recipe(self):

•    try:
  

•        self._prepare_tasks()
  

•        self._prepare_network()
  

•    except (KeyboardInterrupt, Exception) as exc:
  

•        msg = "Exception raised during configuration."
  

•        logging.error(msg)
  

•    finally:
        self._cleanup_slaves()
  

•        raise
  
  

•    if self._packet_capture:
  

•        self._start_packet_capture()
  

•    return res
  
  

•    err = None
  

• def run_recipe(self): try: res = self._run_recipe() except Exception as exc:

@@ -699,70 +546,44 @@ class NetTestController: def _run_recipe(self): overall_res = {"passed": True}

•    for task in self._tasks:
  

•    try:
        self._res_serializer.add_task()
  

•        try:
  

•            res = self._run_task(task)
  

•        except CommandException as exc:
  

•            logging.debug(exc)
  

•            overall_res["passed"] = False
  

•            overall_res["err_msg"] = "Command exception raised."
  

•            break
  

•        for machine in self._machines.itervalues():
  

•            machine.restore_system_config()
  

•        # task failed, check if we should quit_on_fail
  

•        if not res:
  

•            overall_res["passed"] = False
  

•            overall_res["err_msg"] = "At least one command failed."
  

•            if task["quit_on_fail"]:
  

•                break
  

•        res = self._run_python_task()
  

•    except CommandException as exc:
  

•        logging.debug(exc)
  

•        overall_res["passed"] = False
  

•        overall_res["err_msg"] = "Command exception raised."
  

•    for machine in self._machines.itervalues():
  

•        machine.restore_system_config()
  

•    # task failed
  

•    if not res:
  

•        overall_res["passed"] = False
  

•        overall_res["err_msg"] = "At least one command failed."
  
    return overall_res
  
  

  def init_taskapi(self): Task.ctl = Task.ControllerAPI(self)

• def _run_python_task(self, task):

• def _run_python_task(self): #backup of resource table res_table_bkp = copy.deepcopy(self._resource_table)

•    if 'module_dir' in task:
  

•        modules = self._load_test_modules([task['module_dir']])
  

•        self._resource_table['module'].update(modules)
  

•    if 'tools_dir' in task:
  

•        tools = self._load_test_tools([task['tools_dir']])
  

•        self._resource_table['tools'].update(tools)
  

•    # Initialize the API handle
  

•    Task.ctl = Task.ControllerAPI(self, self._machines)
  
    cwd = os.getcwd()
  

•    task_path = task["python"]
  

•    task_path = self._recipe_path
    name = os.path.basename(task_path.abs_path()).split(".")[0]
    sys.path.append(os.path.dirname(task_path.resolve()))
    os.chdir(os.path.dirname(task_path.resolve()))
  

•    module = imp.load_source(name, task_path.resolve())
  

•    imp.load_source(name, task_path.resolve())
    os.chdir(cwd)
    sys.path.remove(os.path.dirname(task_path.resolve()))
  
    #restore resource table
    self._resource_table = res_table_bkp
  
  

•    return module.ctl._result
  

• def _run_task(self, task):

•    if "python" in task:
  

•        return self._run_python_task(task)
  

•    seq_passed = True
  

•    for cmd_data in task["commands"]:
  

•        cmd = self._prepare_command(cmd_data)
  

•        cmd_res = self._run_command(cmd)
  

•        if not cmd_res["passed"]:
  

•            seq_passed = False
  

•    return seq_passed
  

•    return Task.ctl._result
  
  

  def _run_command(self, command): logging.info("Executing command: [%s]", str_command(command))

@@ -891,12 +712,11 @@ class NetTestController: return packages

def _get_alias(self, alias):

•    templates = self._parser._template_proc
  

•    return templates._find_definition(alias)
  

•    if alias in self._defined_aliases:
  

•        return self._defined_aliases[alias]
  
  

  def _get_aliases(self):

•    templates = self._parser._template_proc
  

•    return templates._dump_definitions()
  

•    return self._defined_aliases
  
  

class MessageDispatcher(ConnectionHandler): def __init__(self, log_ctl): diff --git a/lnst/Controller/Task.py b/lnst/Controller/Task.py index e36b79f..9f40846 100644 --- a/lnst/Controller/Task.py +++ b/lnst/Controller/Task.py @@ -88,7 +88,7 @@ class TaskError(Exception): pass class ControllerAPI(object): """ An API class representing the controller. """

• def __init__(self, ctl, hosts):

• def __init__(self, ctl): self._ctl = ctl self.run_mode = ctl.run_mode self._result = True

@@ -99,8 +99,6 @@ class ControllerAPI(object): self._perf_repo_api = PerfRepoAPI()

    self._hosts = {}

•    for host_id, host in hosts.iteritems():
  

•        self._hosts[host_id] = HostAPI(self, host_id, host)
  
  

  def get_mreq(self): return self.mreq

@@ -188,7 +186,7 @@ class ControllerAPI(object): cmd = {"type": "ctl_wait", "seconds": int(seconds)} return self._ctl._run_command(cmd)

• def get_alias(self, alias):

• def get_alias(self, alias, default=None): """ Get the value of user defined alias.

@@ -199,9 +197,13 @@ class ControllerAPI(object): :rtype: string """ try:

•        return self._ctl._get_alias(alias)
  

•        val =  self._ctl._get_alias(alias)
  

•        if val is None:
  

•            return default
  

•        else:
  

•            return val
    except XmlTemplateError:
  

•        return None
  

•        return default
  
  

  def get_aliases(self): """

@@ -269,16 +271,12 @@ class ControllerAPI(object): class HostAPI(object): """ An API class representing a host machine. """

• def __init__(self, ctl, host_id, host):

• def __init__(self, ctl, host_id): self._ctl = ctl self._id = host_id

•    self._m = host
  

•    self._m = None
  
    self._ifaces = {}
  

•    for interface in self._m.get_interfaces():
  

•        if interface.get_id() is None:
  

•            continue
  

•        self._ifaces[interface.get_id()] = InterfaceAPI(interface,
  

self)

...

    self._if_id_seq = 0
    self._bg_id_seq = 0

@@ -313,7 +311,7 @@ class HostAPI(object): return self._ifaces[if_id]

def get_id(self):

•    return self._m.get_id()
  

•    return self._id
  
  

  def get_configuration(self): return self._m.get_configuration()

-- 2.4.11 _______________________________________________ LNST-developers mailing list lnst-developers@lists.fedorahosted.org

https://lists.fedorahosted.org/admin/lists/lnst-developers@lists.fedorahoste...