[lnst] ctl: Slightly changing the interface of SlavePool
by Jiří Pírko
commit fb52d65c40ad2e6752876a070007c68ae385c49f
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:41:02 2013 +0200
ctl: Slightly changing the interface of SlavePool
This patch contains a small change in the provision_machines() function
that is used to find a match for a network setup. The dictionary for
results is no longer constructed within the object; it is pased to the
function via a parameter and the machines are filled in.
The method then returns a boolean value that indicates whether the
matching was successful or not.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Controller/SlavePool.py | 17 +++++++----------
1 files changed, 7 insertions(+), 10 deletions(-)
---
diff --git a/lnst/Controller/SlavePool.py b/lnst/Controller/SlavePool.py
index f621d89..e8baf83 100644
--- a/lnst/Controller/SlavePool.py
+++ b/lnst/Controller/SlavePool.py
@@ -68,9 +68,7 @@ class SlavePool:
m_id = re.sub("\.[xX][mM][lL]$", "", basename)
slavemachine = dom.getElementsByTagName("slavemachine")[0]
- xml_data = XmlData(slavemachine)
- parser.set_target(xml_data)
- parser.parse(slavemachine)
+ xml_data = parser.parse(slavemachine)
machine_spec = self._process_machine_xml_data(m_id, xml_data)
@@ -155,7 +153,7 @@ class SlavePool:
return iface_spec
- def provision_machines(self, mreqs):
+ def provision_machines(self, mreqs, machines):
"""
This method will try to map a dictionary of machines'
requirements to a pool of machines that is available to
@@ -172,9 +170,8 @@ class SlavePool:
self._map = mapper.map_setup(mreqs, self._pool)
if self._map == None:
- return None
+ return False
- machines = {}
if self._map["virtual"]:
for m_id in self._map["machines"]:
machines[m_id] = self._prepare_virtual_slave(m_id, mreqs[m_id])
@@ -182,7 +179,7 @@ class SlavePool:
for m_id in self._map["machines"]:
machines[m_id] = self._get_mapped_slave(m_id)
- return machines
+ return True
def is_setup_virtual(self):
return self._map["virtual"]
@@ -232,13 +229,13 @@ class SlavePool:
break
iface = machine.new_static_interface(t_if, "eth")
- iface.set_hwaddr(if_data["hwaddr"])
+ iface.set_hwaddr(if_data["params"]["hwaddr"])
iface.set_network(t_net)
for if_id, if_data in pm["interfaces"].iteritems():
if if_id not in used:
iface = machine.new_unused_interface("eth")
- iface.set_hwaddr(if_data["hwaddr"])
+ iface.set_hwaddr(if_data["params"]["hwaddr"])
iface.set_network(t_net)
return machine
@@ -255,7 +252,7 @@ class SlavePool:
# make all the existing unused
for if_id, if_data in pm["interfaces"].iteritems():
iface = machine.new_unused_interface("eth")
- iface.set_hwaddr(if_data["hwaddr"])
+ iface.set_hwaddr(if_data["params"]["hwaddr"])
iface.set_network(if_data["network"])
# add all the other devices
10 years, 8 months
[lnst] XmlTemplates: Template functions are now objects
by Jiří Pírko
commit 7ed4d4cfa6c1d446054ab9ea7b7cdfc01278a322
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:41:01 2013 +0200
XmlTemplates: Template functions are now objects
With the introduction of lazy evaluation using the XmlTemplateString,
the template functions were converted to classes which are instatiated
and used in the XmlTemplateString objects. These don't have to be
evaluated until the template is conveted to string.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Common/XmlTemplates.py | 249 ++++++++++++++++++++++---------------------
1 files changed, 127 insertions(+), 122 deletions(-)
---
diff --git a/lnst/Common/XmlTemplates.py b/lnst/Common/XmlTemplates.py
index 88ed75b..e1f0f86 100644
--- a/lnst/Common/XmlTemplates.py
+++ b/lnst/Common/XmlTemplates.py
@@ -28,15 +28,14 @@ class XmlTemplateError(Exception):
pass
class TemplateFunc(object):
- _args = []
- _machines = None
-
def __init__(self, args, machines):
self._check_args(args)
+ self._args = args
+
self._machines = machines
def __str__(self):
- self._implementation(self)
+ return self._implementation()
def _check_args(self, args):
pass
@@ -44,12 +43,122 @@ class TemplateFunc(object):
def _implementation(self):
pass
+class IpFunc(TemplateFunc):
+ def _check_args(self, args):
+ if len(args) > 3:
+ msg = "Function ip() takes at most 3 arguments, %d passed" \
+ % len(args)
+ raise XmlTemplateError(msg)
+ if len(args) < 2:
+ msg = "Function ip() must have at least 2 arguments, %d passed" \
+ % len(args)
+ raise XmlTemplateError(msg)
+
+ if len(args) == 3:
+ try:
+ int(args[2])
+ except ValueError:
+ msg = "The third argument of ip() function must be an integer"
+ raise XmlTemplateError(msg)
+
+ def _implementation(self):
+ m_id = self._args[0]
+ if_id = self._args[1]
+ addr = 0
+ if len(self._args) == 3:
+ addr = self._args[2]
+
+ try:
+ machine = self._machines[m_id]
+ except KeyError:
+ msg = "First parameter of function ip() is invalid: " \
+ "Machine %s does not exist." % m_id
+ raise XmlTemplateError(msg)
+
+ try:
+ iface = machine.get_interface(if_id)
+ except MachineError:
+ msg = "Second parameter of function ip() is invalid: "\
+ "Interface %s does not exist." % if_id
+ raise XmlTemplateError(msg)
+
+ try:
+ return iface.get_address(int(addr))
+ except IndexError:
+ msg = "There is no address with index %s on machine %s, " \
+ "interface %s." % (addr, m_id, if_id)
+ raise XmlTemplateError(msg)
+
+class DevnameFunc(TemplateFunc):
+ def _check_args(self, args):
+ if len(args) != 2:
+ msg = "Function devname() takes 2 arguments, %d passed." % len(args)
+ raise XmlTemplateError(msg)
+
+ def _implementation(self):
+ m_id = self._args[0]
+ if_id = self._args[1]
+
+ try:
+ machine = self._machines[m_id]
+ except KeyError:
+ msg = "First parameter of function devname() is invalid: " \
+ "Machine %s does not exist." % m_id
+ raise XmlTemplateError(msg)
+
+ try:
+ iface = machine.get_interface(if_id)
+ except MachineError:
+ msg = "Second parameter of function devname() is invalid: "\
+ "Interface %s does not exist." % if_id
+ raise XmlTemplateError(msg)
+
+ try:
+ return iface.get_devname()
+ except MachineError:
+ msg = "Devname not availablefor interface '%s' on machine '%s'." \
+ % (m_id, if_id)
+ raise XmlTemplateError(msg)
+
+class HwaddrFunc(TemplateFunc):
+ def _check_args(self, args):
+ if len(args) != 2:
+ msg = "Function hwaddr() takes 2 arguments, %d passed." % len(args)
+ raise XmlTemplateError(msg)
+
+ def _implementation(self):
+ m_id = self._args[0]
+ if_id = self._args[1]
+
+ try:
+ machine = self._machines[m_id]
+ except KeyError:
+ msg = "First parameter of function hwaddr() is invalid: " \
+ "Machine %s does not exist." % m_id
+ raise XmlTemplateError(msg)
+
+ try:
+ iface = machine.get_interface(if_id)
+ except MachineError:
+ msg = "Second parameter of function hwaddr() is invalid: "\
+ "Interface %s does not exist." % if_id
+ raise XmlTemplateError(msg)
+
+ try:
+ return iface.get_hwaddr()
+ except MachineError:
+ msg = "Hwaddr not availablefor interface '%s' on machine '%s'." \
+ % (m_id, if_id)
+ raise XmlTemplateError(msg)
+
class XmlTemplates:
""" This class serves as template processor """
_alias_re = "\{\$([a-zA-Z0-9_]+)\}"
_func_re = "\{([a-zA-Z0-9_]+)\(([^\(\)]*)\)\}"
+ _func_map = {"ip": IpFunc, "hwaddr": HwaddrFunc, "devname": DevnameFunc}
+
def __init__(self, definitions=None):
if definitions:
self._definitions = [definitions]
@@ -166,11 +275,7 @@ class XmlTemplates:
value = XmlTemplateString(node=node)
for part in parts:
- if type(part) is list:
- # TODO construct the template function
- pass
- else:
- value.add_part(part)
+ value.add_part(part)
return value
@@ -203,9 +308,9 @@ class XmlTemplates:
suffix = string[func_match.end(0)+1:]
template = func_match.group(0)
- result = self._process_func_template(template)
+ func = self._process_func_template(template)
- return self._partition_string(prefix) + [template] + \
+ return self._partition_string(prefix) + [func] + \
self._partition_string(suffix)
return [string]
@@ -221,128 +326,28 @@ class XmlTemplates:
return result
def _process_func_template(self, string):
- result = None
-
func_match = re.match(self._func_re, string)
if func_match:
func_name = func_match.group(1)
- func_params = func_match.group(2)
+ func_args = func_match.group(2)
- if func_params == None:
- func_params = []
+ if func_args == None:
+ func_args = []
else:
- func_params = func_params.split(",")
+ func_args = func_args.split(",")
param_values = []
- for param in func_params:
+ for param in func_args:
param = param.strip()
if re.match(self._alias_re, param):
param = self._process_alias_template(param)
param_values.append(param)
- result = self._call_preprocessor_func(func_name, param_values)
-
- return result
+ if func_name not in self._func_map:
+ msg = "Unknown template function '%s'." % func_name
+ raise XmlTemplateError(msg)
- def _call_preprocessor_func(self, name, params):
- if name == "ip":
- result = self._ip_func(params)
- elif name == "hwaddr":
- result = self._hwaddr_func(params)
- elif name == "devname":
- result = self._devname_func(params)
+ func = self._func_map[func_name](param_values, self._machines)
+ return func
else:
- raise XmlTemplateError("Unknown preprocessor function '%s'" % name)
-
- return result
-
- def _ip_func(self, params):
- self._validate_func_params("ip", params, 2, 1)
- machines = self._machines
-
- m_id = params[0]
- if_id = params[1]
- ip_id = int(params[2]) if len(params) == 3 else 0
-
- if m_id not in machines:
- msg = "First parameter of function ip() is invalid: "\
- "Machine %s does not exist." % m_id
- raise XmlTemplateError(msg)
- machine = machines[m_id]
-
- try:
- iface = machine.get_interface(if_id)
- except:
- msg = "Second parameter of function ip() is invalid: "\
- "Interface %s does not exist." % if_id
- raise XmlTemplateError(msg)
-
- try:
- addr = iface.get_address(ip_id)
- except:
- msg = "Third parameter of function ip() is invalid: "\
- "Address %s does not exist." % ip_id
- raise XmlTemplateError(msg)
-
- return addr.split('/')[0]
-
- def _hwaddr_func(self, params):
- self._validate_func_params("hwaddr", params, 2, 0)
- machines = self._machines
- m_id = params[0]
- if_id = params[1]
-
- if m_id not in machines:
- msg = "First parameter of function hwaddr() is invalid: "\
- "Machine %s does not exist." % m_id
- raise XmlTemplateError(msg)
- machine = machines[m_id]
-
- try:
- iface = machine.get_interface(if_id)
- except:
- msg = "Second parameter of function ip() is invalid: "\
- "Interface %s does not exist." % if_id
- raise XmlTemplateError(msg)
-
- return iface.get_hwaddr()
-
-
- def _devname_func(self, params):
- self._validate_func_params("devname", params, 2, 0)
- machines = self._machines
- m_id = params[0]
- if_id = params[1]
-
- if m_id not in machines:
- msg = "First parameter of function devname() is invalid: "\
- "Machine %s does not exist." % m_id
- raise XmlTemplateError(msg)
- machine = machines[m_id]
-
- try:
- iface = machine.get_interface(if_id)
- except:
- msg = "Second parameter of function ip() is invalid: "\
- "Interface %s does not exist." % if_id
- raise XmlTemplateError(msg)
-
- return iface.get_devname()
-
- @staticmethod
- def _validate_func_params(name, params, mandatory, optional):
- num_params = len(params)
- if num_params > (mandatory + optional) or num_params < mandatory:
- if optional:
- err = "Function %s takes between %d-%d arguments, %d passed" \
- % (name, mandatory, mandatory + optional, num_params)
- else:
- err = "Function %s takes %d arguments, %d passed" \
- % (name, mandatory, num_params)
- raise XmlTemplateError(err)
- for param in params[2:]:
- try:
- int(param)
- except ValueError:
- err = "Non-integer parameter passed to '%s'" % name
- raise XmlTemplateError(err)
+ raise RuntimeError("The passed string is not a template function.")
10 years, 8 months
[lnst] results: Support the new command sequence format
by Jiří Pírko
commit 37d5579dc3dcb46847ab27cfccc15c843649cd0d
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:41:00 2013 +0200
results: Support the new command sequence format
This patch contains updates for the NetTestResultSerializer class that
were necessary due to the transition from command sequences to tasks.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Controller/NetTestResultSerializer.py | 32 +++++++++++++++++++--------
1 files changed, 22 insertions(+), 10 deletions(-)
---
diff --git a/lnst/Controller/NetTestResultSerializer.py b/lnst/Controller/NetTestResultSerializer.py
index b0bca12..b260bf0 100644
--- a/lnst/Controller/NetTestResultSerializer.py
+++ b/lnst/Controller/NetTestResultSerializer.py
@@ -67,7 +67,7 @@ class NetTestResultSerializer:
else:
self._cur_recipe_el.setAttribute("result", "FAIL")
- def add_command_sequence(self):
+ def add_task(self):
cmd_seq_el = self._dom.createElement("command_sequence")
self._cur_recipe_el.appendChild(cmd_seq_el)
self._cur_cmd_seq_el = cmd_seq_el
@@ -135,8 +135,8 @@ class NetTestResultSerializer:
self._format_ctl_wait_command(command, output_pairs)
elif cmd_type == "exec":
self._format_exec_command(command, output_pairs)
- elif cmd_type == "system_config":
- self._format_system_config(command, output_pairs)
+ elif cmd_type == "config":
+ self._format_config(command, output_pairs)
result_node = command.getElementsByTagName("result")[0]
cmd_res = result_node.getAttribute("result")
@@ -152,7 +152,7 @@ class NetTestResultSerializer:
result_node = command.getElementsByTagName("result")[0]
cmd_res = result_node.getAttribute("result")
- cmd_val = command.getAttribute("value")
+ cmd_val = command.getAttribute("module")
cmd_type = command.getAttribute("type")
if command.hasAttribute("bg_id"):
bg_id = " bg_id: %s" % command.getAttribute("bg_id")
@@ -165,7 +165,7 @@ class NetTestResultSerializer:
result_node = command.getElementsByTagName("result")[0]
cmd_res = result_node.getAttribute("result")
- cmd_val = command.getAttribute("value")
+ cmd_val = command.getAttribute("proc_id")
cmd_type = command.getAttribute("type")
if command.hasAttribute("bg_id"):
bg_id = " bg_id: %s" % command.getAttribute("bg_id")
@@ -181,18 +181,28 @@ class NetTestResultSerializer:
self._format_wait_command(command, output_pairs)
def _format_exec_command(self, command, output_pairs):
- self._format_test_command(command, output_pairs)
+ result_node = command.getElementsByTagName("result")[0]
+ cmd_res = result_node.getAttribute("result")
+
+ cmd_val = command.getAttribute("command")
+ cmd_type = command.getAttribute("type")
+ if command.hasAttribute("bg_id"):
+ bg_id = " bg_id: %s" % command.getAttribute("bg_id")
+ else:
+ bg_id = ""
+ cmd = 8*" "+"%-14s%s%s" %(cmd_type, cmd_val, bg_id)
+ output_pairs.append((cmd, cmd_res))
def _format_ctl_wait_command(self, command, output_pairs):
result_node = command.getElementsByTagName("result")[0]
cmd_res = result_node.getAttribute("result")
- cmd_val = command.getAttribute("value")
+ cmd_val = command.getAttribute("seconds")
cmd_type = command.getAttribute("type")
cmd = 8*" "+"%-14s%ss" %(cmd_type, cmd_val)
output_pairs.append((cmd, cmd_res))
- def _format_system_config(self, command, output_pairs):
+ def _format_config(self, command, output_pairs):
result_node = command.getElementsByTagName("result")[0]
cmd_res = result_node.getAttribute("result")
@@ -209,12 +219,14 @@ class NetTestResultSerializer:
result_data_node = result_data_nodes[0]
options_nodes = result_data_node.getElementsByTagName("options")
for options_node in options_nodes:
- for option in options_node.getElementsByTagName("option"):
+ for option in options_node.getElementsByTagName("options_item"):
previous_node = option.getElementsByTagName("previous_val")[0]
current_node = option.getElementsByTagName("current_val")[0]
+ name_node = option.getElementsByTagName("name")[0]
previous_val = get_node_val(previous_node)
current_val = get_node_val(current_node)
- opt_left = 12*" "+"%s" % option.getAttribute("name")
+ name = get_node_val(name_node)
+ opt_left = 12*" "+"%s" % name
opt_right = "previous: %s current: %s" \
% (previous_val, current_val)
output_pairs.append((opt_left, opt_right))
10 years, 8 months
[lnst] Common: Adding XmlTemplateString support to parser
by Jiří Pírko
commit 95e83ae3f53dd98bd7bf5c045ccfaa9fcad8753a
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:40:59 2013 +0200
Common: Adding XmlTemplateString support to parser
The _get_attribute() method now returns XmlTemplateString instead of
plain strings. In case of includes, we need to convert them to string,
before we pass them on.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Common/XmlParser.py | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
---
diff --git a/lnst/Common/XmlParser.py b/lnst/Common/XmlParser.py
index 69e21e5..86c9959 100644
--- a/lnst/Common/XmlParser.py
+++ b/lnst/Common/XmlParser.py
@@ -185,7 +185,7 @@ class LnstParser(XmlParser):
def _process_node(self, node, handler, params):
old_include_root = None
if self._has_attribute(node, "source"):
- source = self._get_attribute(node, "source")
+ source = str(self._get_attribute(node, "source"))
source_rp = RecipePath(self._include_root, source)
@@ -255,13 +255,13 @@ class LnstParser(XmlParser):
def _alias_handler(self, node, params):
if self._has_attribute(node, "name"):
- name = self._get_attribute(node, "name")
+ name = str(self._get_attribute(node, "name"))
else:
msg = "Alias tag must have the 'name' attribute"
raise XmlProcessingError(msg, node)
if self._has_attribute(node, "value"):
- value = self._get_attribute(node, "value")
+ value = str(self._get_attribute(node, "value"))
else:
value = self._get_text_content(node)
10 years, 8 months
[lnst] slave: Support the new interface of Config cmd
by Jiří Pírko
commit 3579779c4e924a9c30b479c78628622b3d1c0d57
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:40:58 2013 +0200
slave: Support the new interface of Config cmd
The interface of the NetTestCommandConfig has changed a bit. This commit
makes the necessary modifications for the slave side.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Slave/NetTestSlave.py | 15 +++++++++------
1 files changed, 9 insertions(+), 6 deletions(-)
---
diff --git a/lnst/Slave/NetTestSlave.py b/lnst/Slave/NetTestSlave.py
index 4775a42..a3d7e66 100644
--- a/lnst/Slave/NetTestSlave.py
+++ b/lnst/Slave/NetTestSlave.py
@@ -36,8 +36,8 @@ from lnst.Slave.NmConfigDevice import is_nm_managed_by_name
from lnst.Common.Utils import check_process_running
from lnst.Common.ConnectionHandler import recv_data, send_data
from lnst.Common.ConnectionHandler import ConnectionHandler
-from lnst.Common.NetTestCommand import NetTestCommandSystemConfig
from lnst.Common.Config import lnst_config
+from lnst.Common.NetTestCommand import NetTestCommandConfig
DefaultRPCPort = 9999
@@ -210,15 +210,18 @@ class SlaveMethods:
def _update_system_config(self, options, persistent):
system_config = self._system_config
- for option, values in options.iteritems():
+ for opt in options:
+ option = opt["name"]
+ prev = opt["previous_val"]
+ curr = opt["current_val"]
+
if persistent:
if option in system_config:
del system_config[option]
else:
if not option in system_config:
- initial_val = {"initial_val": values["previous_val"]}
- system_config[option] = initial_val
- system_config[option]["current_val"] = values["current_val"]
+ system_config[option] = {"initial_val": prev}
+ system_config[option]["current_val"] = curr
def restore_system_config(self):
logging.info("Restoring system configuration")
@@ -242,7 +245,7 @@ class SlaveMethods:
if not cmd.forked():
self._command_context.del_cmd(cmd)
- if command["type"] == "system_config":
+ if command["type"] == "config":
if res["passed"]:
self._update_system_config(res["res_data"]["options"],
command["persistent"])
10 years, 8 months
[lnst] utils: Extend bool_it to support yes/no
by Jiří Pírko
commit 8c12f9fa8ed4ac50499b42ac77dac190eddc0913
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:40:57 2013 +0200
utils: Extend bool_it to support yes/no
We are using yes/no for some boolean values within the recipe. This
patch extends the bool_it function, so it can be used to handle the
conversions instead of any other approaches.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Common/Utils.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
---
diff --git a/lnst/Common/Utils.py b/lnst/Common/Utils.py
index b2315f8..c3acafb 100644
--- a/lnst/Common/Utils.py
+++ b/lnst/Common/Utils.py
@@ -85,9 +85,9 @@ def int_it(val):
def bool_it(val):
if isinstance(val, str):
- if re.match("^\s*(?i)(true)", val):
+ if re.match("^\s*(?i)(true)", val) or re.match("^\s*(?i)(yes)", val):
return True
- elif re.match("^\s*(?i)(false)", val):
+ elif re.match("^\s*(?i)(false)", val) or re.match("^\s*(?i)(no)", val):
return False
return True if int_it(val) else False
10 years, 8 months
[lnst] SlaveMachineParse: Decouple parsing and semantics
by Jiří Pírko
commit ca28f6009c25e34b29c3536e35b435d4129b9b72
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:40:56 2013 +0200
SlaveMachineParse: Decouple parsing and semantics
This patch makes several changes in SlaveMachineParse and SlavePool
classes. The parser will process the file only into the internal XmlData
representation. The semantic checks are performed later on using this
structure in the SlavePool.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Controller/RecipeParse.py | 10 +++-
lnst/Controller/SlaveMachineParse.py | 75 ++++++++++++++-----------------
lnst/Controller/SlavePool.py | 81 ++++++++++++++++++++++++++++++----
3 files changed, 113 insertions(+), 53 deletions(-)
---
diff --git a/lnst/Controller/RecipeParse.py b/lnst/Controller/RecipeParse.py
index 311243d..e706597 100644
--- a/lnst/Controller/RecipeParse.py
+++ b/lnst/Controller/RecipeParse.py
@@ -20,6 +20,9 @@ from lnst.Common.XmlProcessing import XmlDomTreeInit, XmlData
from lnst.Common.XmlProcessing import XmlProcessingError
from lnst.Common.XmlParser import LnstParser
+class RecipeError(XmlProcessingError):
+ pass
+
class RecipeParse(LnstParser):
def __init__(self, recipe_filepath):
super(RecipeParse, self).__init__()
@@ -338,7 +341,7 @@ class InterfaceParse(LnstParser):
class ParamsParse(LnstParser):
_params = None
- def set_params_dict(self, target):
+ def set_target(self, target):
self._params = target
def parse(self, node):
@@ -353,7 +356,10 @@ class ParamsParse(LnstParser):
else:
value = self._get_text_content(node)
- self._params[name] = value
+ param = XmlData(node)
+ param["name"] = name
+ param["value"] = value
+ self._params.append(param)
class CommandSequenceParse(LnstParser):
def parse(self, node):
diff --git a/lnst/Controller/SlaveMachineParse.py b/lnst/Controller/SlaveMachineParse.py
index 08f84e0..709c478 100644
--- a/lnst/Controller/SlaveMachineParse.py
+++ b/lnst/Controller/SlaveMachineParse.py
@@ -14,68 +14,59 @@ rpazdera(a)redhat.com (Radek Pazdera)
import logging
import os
import re
-from lnst.Common.NetUtils import normalize_hwaddr
from lnst.Controller.RecipeParse import ParamsParse
from lnst.Common.XmlParser import LnstParser
-from lnst.Common.XmlProcessing import XmlDomTreeInit
-from lnst.Common.XmlProcessing import XmlProcessingError, XmlData
+from lnst.Common.XmlProcessing import XmlDomTreeInit, XmlProcessingError
+from lnst.Common.XmlProcessing import XmlData, XmlCollection
-class SlaveMachineParse(LnstParser):
- _machine_id = None
- _machine = None
-
- def set_machine(self, machine_id, machine):
- self._machine_id = machine_id
- self._machine = machine
+class SlaveMachineError(XmlProcessingError):
+ pass
+class SlaveMachineParse(LnstParser):
def parse(self, node):
+ self._data = XmlData(node)
scheme = {"params": self._params,
"interfaces": self._interfaces}
- params = {"target": self._machine["params"]}
- self._process_child_nodes(node, scheme, params)
-
- self._machine["params"]["skip_cleanup"] = False
- mandatory_params = ["hostname"]
- for mandatory in mandatory_params:
- if mandatory not in self._machine["params"]:
- msg = "Missing required parameter '%s'" % mandatory
- raise XmlProcessingError(msg, node)
+ self._process_child_nodes(node, scheme)
+ return self._data
def _params(self, node, params):
+ if "params" in self._data:
+ msg = "Only a single <params> child allowed under <slavemachine>."
+ raise SlaveMachineError(msg, node)
+
subparser = ParamsParse(self)
- subparser.set_params_dict(params["target"])
- subparser.parse(node)
+ self._data["params"] = subparser.parse(node)
def _interfaces(self, node, params):
- scheme = {"eth": self._eth}
+ if not "interfaces" in self._data:
+ self._data["interfaces"] = XmlCollection(node)
+ else:
+ msg = "Only a single <interfaces> child allowed under <slavemachine>."
+ raise SlaveMachineError(msg, node)
- try:
- self._process_child_nodes(node, scheme)
- except XmlProcessingError as err:
- msg = "Interface type other than 'eth' is not allowed here. " \
- "Other types must be configured in LNST recipes directly."
- logging.error(msg)
- raise
+ scheme = {"eth": self._eth}
+ self._process_child_nodes(node, scheme)
def _eth(self, node, params):
- machine = self._machine
- iface_id = self._get_attribute(node, "id")
+ machine = self._data
- iface = machine["interfaces"][iface_id] = {}
+ iface = XmlData(node)
+ iface["id"] = self._get_attribute(node, "id")
iface["network"] = self._get_attribute(node, "network")
- iface["params"] = {}
iface["type"] = "eth"
# parse interface parameters
- scheme = {"params": self._params}
- params = {"target": iface["params"]}
+ scheme = {"params": self._iface_params}
+ params = {"iface": iface}
self._process_child_nodes(node, scheme, params)
- if "hwaddr" in iface["params"]:
- iface["hwaddr"] = normalize_hwaddr(iface["params"]["hwaddr"])
- else:
- msg = "Missing required parameter 'hwaddr'"
- raise XmlProcessingError(msg, node)
+ machine["interfaces"].append(iface)
+
+ def _iface_params(self, node, params):
+ if "params" in params["iface"]:
+ msg = "Only a single <params> child allowed under <interface>."
+ raise SlaveMachineError(msg, node)
- if "name" in iface["params"]:
- iface["name"] = iface["params"]["name"]
+ subparser = ParamsParse(self)
+ params["iface"]["params"] = subparser.parse(node)
diff --git a/lnst/Controller/SlavePool.py b/lnst/Controller/SlavePool.py
index db59ed4..f621d89 100644
--- a/lnst/Controller/SlavePool.py
+++ b/lnst/Controller/SlavePool.py
@@ -18,12 +18,14 @@ import os
import re
import copy
from xml.dom import minidom
+from lnst.Common.NetUtils import normalize_hwaddr
from lnst.Common.NetUtils import test_tcp_connection
-from lnst.Controller.SlaveMachineParse import SlaveMachineParse
from lnst.Common.XmlProcessing import XmlDomTreeInit
from lnst.Common.XmlProcessing import XmlProcessingError, XmlData
from lnst.Controller.Machine import Machine
from lnst.Common.Config import lnst_config
+from lnst.Controller.SlaveMachineParse import SlaveMachineParse
+from lnst.Controller.SlaveMachineParse import SlaveMachineError
class SlavePool:
"""
@@ -63,18 +65,19 @@ class SlavePool:
parser.set_include_root(dirname)
parser.disable_events()
- machine = {"params": {}, "interfaces": {}}
m_id = re.sub("\.[xX][mM][lL]$", "", basename)
- parser.set_machine(m_id, machine)
slavemachine = dom.getElementsByTagName("slavemachine")[0]
-
+ xml_data = XmlData(slavemachine)
+ parser.set_target(xml_data)
parser.parse(slavemachine)
+ machine_spec = self._process_machine_xml_data(m_id, xml_data)
+
if self._pool_checks:
- hostname = machine["params"]["hostname"]
- if "rpcport" in machine:
- port = machine["params"]["rpcport"]
+ hostname = machine_spec["params"]["hostname"]
+ if "rpcport" in machine_spec:
+ port = machine_spec["params"]["rpcport"]
else:
port = lnst_config.get_option('environment', 'rpcport')
@@ -84,13 +87,73 @@ class SlavePool:
logging.warning(msg)
return
- if 'libvirt_domain' in machine['params'] and \
+ if 'libvirt_domain' in machine_spec['params'] and \
not self._allow_virt:
msg = "libvird not running. Skipping machine '%s'." % m_id
logging.warning(msg)
logging.info("Adding slave machine %s to slave pool." % m_id)
- self._pool[m_id] = machine
+ self._pool[m_id] = machine_spec
+
+ def _process_machine_xml_data(self, m_id, machine_xml_data):
+ machine_spec = {"interfaces": {}, "params":{}}
+
+ # process parameters
+ if "params" in machine_xml_data:
+ for param in machine_xml_data["params"]:
+ name = str(param["name"])
+ value = str(param["value"])
+ machine_spec["params"][name] = value
+
+ mandatory_params = ["hostname"]
+ for p in mandatory_params:
+ if p not in machine_spec["params"]:
+ msg = "Mandatory parameter '%s' missing for machine %s." \
+ % (p, m_id)
+ raise SlaveMachineError(msg, machine_xml_data["params"])
+
+ # process interfaces
+ if "interfaces" in machine_xml_data:
+ for iface in machine_xml_data["interfaces"]:
+ if_id = iface["id"]
+ iface_spec = self._process_iface_xml_data(m_id, iface)
+
+ if if_id not in machine_spec["interfaces"]:
+ machine_spec["interfaces"][if_id] = iface_spec
+ else:
+ msg = "Duplicate interface id '%s'." % if_id
+ raise SlaveMachineError(msg, iface)
+ else:
+ if "libvirt_domain" not in machine_spec["params"]:
+ msg = "Machine '%s' has no testing interfaces. " \
+ "This setup is supported only for virtual slaves." \
+ % m_id
+ raise SlaveMachineError(msg, machine_xml_data)
+
+ return machine_spec
+
+ def _process_iface_xml_data(self, m_id, iface):
+ if_id = iface["id"]
+ iface_spec = {"params": {}}
+ iface_spec["network"] = iface["network"]
+
+ for param in iface["params"]:
+ name = str(param["name"])
+ value = str(param["value"])
+
+ if name == "hwaddr":
+ iface_spec["params"][name] = normalize_hwaddr(value)
+ else:
+ iface_spec["params"][name] = value
+
+ mandatory_params = ["hwaddr"]
+ for p in mandatory_params:
+ if p not in iface_spec["params"]:
+ msg = "Mandatory parameter '%s' missing for machine %s, " \
+ "interface '%s'." % (p, m_id, if_id)
+ raise SlaveMachineError(msg, iface["params"])
+
+ return iface_spec
def provision_machines(self, mreqs):
"""
10 years, 8 months
[lnst] XmlTemplates: Return XmlTemplateString
by Jiří Pírko
commit 0e34ad24f3488c6f1dccb22dd87cd2865e2a120d
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:40:55 2013 +0200
XmlTemplates: Return XmlTemplateString
This patch modifies the XmlTemplates processor to return processed data
unevaluated, but wrapped in the XmlTemplateString wrapper. The
controller will evaluate the templates later on during sematic
processing.
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Common/XmlTemplates.py | 67 +++++++++++++++++++++++++++++++++++--------
1 files changed, 55 insertions(+), 12 deletions(-)
---
diff --git a/lnst/Common/XmlTemplates.py b/lnst/Common/XmlTemplates.py
index ed50e83..88ed75b 100644
--- a/lnst/Common/XmlTemplates.py
+++ b/lnst/Common/XmlTemplates.py
@@ -27,6 +27,23 @@ from lnst.Common.XmlProcessing import XmlTemplateString
class XmlTemplateError(Exception):
pass
+class TemplateFunc(object):
+ _args = []
+ _machines = None
+
+ def __init__(self, args, machines):
+ self._check_args(args)
+ self._machines = machines
+
+ def __str__(self):
+ self._implementation(self)
+
+ def _check_args(self, args):
+ pass
+
+ def _implementation(self):
+ pass
+
class XmlTemplates:
""" This class serves as template processor """
@@ -142,30 +159,56 @@ class XmlTemplates:
for node in group:
self.expand_dom(node)
- def expand_string(self, string):
- """ Expand templates in a string
+ def expand_string(self, string, node=None):
+ """ Process a string and expand it into a XmlTemplateString. """
+
+ parts = self._partition_string(string)
+ value = XmlTemplateString(node=node)
+
+ for part in parts:
+ if type(part) is list:
+ # TODO construct the template function
+ pass
+ else:
+ value.add_part(part)
+
+ return value
- This method will process and expand all templates
- contained inside a string.
+ def _partition_string(self, string):
+ """ Process templates in a string
+
+ This method will process and expand all templates contained
+ within a string. It handles both aliases and template
+ function.
+
+ The function returns an array of string partitions and
+ unresolved template functions for further processing.
"""
+
+ result = None
+
while True:
alias_match = re.search(self._alias_re, string)
- func_match = re.search(self._func_re, string)
-
- result = None
if alias_match:
template = alias_match.group(0)
result = self._process_alias_template(template)
- elif func_match:
- template = func_match.group(0)
- result = self._process_func_template(template)
+ string = string.replace(template, result)
else:
break
- string = string.replace(template, result)
+ func_match = re.search(self._func_re, string)
+ if func_match:
+ prefix = string[0:func_match.start(0)]
+ suffix = string[func_match.end(0)+1:]
+
+ template = func_match.group(0)
+ result = self._process_func_template(template)
+
+ return self._partition_string(prefix) + [template] + \
+ self._partition_string(suffix)
- return string
+ return [string]
def _process_alias_template(self, string):
result = None
10 years, 8 months
[lnst] XmlProcessing: Split into two separate modules
by Jiří Pírko
commit 395df7295792b222e0d4a6d53c2ef323702f5197
Author: Radek Pazdera <rpazdera(a)redhat.com>
Date: Tue Aug 6 13:40:54 2013 +0200
XmlProcessing: Split into two separate modules
I added a few new classes to the XmlProcessing file and split it into
two separate modules XmlProcessing and XmlParser.
The XmlProcessing module now contains only the basic classes and the
parser bases were moved to the new module.
New classes:
class XmlData - a structure representing XML data
class XmlCollection - an array (collection) of XmlData structures
Signed-off-by: Radek Pazdera <rpazdera(a)redhat.com>
Signed-off-by: Jiri Pirko <jiri(a)resnulli.us>
lnst/Common/XmlParser.py | 271 +++++++++++++++++++++++
lnst/Common/XmlProcessing.py | 399 +++++++++++-----------------------
lnst/Common/XmlTemplates.py | 1 +
lnst/Controller/RecipeParse.py | 6 +-
lnst/Controller/SlaveMachineParse.py | 6 +-
lnst/Controller/SlavePool.py | 3 +-
6 files changed, 408 insertions(+), 278 deletions(-)
---
diff --git a/lnst/Common/XmlParser.py b/lnst/Common/XmlParser.py
new file mode 100644
index 0000000..69e21e5
--- /dev/null
+++ b/lnst/Common/XmlParser.py
@@ -0,0 +1,271 @@
+"""
+This module contains the XmlParser and LnstParser classes.
+
+Copyright 2013 Red Hat, Inc.
+Licensed under the GNU General Public License, version 2 as
+published by the Free Software Foundation; see COPYING for details.
+"""
+
+__author__ = """
+rpazdera(a)redhat.com (Radek Pazdera)
+"""
+
+import os
+import logging
+from xml.dom.minidom import parseString
+from xml import sax
+from lnst.Common.XmlTemplates import XmlTemplates, XmlTemplateError
+from lnst.Common.RecipePath import RecipePath
+from lnst.Common.XmlProcessing import XmlProcessingError, XmlDomTreeInit
+from lnst.Common.XmlProcessing import XmlData
+
+class XmlParser(object):
+ """ Parent class for XML processors
+
+ This class handles manipulation of XML DOM objects
+ that are used for processing XML files.
+
+ The standard DOM objects are extended with position data
+ (file name, line number and column number) that can be
+ used in error reporting.
+ """
+
+ def _process_child_nodes(self, parent, scheme, params=None,
+ default_handler=None):
+ child_nodes = parent.childNodes
+
+ if not params:
+ params = {}
+
+ for node in child_nodes:
+ if node.nodeType == node.COMMENT_NODE or \
+ node.nodeType == node.TEXT_NODE:
+ continue
+ elif node.nodeType == node.ELEMENT_NODE:
+ node_name = node.nodeName
+ if node_name in scheme:
+ handler = scheme[node_name]
+ self._process_node(node, handler, params)
+ elif default_handler:
+ self._process_node(node, default_handler, params)
+ else:
+ msg = "Unexpected '%s' tag under '%s'" % (node_name,
+ parent.nodeName)
+ raise XmlProcessingError(msg, node)
+ else:
+ msg = "Only XML elements are allowed here!"
+ raise XmlProcessingError(msg, node)
+
+ def _process_node(self, node, handler, params):
+ handler(node, params)
+
+ @staticmethod
+ def _convert_string(node, string, conversion_cb):
+ if conversion_cb:
+ try:
+ converted = conversion_cb(string)
+ except ValueError as err:
+ raise XmlProcessingError("Conversion error: " + str(err), node)
+ return converted
+
+ return string
+
+ def _has_attribute(self, node, attr_name):
+ return node.hasAttribute(attr_name)
+
+ def _get_attribute(self, node, attr_name, conversion_cb=None):
+ if not self._has_attribute(node, attr_name):
+ msg = "Expected attribute '%s' missing" % attr_name
+ raise XmlProcessingError(msg, node)
+ attr_val = str(node.getAttribute(attr_name))
+ return self._convert_string(node, attr_val, conversion_cb)
+
+ def _get_text_content(self, node, conversion_cb=None):
+ content = []
+ for child in node.childNodes:
+ if child.nodeType == child.TEXT_NODE:
+ content.append(child.nodeValue)
+
+ text = str(''.join(content).strip())
+ return self._convert_string(node, text, conversion_cb)
+
+ def _get_all_attributes(self, node):
+ res = {}
+ for i in range(0, node.attributes.length):
+ attr = node.attributes.item(i)
+ res[attr.name] = attr.value
+
+ return res
+
+class LnstParser(XmlParser):
+ """ Enhanced XmlParser
+
+ This class enhances XmlParser with advanced features that are
+ used in parsing LNST XML files. All (sub)parsers should
+ use this as their base class.
+ """
+
+ _data = None
+ _template_proc = None
+ _include_root = None
+ _events_enabled = None
+ _event_handlers = None
+
+ def __init__(self, parent=None):
+ super(LnstParser, self).__init__()
+
+ if parent:
+ self._data = parent._data
+ self._template_proc = parent._template_proc
+ self._include_root = parent._include_root
+ self._events_enabled = parent._events_enabled
+ self._event_handlers = parent._event_handlers
+ else:
+ self._data = None
+ self._template_proc = XmlTemplates()
+ self._include_root = os.getcwd()
+ self._events_enabled = True
+ self._event_handlers = {}
+
+ def set_target(self, data_dict):
+ self._data = data_dict
+
+ def get_data(self):
+ return self._data
+
+ def set_machines(self, machines):
+ self._template_proc.set_machines(machines)
+
+ def set_definitions(self, defs):
+ self._template_proc.set_definitions(defs)
+
+ def set_include_root(self, include_root_path):
+ self._include_root = include_root_path
+
+ def enable_events(self):
+ self._events_enabled = True
+
+ def disable_events(self):
+ self._events_enabled = False
+
+ def register_event_handler(self, event_id, handler):
+ self._event_handlers[event_id] = handler
+
+ def _trigger_event(self, event_id, args):
+ if not self._events_enabled:
+ return
+
+ try:
+ handler = self._event_handlers[event_id]
+ except KeyError as err:
+ logging.warn("No handler found for %s event, ignoring", event_id)
+ return
+
+ handler(**args)
+
+ def _process_child_nodes(self, node, scheme, params=None,
+ default_handler=None, new_ns_level=True):
+ scheme["define"] = self._define_handler
+
+ if not params:
+ params = {}
+
+ if new_ns_level:
+ self._template_proc.add_namespace_level()
+
+ parent = super(LnstParser, self)
+ result = parent._process_child_nodes(node, scheme, params,
+ default_handler)
+
+ if new_ns_level:
+ self._template_proc.drop_namespace_level()
+
+ return result
+
+ def _process_node(self, node, handler, params):
+ old_include_root = None
+ if self._has_attribute(node, "source"):
+ source = self._get_attribute(node, "source")
+
+ source_rp = RecipePath(self._include_root, source)
+
+ old_include_root = self._include_root
+ self._include_root = source_rp.get_root()
+ xmlstr = source_rp.to_str()
+
+ dom_init = XmlDomTreeInit()
+ try:
+ dom = dom_init.parse_string(xmlstr,
+ filename=source_rp.abs_path())
+ except IOError as err:
+ msg = "Unable to resolve include: %s" % str(err)
+ raise XmlProcessingError(msg, node)
+
+ loaded_node = None
+ try:
+ loaded_node = dom.getElementsByTagName(node.nodeName)[0]
+ except Exception:
+ msg = ("No '%s' element present in included file '%s'."
+ % (node.nodeName, source_rp.abs_path()))
+ raise XmlProcessingError(msg, node)
+
+ old_attrs = self._get_all_attributes(node)
+
+ parent = node.parentNode
+ parent.replaceChild(loaded_node, node)
+ node = loaded_node
+
+ # copy all of the original attributes to the sourced node
+ for name, value in old_attrs.iteritems():
+ # do not overwrite sourced attributes
+ if not node.hasAttribute(name):
+ node.setAttribute(name, value)
+
+ parent = super(LnstParser, self)
+ parent._process_node(node, handler, params)
+
+ if old_include_root:
+ self._include_root = old_include_root
+
+ def _get_attribute(self, node, attr_name, conversion_cb=None):
+ parent = super(LnstParser, self)
+ raw_attr_val = parent._get_attribute(node, attr_name)
+
+ try:
+ attr_val = self._template_proc.expand_string(raw_attr_val, node)
+ except XmlTemplateError as err:
+ raise XmlProcessingError(str(err), node)
+
+ return self._convert_string(node, attr_val, conversion_cb)
+
+ def _get_text_content(self, node, conversion_cb=None):
+ parent = super(LnstParser, self)
+ raw_content = parent._get_text_content(node)
+
+ try:
+ content = self._template_proc.expand_string(raw_content, node)
+ except XmlTemplateError as err:
+ raise XmlProcessingError(str(err), node)
+
+ return self._convert_string(node, content, conversion_cb)
+
+ def _define_handler(self, node, params):
+ scheme = {"alias": self._alias_handler}
+ self._process_child_nodes(node, scheme, new_ns_level=False)
+
+ def _alias_handler(self, node, params):
+ if self._has_attribute(node, "name"):
+ name = self._get_attribute(node, "name")
+ else:
+ msg = "Alias tag must have the 'name' attribute"
+ raise XmlProcessingError(msg, node)
+
+ if self._has_attribute(node, "value"):
+ value = self._get_attribute(node, "value")
+ else:
+ value = self._get_text_content(node)
+
+ try:
+ self._template_proc.define_alias(name, value)
+ except XmlTemplateError as err:
+ raise XmlProcessingError(str(err), node)
diff --git a/lnst/Common/XmlProcessing.py b/lnst/Common/XmlProcessing.py
index 8e28915..a709659 100644
--- a/lnst/Common/XmlProcessing.py
+++ b/lnst/Common/XmlProcessing.py
@@ -14,9 +14,6 @@ import os
import logging
from xml.dom.minidom import parseString
from xml import sax
-from lnst.Common.XmlTemplates import XmlTemplates, XmlTemplateError
-from lnst.Common.RecipePath import RecipePath
-
class XmlProcessingError(Exception):
""" Exception thrown on parsing errors """
@@ -25,26 +22,26 @@ class XmlProcessingError(Exception):
_line = None
_col = None
- def __init__(self, msg, node=None):
+ def __init__(self, msg, obj=None):
super(XmlProcessingError, self).__init__()
self._msg = msg
- if node and hasattr(node, "parse_position"):
- self.set_pos(node.parse_position)
+ if obj and hasattr(obj, "loc"):
+ self.set_loc(obj.loc)
#logging.error(self.__str__())
- def set_pos(self, pos):
- self._filename = pos["file"]
- self._line = pos["line"]
- #self._col = pos["col"]
+ def set_loc(self, loc):
+ self._filename = loc["file"]
+ self._line = loc["line"]
+ #self._col = loc["col"]
def __str__(self):
line = ""
col = ""
sep = ""
- pos = ""
+ loc = ""
filename = "<unknown>"
if self._filename:
@@ -58,18 +55,127 @@ class XmlProcessingError(Exception):
col = "%s%d" % (sep, self._col)
if self._line or self._col:
- pos = "%s%s:" % (line, col)
+ loc = "%s%s:" % (line, col)
+
+ return "%s:%s %s" % (filename, loc, self._msg)
+
+class XmlDataIterator:
+ def __init__(self, iterator):
+ self._iterator = iterator
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ n = self._iterator.next()
+
+ # For normal iterators
+ if type(n) == XmlTemplateString:
+ return str(n)
+
+ # For iteritems() iterators
+ if type(n) == tuple and len(n) == 2 and type(n[1]) == XmlTemplateString:
+ return (n[0], str(n[1]))
+
+ return n
+
+class XmlCollection(list):
+ def __init__(self, node=None):
+ super(XmlCollection, self).__init__()
+ if node:
+ self.loc = node.loc
+
+ def __getitem__(self, key):
+ value = super(XmlCollection, self).__getitem__(key)
+ if type(value) == XmlData or type(value) == XmlCollection:
+ return value
+
+ return str(value)
+
+ def __iter__(self):
+ it = super(XmlCollection, self).__iter__()
+ return XmlDataIterator(it)
- return "XmlProcessingError:%s:%s %s" % (filename, pos, self._msg)
+class XmlData(dict):
+ def __init__(self, node=None):
+ super(XmlData, self).__init__()
+ if node:
+ self.loc = node.loc
+
+ def __getitem__(self, key):
+ value = super(XmlData, self).__getitem__(key)
+ if type(value) == XmlData or type(value) == XmlCollection:
+ return value
+
+ return str(value)
+
+ def __iter__(self):
+ it = super(XmlData, self).__iter__()
+ return XmlDataIterator(it)
+
+ def iteritems(self):
+ it = super(XmlData, self).iteritems()
+ return XmlDataIterator(it)
+
+ def iterkeys(self):
+ it = super(XmlData, self).iterkeys()
+ return XmlDataIterator(it)
+
+ def itervalues(self):
+ it = super(XmlData, self).itervalues()
+ return XmlDataIterator(it)
+
+class XmlTemplateString(object):
+ def __init__(self, param=None, node=None):
+ if type(param) == str:
+ self._parts = [param]
+ elif type(param) == list:
+ self._parts = param
+ else:
+ self._parts = []
+
+ if node and hasattr(node, "loc"):
+ self.loc = node.loc
+
+ def __add__(self, other):
+ if type(other) is str:
+ self.add_part(other)
+ elif type(other) is self.__class__:
+ self._parts += other._parts
+ else:
+ raise XmlProcessingError("Cannot concatenate %s and %s" % \
+ str(type(self)), str(type(other)))
+ return self
+
+ def __str__(self):
+ string = ""
+ for part in self._parts:
+ string += str(part)
+ return string
+
+ def __hash__(self):
+ return hash(str(self))
+
+ def __eq__(self, other):
+ return str(self) == str(other)
+
+ def __ne__(self, other):
+ return str(self) != str(other)
+
+ def __len__(self):
+ return len(str(self))
+
+ def add_part(self, part):
+ self._parts.append(part)
class XmlDomTreeInit:
""" Handles creation/initialization of DOM trees
It allows you to parse XML file or string into a DOM tree.
It also adds an extra parameter to each node of the tree
- called `parse_position' which can be used to determine
- where exactly was the element placed in the original source
- XML file. This is useful for error reporting.
+ called `loc' which can be used to determine where exactly
+ was the element placed in the original source XML file.
+ This is useful for error reporting.
"""
_sax = None
@@ -89,10 +195,10 @@ class XmlDomTreeInit:
def start_element_ns(name, tag_name , attrs):
orig_start_cb(name, tag_name, attrs)
cur_elem = dom_handler.elementStack[-1]
- pos = {"file": self._filename,
+ loc = {"file": self._filename,
"line": self._sax.getLineNumber(),
"col": self._sax.getColumnNumber()}
- cur_elem.parse_position = pos
+ cur_elem.loc = loc
orig_start_cb = dom_handler.startElementNS
dom_handler.startElementNS = start_element_ns
@@ -115,262 +221,13 @@ class XmlDomTreeInit:
try:
dom = parseString(xml_text, self._sax)
except sax.SAXParseException as err:
- pos = {"file": self._filename,
+ loc = {"file": self._filename,
"line": err.getLineNumber(),
"col": err.getColumnNumber()}
exc = XmlProcessingError(err.getMessage())
- exc.set_pos(pos)
+ exc.set_loc(loc)
raise exc
- pos = {"file": self._filename, "line": 0, "col": 0}
- dom.parse_position = pos
+ loc = {"file": self._filename, "line": 0, "col": 0}
+ dom.loc = loc
return dom
-
-
-class XmlParser(object):
- """ Parent class for XML processors
-
- This class handles manipulation of XML DOM objects
- that are used for processing XML files.
-
- The standard DOM objects are extended with position data
- (file name, line number and column number) that can be
- used in error reporting.
- """
-
- def _process_child_nodes(self, parent, scheme, params=None,
- default_handler=None):
- child_nodes = parent.childNodes
-
- if not params:
- params = {}
-
- for node in child_nodes:
- if node.nodeType == node.COMMENT_NODE or \
- node.nodeType == node.TEXT_NODE:
- continue
- elif node.nodeType == node.ELEMENT_NODE:
- node_name = node.nodeName
- if node_name in scheme:
- handler = scheme[node_name]
- self._process_node(node, handler, params)
- elif default_handler:
- self._process_node(node, default_handler, params)
- else:
- msg = "Unexpected '%s' tag under '%s'" % (node_name,
- parent.nodeName)
- raise XmlProcessingError(msg, node)
- else:
- msg = "Only XML elements are allowed here!"
- raise XmlProcessingError(msg, node)
-
- def _process_node(self, node, handler, params):
- handler(node, params)
-
- @staticmethod
- def _convert_string(node, string, conversion_cb):
- if conversion_cb:
- try:
- converted = conversion_cb(string)
- except ValueError as err:
- raise XmlProcessingError("Conversion error: " + str(err), node)
- return converted
-
- return string
-
- def _has_attribute(self, node, attr_name):
- return node.hasAttribute(attr_name)
-
- def _get_attribute(self, node, attr_name, conversion_cb=None):
- if not self._has_attribute(node, attr_name):
- msg = "Expected attribute '%s' missing" % attr_name
- raise XmlProcessingError(msg, node)
- attr_val = str(node.getAttribute(attr_name))
- return self._convert_string(node, attr_val, conversion_cb)
-
- def _get_text_content(self, node, conversion_cb=None):
- content = []
- for child in node.childNodes:
- if child.nodeType == child.TEXT_NODE:
- content.append(child.nodeValue)
-
- text = str(''.join(content).strip())
- return self._convert_string(node, text, conversion_cb)
-
- def _get_all_attributes(self, node):
- res = {}
- for i in range(0, node.attributes.length):
- attr = node.attributes.item(i)
- res[attr.name] = attr.value
-
- return res
-
-class LnstParser(XmlParser):
- """ Enhanced XmlParser
-
- This class enhances XmlParser with advanced features that are
- used in parsing LNST XML files. All (sub)parsers should
- use this as their base class.
- """
-
- _data = None
- _template_proc = None
- _include_root = None
- _events_enabled = None
- _event_handlers = None
-
- def __init__(self, parent=None):
- super(LnstParser, self).__init__()
-
- if parent:
- self._data = parent._data
- self._template_proc = parent._template_proc
- self._include_root = parent._include_root
- self._events_enabled = parent._events_enabled
- self._event_handlers = parent._event_handlers
- else:
- self._data = {}
- self._template_proc = XmlTemplates()
- self._include_root = os.getcwd()
- self._events_enabled = True
- self._event_handlers = {}
-
- def set_target(self, data_dict):
- self._data = data_dict
-
- def set_machines(self, machines):
- self._template_proc.set_machines(machines)
-
- def set_definitions(self, defs):
- self._template_proc.set_definitions(defs)
-
- def set_include_root(self, include_root_path):
- self._include_root = include_root_path
-
- def enable_events(self):
- self._events_enabled = True
-
- def disable_events(self):
- self._events_enabled = False
-
- def register_event_handler(self, event_id, handler):
- self._event_handlers[event_id] = handler
-
- def _trigger_event(self, event_id, args):
- if not self._events_enabled:
- return
-
- try:
- handler = self._event_handlers[event_id]
- except KeyError as err:
- logging.warn("No handler found for %s event, ignoring", event_id)
- return
-
- handler(**args)
-
- def _process_child_nodes(self, node, scheme, params=None,
- default_handler=None, new_ns_level=True):
- scheme["define"] = self._define_handler
-
- if not params:
- params = {}
-
- if new_ns_level:
- self._template_proc.add_namespace_level()
-
- parent = super(LnstParser, self)
- result = parent._process_child_nodes(node, scheme, params,
- default_handler)
-
- if new_ns_level:
- self._template_proc.drop_namespace_level()
-
- return result
-
- def _process_node(self, node, handler, params):
- old_include_root = None
- if self._has_attribute(node, "source"):
- source = self._get_attribute(node, "source")
-
- source_rp = RecipePath(self._include_root, source)
-
- old_include_root = self._include_root
- self._include_root = source_rp.get_root()
- xmlstr = source_rp.to_str()
-
- dom_init = XmlDomTreeInit()
- try:
- dom = dom_init.parse_string(xmlstr,
- filename=source_rp.abs_path())
- except IOError as err:
- msg = "Unable to resolve include: %s" % str(err)
- raise XmlProcessingError(msg, node)
-
- loaded_node = None
- try:
- loaded_node = dom.getElementsByTagName(node.nodeName)[0]
- except Exception:
- msg = ("No '%s' element present in included file '%s'."
- % (node.nodeName, source_rp.abs_path()))
- raise XmlProcessingError(msg, node)
-
- old_attrs = self._get_all_attributes(node)
-
- parent = node.parentNode
- parent.replaceChild(loaded_node, node)
- node = loaded_node
-
- # copy all of the original attributes to the sourced node
- for name, value in old_attrs.iteritems():
- # do not overwrite sourced attributes
- if not node.hasAttribute(name):
- node.setAttribute(name, value)
-
- parent = super(LnstParser, self)
- parent._process_node(node, handler, params)
-
- if old_include_root:
- self._include_root = old_include_root
-
- def _get_attribute(self, node, attr_name, conversion_cb=None):
- parent = super(LnstParser, self)
- raw_attr_val = parent._get_attribute(node, attr_name)
-
- try:
- attr_val = self._template_proc.expand_string(raw_attr_val)
- except XmlTemplateError as err:
- raise XmlProcessingError(str(err), node)
-
- return self._convert_string(node, attr_val, conversion_cb)
-
- def _get_text_content(self, node, conversion_cb=None):
- parent = super(LnstParser, self)
- raw_content = parent._get_text_content(node)
-
- try:
- content = self._template_proc.expand_string(raw_content)
- except XmlTemplateError as err:
- raise XmlProcessingError(str(err), node)
-
- return self._convert_string(node, content, conversion_cb)
-
- def _define_handler(self, node, params):
- scheme = {"alias": self._alias_handler}
- self._process_child_nodes(node, scheme, new_ns_level=False)
-
- def _alias_handler(self, node, params):
- if self._has_attribute(node, "name"):
- name = self._get_attribute(node, "name")
- else:
- msg = "Alias tag must have the 'name' attribute"
- raise XmlProcessingError(msg, node)
-
- if self._has_attribute(node, "value"):
- value = self._get_attribute(node, "value")
- else:
- value = self._get_text_content(node)
-
- try:
- self._template_proc.define_alias(name, value)
- except XmlTemplateError as err:
- raise XmlProcessingError(str(err), node)
diff --git a/lnst/Common/XmlTemplates.py b/lnst/Common/XmlTemplates.py
index 28a6d51..ed50e83 100644
--- a/lnst/Common/XmlTemplates.py
+++ b/lnst/Common/XmlTemplates.py
@@ -22,6 +22,7 @@ rpazdera(a)redhat.com (Radek Pazdera)
"""
import re
+from lnst.Common.XmlProcessing import XmlTemplateString
class XmlTemplateError(Exception):
pass
diff --git a/lnst/Controller/RecipeParse.py b/lnst/Controller/RecipeParse.py
index 04cb648..311243d 100644
--- a/lnst/Controller/RecipeParse.py
+++ b/lnst/Controller/RecipeParse.py
@@ -13,12 +13,12 @@ jpirko(a)redhat.com (Jiri Pirko)
import logging
import os
import re
-from lnst.Common.XmlProcessing import LnstParser
-from lnst.Common.XmlProcessing import XmlDomTreeInit
-from lnst.Common.XmlProcessing import XmlProcessingError
from lnst.Common.NetUtils import normalize_hwaddr
from lnst.Common.Utils import bool_it
from lnst.Common.RecipePath import RecipePath
+from lnst.Common.XmlProcessing import XmlDomTreeInit, XmlData
+from lnst.Common.XmlProcessing import XmlProcessingError
+from lnst.Common.XmlParser import LnstParser
class RecipeParse(LnstParser):
def __init__(self, recipe_filepath):
diff --git a/lnst/Controller/SlaveMachineParse.py b/lnst/Controller/SlaveMachineParse.py
index 4be2dc5..08f84e0 100644
--- a/lnst/Controller/SlaveMachineParse.py
+++ b/lnst/Controller/SlaveMachineParse.py
@@ -14,11 +14,11 @@ rpazdera(a)redhat.com (Radek Pazdera)
import logging
import os
import re
-from lnst.Common.XmlProcessing import LnstParser
-from lnst.Common.XmlProcessing import XmlDomTreeInit
-from lnst.Common.XmlProcessing import XmlProcessingError
from lnst.Common.NetUtils import normalize_hwaddr
from lnst.Controller.RecipeParse import ParamsParse
+from lnst.Common.XmlParser import LnstParser
+from lnst.Common.XmlProcessing import XmlDomTreeInit
+from lnst.Common.XmlProcessing import XmlProcessingError, XmlData
class SlaveMachineParse(LnstParser):
_machine_id = None
diff --git a/lnst/Controller/SlavePool.py b/lnst/Controller/SlavePool.py
index 05c6041..db59ed4 100644
--- a/lnst/Controller/SlavePool.py
+++ b/lnst/Controller/SlavePool.py
@@ -18,9 +18,10 @@ import os
import re
import copy
from xml.dom import minidom
-from lnst.Common.XmlProcessing import XmlDomTreeInit
from lnst.Common.NetUtils import test_tcp_connection
from lnst.Controller.SlaveMachineParse import SlaveMachineParse
+from lnst.Common.XmlProcessing import XmlDomTreeInit
+from lnst.Common.XmlProcessing import XmlProcessingError, XmlData
from lnst.Controller.Machine import Machine
from lnst.Common.Config import lnst_config
10 years, 8 months
[PATCH v2 00/19] Massive Controller Refactoring + cmd seq facelift
by Radek Pazdera
Hello everyone,
I finaly completed the refactoring of XML parsing at the controller
and finalized the patch set. I'll try to briefly summarize here what
has changed.
The goal of this series is to decouple parsing of the XMLs from processing
their contents. Up to now, the execution of lnst-ctl was controlled by the
parser, which proved to be very limiting and also quite convoluted.
This series introduces an intermediary data structure, a sort-of abstract
syntax tree, that is generated by the XML parser. The controller then
doesn't interact with the parser directly; it works with this new
structure instead.
Using this approach, I was able to move all the semantic checks to the
controller while keeping the structural checks in the parser. This will
allow us to use XML schemes for checking the structure in the future.
Apart from that, it makes the parser code much more readable (in my
opinion). Each node of the abstract syntax tree also contains references
back to the original XML file that are used for much more accurate
error reporting.
The series also contains a facelift of the former <command_sequence>
tag (now <task>), which is very closely related to the refactoring
process. The changes are the most prominent with commands. Each
command used to be represented by the <command> tag with a type
attribute. This was changed, so each command type now has it's own
tag instead.
For example:
<command type="test" value="IcmpPing" machine_id="tm">
...
</command>
changed to
<run module="IcmpPing" machine="tm">
...
</run>.
The commands are usualy shorter and easier to distinguish from each
another now.
To help with converting the existing recipes to the up-to-date format,
I modified the 'misc/recipe_conv.py' script, which can be used to
automatically conver all your recipes.
I tested everything I could with my current setup. Please let me know
if you run into problems :-).
Cheers,
-Radek
Radek Pazdera (19):
XmlProcessing: Split into two separate modules
XmlTemplates: Return XmlTemplateString
SlaveMachineParse: Decouple parsing and semantics
utils: Extend bool_it to support yes/no
slave: Support the new interface of Config cmd
Common: Adding XmlTemplateString support to parser
results: Support the new command sequence format
XmlTemplates: Template functions are now objects
ctl: Slightly changing the interface of SlavePool
ctl: Slight change of the interface of Interface
ctl: Massive refactoring of the RecipeParser
commands: Changing the internal structures
ctl: Changing pass_result to expect
ctl: Massive refactoring of NetTestController
misc: Updating the recipe conversion script
recipes: Updating example recipes
recipes: Updating multicast recipes
recipes: Updating the team recipes
recipes: Updating smoke tests
lnst/Common/NetTestCommand.py | 93 ++--
lnst/Common/Utils.py | 4 +-
lnst/Common/XmlParser.py | 271 ++++++++++
lnst/Common/XmlProcessing.py | 399 +++++---------
lnst/Common/XmlTemplates.py | 295 ++++++-----
lnst/Controller/Machine.py | 15 +-
lnst/Controller/NetTestController.py | 404 ++++++++++-----
lnst/Controller/NetTestResultSerializer.py | 32 +-
lnst/Controller/RecipeParse.py | 572 ++++++++-------------
lnst/Controller/SlaveMachineParse.py | 77 ++-
lnst/Controller/SlavePool.py | 95 +++-
lnst/Slave/NetTestSlave.py | 15 +-
misc/recipe_conv.py | 56 +-
recipes/examples/quick_guides/ping_check.xml | 27 +-
recipes/multicast/cmd_sequences/block_source.xml | 36 +-
recipes/multicast/cmd_sequences/if.xml | 37 +-
recipes/multicast/cmd_sequences/loop.xml | 32 +-
recipes/multicast/cmd_sequences/max_groups.xml | 14 +-
recipes/multicast/cmd_sequences/membership.xml | 22 +-
recipes/multicast/cmd_sequences/simple.xml | 19 +-
.../multicast/cmd_sequences/source_membership.xml | 39 +-
recipes/multicast/cmd_sequences/ttl.xml | 49 +-
recipes/multicast/multicast-all.xml | 32 +-
recipes/smoke/generate-recipes.py | 6 +-
recipes/smoke/lib/conf-bond.xml | 2 +-
recipes/smoke/lib/conf-eth.xml | 2 +-
recipes/smoke/lib/conf-team.xml | 2 +-
recipes/smoke/lib/conf-vlan.xml | 2 +-
recipes/smoke/lib/recipe-temp.xml | 2 +-
recipes/smoke/lib/seq-bg.xml | 13 -
recipes/smoke/lib/seq-exec.xml | 7 -
recipes/smoke/lib/seq-ping.xml | 10 -
recipes/smoke/lib/seq-sysconfig.xml | 10 -
recipes/smoke/lib/task-bg.xml | 12 +
recipes/smoke/lib/task-config.xml | 10 +
recipes/smoke/lib/task-exec.xml | 6 +
recipes/smoke/lib/task-ping.xml | 10 +
recipes/team/interfaces-bond_lacp.xml | 2 +-
recipes/team/interfaces-br_queue_mapping.xml | 2 +-
recipes/team/interfaces-simple.xml | 2 +-
recipes/team/interfaces-simple_br.xml | 2 +-
recipes/team/interfaces-team_ab.xml | 2 +-
recipes/team/interfaces-team_ab_lw_001.xml | 2 +-
recipes/team/interfaces-team_ab_lw_002.xml | 2 +-
recipes/team/interfaces-team_ab_lw_003.xml | 2 +-
recipes/team/interfaces-team_ab_lw_004.xml | 2 +-
recipes/team/interfaces-team_ab_lw_005.xml | 2 +-
recipes/team/interfaces-team_ab_lw_006.xml | 2 +-
recipes/team/interfaces-team_ab_lw_007.xml | 2 +-
recipes/team/interfaces-team_ab_lw_008.xml | 2 +-
recipes/team/interfaces-team_bcast.xml | 2 +-
recipes/team/interfaces-team_lacp.xml | 2 +-
recipes/team/interfaces-team_lacp_act.xml | 2 +-
recipes/team/interfaces-team_lacp_passive.xml | 2 +-
recipes/team/interfaces-team_lacp_slowrate.xml | 2 +-
recipes/team/interfaces-team_lb.xml | 2 +-
recipes/team/interfaces-team_lb_act.xml | 8 +-
recipes/team/interfaces-team_rr.xml | 2 +-
recipes/team/interfaces-team_rr_queue_mapping.xml | 2 +-
recipes/team/machineconfig-dhcp-37-128.xml | 8 +-
recipes/team/machineconfig-peanut.xml | 8 +-
recipes/team/machineconfig-test1.xml | 6 +-
recipes/team/machineconfig-test2.xml | 6 +-
recipes/team/recipex_ab_001.xml | 4 +-
recipes/team/recipex_ab_002.xml | 18 +-
recipes/team/recipex_ab_003.xml | 4 +-
recipes/team/recipex_ab_004.xml | 4 +-
recipes/team/recipex_bcast_001.xml | 4 +-
recipes/team/recipex_bcast_002.xml | 18 +-
recipes/team/recipex_bcast_003.xml | 4 +-
recipes/team/recipex_bcast_004.xml | 4 +-
recipes/team/recipex_lacp_001.xml | 4 +-
recipes/team/recipex_lacp_002.xml | 18 +-
recipes/team/recipex_lacp_003.xml | 4 +-
recipes/team/recipex_lacp_004.xml | 4 +-
recipes/team/recipex_lacp_act_001.xml | 4 +-
recipes/team/recipex_lacp_act_002.xml | 18 +-
recipes/team/recipex_lacp_act_003.xml | 4 +-
recipes/team/recipex_lacp_act_004.xml | 4 +-
recipes/team/recipex_lb_001.xml | 4 +-
recipes/team/recipex_lb_002.xml | 18 +-
recipes/team/recipex_lb_003.xml | 4 +-
recipes/team/recipex_lb_004.xml | 4 +-
recipes/team/recipex_lb_act_001.xml | 4 +-
recipes/team/recipex_lb_act_002.xml | 18 +-
recipes/team/recipex_lb_act_003.xml | 4 +-
recipes/team/recipex_lb_act_004.xml | 4 +-
recipes/team/recipex_rr_001.xml | 4 +-
recipes/team/recipex_rr_002.xml | 18 +-
recipes/team/recipex_rr_003.xml | 4 +-
recipes/team/recipex_rr_004.xml | 4 +-
recipes/team/recipey_ab_lw_001.xml | 4 +-
recipes/team/recipey_ab_lw_002.xml | 4 +-
recipes/team/recipey_ab_lw_003.xml | 4 +-
recipes/team/recipey_ab_lw_004.xml | 4 +-
recipes/team/recipey_ab_lw_005.xml | 4 +-
recipes/team/recipey_ab_lw_006.xml | 4 +-
recipes/team/recipey_ab_lw_007.xml | 4 +-
recipes/team/recipey_ab_lw_008.xml | 4 +-
recipes/team/recipey_lacp_act_flows.xml | 4 +-
recipes/team/recipey_lacp_hashes.xml | 4 +-
recipes/team/recipey_lacp_passive.xml | 4 +-
recipes/team/recipey_lacp_slowrate.xml | 4 +-
recipes/team/recipey_lb_act_flows.xml | 4 +-
recipes/team/recipey_lb_hashes.xml | 4 +-
recipes/team/recipey_rr_queue_mapping.xml | 4 +-
recipes/team/sequence_iperf.xml | 26 +-
recipes/team/sequence_ping_simple.xml | 14 +-
recipes/team/sequence_pktgen.xml | 16 +-
recipes/team/sequence_pktgen_flows.xml | 14 +-
recipes/team/sequence_pktgen_hashes.xml | 10 +-
recipes/team/sequence_pktgen_lacp.xml | 16 +-
recipes/team/sequence_queue_mapping.xml | 38 +-
113 files changed, 1755 insertions(+), 1456 deletions(-)
create mode 100644 lnst/Common/XmlParser.py
delete mode 100644 recipes/smoke/lib/seq-bg.xml
delete mode 100644 recipes/smoke/lib/seq-exec.xml
delete mode 100644 recipes/smoke/lib/seq-ping.xml
delete mode 100644 recipes/smoke/lib/seq-sysconfig.xml
create mode 100644 recipes/smoke/lib/task-bg.xml
create mode 100644 recipes/smoke/lib/task-config.xml
create mode 100644 recipes/smoke/lib/task-exec.xml
create mode 100644 recipes/smoke/lib/task-ping.xml
--
1.8.3.1
10 years, 8 months