This new boolean parameter allows the caller to specify whether to include the stderr output of the executed command in the exception that is thrown.
Some commands have very nice and descriptive one-line error reports that LNST can pass along in the exception and print to user. On the other hand, some different commands print huge error reportings or memory dumps that definitely should not be printed to the user.
Signed-off-by: Radek Pazdera rpazdera@redhat.com --- Common/ExecCmd.py | 13 +++++++++---- 1 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/Common/ExecCmd.py b/Common/ExecCmd.py index 702ad73..174ea39 100644 --- a/Common/ExecCmd.py +++ b/Common/ExecCmd.py @@ -17,10 +17,12 @@ class ExecCmdFail(Exception): _cmd = None _retval = None _stderr = None + _report_stderr = None
- def __init__(self, cmd=None, retval=None, err=""): + def __init__(self, cmd=None, retval=None, err="", report_stderr=False): self._stderr = err self._retval = retval + self._report_stderr = report_stderr
def get_cmd(self): return self._cmd @@ -30,9 +32,12 @@ class ExecCmdFail(Exception):
def __str__(self): retval = "" + stderr = "" if self._retval: retval = " (exited with %d)" % self._retval - return "Command execution failed%s" % retval + if self._report_stderr: + stderr = " [%s]" % self._stderr + return "Command execution failed%s%s" % (retval, stderr)
def log_output(log_func, out_type, out): log_func("%s:\n" @@ -41,7 +46,7 @@ def log_output(log_func, out_type, out): "----------------------------" % (out_type, out))
-def exec_cmd(cmd, die_on_err=True, log_outputs=True): +def exec_cmd(cmd, die_on_err=True, log_outputs=True, report_stderr=False): cmd = cmd.rstrip(" ") logging.debug("Executing: "%s"" % cmd) subp = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, @@ -58,7 +63,7 @@ def exec_cmd(cmd, die_on_err=True, log_outputs=True): if data_stderr: log_output(logging.error, "Stderr", data_stderr) if subp.returncode and die_on_err: - err = ExecCmdFail(cmd, subp.returncode, data_stderr) + err = ExecCmdFail(cmd, subp.returncode, data_stderr,report_stderr) logging.error(err) raise err
When a remote RPC method raises an exception it's traceback is marshalled back to the controller without the error message that the exception carried. The traceback is suitable for debuging, but it is certainly not suited for users.
This patch makes the dispatcher log the traceback to debug first and then send only the error message. The traceback is transfered to controller through the logging facility.
Signed-off-by: Radek Pazdera rpazdera@redhat.com --- Common/XmlRpc.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/Common/XmlRpc.py b/Common/XmlRpc.py index 3570495..9f16d06 100644 --- a/Common/XmlRpc.py +++ b/Common/XmlRpc.py @@ -20,6 +20,7 @@ jpirko@redhat.com (Jiri Pirko) """
from SimpleXMLRPCServer import SimpleXMLRPCServer +from Common.Logs import log_exc_traceback import xmlrpclib import sys import socket @@ -58,9 +59,9 @@ class Server(SimpleXMLRPCServer): def _dispatch(self, method, params): try: return SimpleXMLRPCServer._dispatch(self, method, params) - except: - type, value, tb = sys.exc_info() - raise xmlrpclib.Fault(1, ''.join(traceback.format_exception(type, value, tb))) + except Exception as err: + log_exc_traceback() + raise xmlrpclib.Fault(1, str(err))
def serve_until_done(self): while self.util_inst.running:
If a known error is thrown, LNST drops dead as well as in case when an unknown exception is raised. Known exceptions are a part of standard program operation and should not result in abort with backtrace.
Signed-off-by: Radek Pazdera rpazdera@redhat.com --- nettestctl.py | 13 ++++++++++--- 1 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/nettestctl.py b/nettestctl.py index c4154d4..b0fb51d 100755 --- a/nettestctl.py +++ b/nettestctl.py @@ -16,7 +16,7 @@ import sys import logging import os import re -from NetTest.NetTestController import NetTestController +from NetTest.NetTestController import NetTestController, NetTestError from NetTest.NetTestResultSerializer import NetTestResultSerializer from Common.Logs import Logs from Common.LoggingServer import LoggingServer @@ -77,8 +77,15 @@ def get_recipe_result(args, file_path, remoteexec, cleanup, Logs.set_logging_root_path(file_path) loggingServer = LoggingServer(Logs.root_path, Logs.debug) loggingServer.start() - res = process_recipe(args, file_path, remoteexec, cleanup, res_serializer, - packet_capture, config, loggingServer) + + res = None + try: + res = process_recipe(args, file_path, remoteexec, cleanup, + res_serializer, packet_capture, + config, loggingServer) + except NetTestError as err: + logging.error(err) + loggingServer.stop() return ((file_path, res))
This patch adds pretty-formatted error message to CommandExceptions.
Signed-off-by: Radek Pazdera rpazdera@redhat.com --- NetTest/NetTestSlave.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-)
diff --git a/NetTest/NetTestSlave.py b/NetTest/NetTestSlave.py index 765094f..de44618 100644 --- a/NetTest/NetTestSlave.py +++ b/NetTest/NetTestSlave.py @@ -127,7 +127,11 @@ class NetTestSlaveXMLRPC: return NetTestCommand(self._command_context, command).run() except: log_exc_traceback() - raise CommandException(command) + cmd_type = command["type"] + m_id = command["machine_id"] + msg = "Execution of %s command on machine %s failed" \ + % (cmd_type, m_id) + raise CommandException(msg)
def machine_cleanup(self): NetConfigDeviceAllCleanup()
This commit does some cleanup to various exceptions and errors
Signed-off-by: Radek Pazdera rpazdera@redhat.com --- Common/XmlRpc.py | 2 +- NetTest/NetTestController.py | 42 ++++++++++++++++++++++++------------------ NetTest/NetTestParse.py | 12 +++++++----- nettestctl.py | 2 +- 4 files changed, 33 insertions(+), 25 deletions(-)
diff --git a/Common/XmlRpc.py b/Common/XmlRpc.py index 9f16d06..d8217f8 100644 --- a/Common/XmlRpc.py +++ b/Common/XmlRpc.py @@ -84,7 +84,7 @@ class ExceptionUnmarshaller(xmlrpclib.Unmarshaller): def close(self): try: return xmlrpclib.Unmarshaller.close(self) - except xmlrpclib.Fault, e: + except xmlrpclib.Fault as e: raise ServerException(e.faultString)
class ExceptionTransport(xmlrpclib.Transport): diff --git a/NetTest/NetTestController.py b/NetTest/NetTestController.py index 2e67dd6..dbbaffc 100644 --- a/NetTest/NetTestController.py +++ b/NetTest/NetTestController.py @@ -17,7 +17,7 @@ import os from Common.Logs import Logs, log_exc_traceback from Common.SshUtils import scp_from_remote from pprint import pprint, pformat -from Common.XmlRpc import ServerProxy +from Common.XmlRpc import ServerProxy, ServerException from NetTest.NetTestParse import NetTestParse from Common.SlaveUtils import prepare_client_session from Common.NetUtils import get_corespond_local_ip, MacPool @@ -78,7 +78,7 @@ class NetTestController: info = self._recipe["machines"][machine_id]["info"] except KeyError: msg = "Machine info is required, but not yet available" - raise Exception(msg) + raise NetTestError(msg)
return info
@@ -87,15 +87,15 @@ class NetTestController: rpc = self._get_machineinfo(machine_id)["rpc"] except KeyError: msg = "XMLRPC connection required, but not yet available" - raise Exception(msg) + raise NetTestError(msg)
return rpc
@staticmethod def _session_die(session, status): - logging.error("Session started with cmd %s die with status %s.", - session.command, status) - raise Exception("Session Die.") + logging.debug("%s terminated with status %s", session.command, status) + msg = "SSH session terminated with status %s" % status + raise NetTestError(msg)
def _prepare_provisioning(self): provisioning = self._recipe["provisioning"] @@ -261,7 +261,7 @@ class NetTestController: passwd = info["rootpass"] else: passwd = '' - logging.info("Remote app exec on machine %s", hostname) + logging.info("Executing nettestslave on machine %s", hostname)
port = "22" login = "root" @@ -283,8 +283,10 @@ class NetTestController: url = "http://%s:%d" % (hostname, port) rpc = ServerProxy(url, allow_none = True) if rpc.hello() != "hello": - logging.error("Handshake error with machine %s", hostname) - raise Exception("Hanshake error") + msg = "Unable to establish RPC connection to machine %s. " \ + % hostname + msg += "Handshake failed" + raise NetTestError(msg)
info["rpc"] = rpc
@@ -338,13 +340,13 @@ class NetTestController: # This is achieved by handling parser events (by registering try: self._ntparse.parse_recipe() - except Exception, exc: - log_exc_traceback() + except Exception as exc: logging.debug("Exception raised during recipe parsing. "\ "Deconfiguring machines.") + log_exc_traceback() self._deconfigure_slaves() self._disconnect_slaves() - raise exc + raise NetTestError(exc)
def _run_command(self, command): machine_id = command["machine_id"] @@ -365,8 +367,8 @@ class NetTestController: try: cmd_res = rpc.run_command(command) except socket.timeout: - logging.error("Slave reply timed out") - raise Exception("Slave reply timed out") + msg = "RPC connection to machine %s timed out" % machine_id + raise NetTestError(msg) if "timeout" in command: logging.debug("Setting socket timeout to default value") socket.setdefaulttimeout(None) @@ -394,8 +396,7 @@ class NetTestController: res_data = pformat(cmd_res["res_data"]) logging.info("Result data: %s", (res_data)) if not cmd_res["passed"]: - logging.error("Command failed - command: [%s], " - "Error message: "%s"", + logging.error("Command failed: [%s], Error message: "%s"", str_command(command), cmd_res["err_msg"]) seq_passed = False return seq_passed @@ -421,7 +422,10 @@ class NetTestController: err = None try: res = self._run_recipe() - except Exception, exc: + except ServerException as exc: + err = NetTestError(exc) + except Exception as exc: + logging.info("Recipe execution terminated by unexpected exception") log_exc_traceback() err = exc
@@ -481,7 +485,9 @@ class NetTestController: os.mkdir(slave_logging_dir) except OSError, err: if err.errno != 17: - raise + msg = "Cannot access the logging directory %s" \ + % slave_logging_dir + raise NetTestError(msg)
capture_files = self._remote_capture_files[machine_id] for remote_path in capture_files: diff --git a/NetTest/NetTestParse.py b/NetTest/NetTestParse.py index f2a3364..0d8678c 100644 --- a/NetTest/NetTestParse.py +++ b/NetTest/NetTestParse.py @@ -41,7 +41,7 @@ class NetTestParse(RecipeParser):
try: self._trigger_event("provisioning_requirements_ready", {}) - except Exception, exc: + except Exception as exc: raise XmlProcessingError(str(exc), xml_dom)
# process machine requirements if used in the recipe before @@ -296,7 +296,7 @@ class MachineConfigParse(RecipeParser): try: self._trigger_event("machine_info_ready", {"machine_id": self._machine_id}) - except Exception, exc: + except Exception as exc: raise XmlProcessingError(str(exc), node)
def _netdevices(self, node, params): @@ -339,7 +339,7 @@ class MachineConfigParse(RecipeParser): try: self._trigger_event("netdevice_ready", {"machine_id": self._machine_id, "dev_id": phys_id}) - except Exception, exc: + except Exception as exc: raise XmlProcessingError(str(exc), node)
@@ -394,8 +394,10 @@ class NetConfigParse(RecipeParser): self._trigger_event("interface_config_ready", {"machine_id": self._machine_id, "netdev_config_id": dev_id}) - except Exception, exc: - raise XmlProcessingError(str(exc), node) + except Exception as exc: + msg = "Unable to configure interface %s on machine %s [%s]." % \ + (dev_id, self._machine_id, str(exc)) + raise XmlProcessingError(msg, node)
def _process_phys_id_attr(self, node, dev): netconfig = self._netconfig diff --git a/nettestctl.py b/nettestctl.py index b0fb51d..201ed7f 100755 --- a/nettestctl.py +++ b/nettestctl.py @@ -100,7 +100,7 @@ def main(): ["debug", "help", "recipe=", "remoteexec", "cleanup", "result=", "packet_capture"] ) - except getopt.GetoptError, err: + except getopt.GetoptError as err: print str(err) usage() sys.exit()
lnst-developers@lists.fedorahosted.org