[PATCH 1/3] Break all the log monitoring stuff from LMC out into its own module.
Chris Lumens
clumens at redhat.com
Fri May 29 17:58:30 UTC 2015
---
src/pylorax/monitor.py | 191 +++++++++++++++++++++++++++++++++++++++++++++
src/sbin/livemedia-creator | 171 +---------------------------------------
2 files changed, 192 insertions(+), 170 deletions(-)
create mode 100644 src/pylorax/monitor.py
diff --git a/src/pylorax/monitor.py b/src/pylorax/monitor.py
new file mode 100644
index 0000000..f6acaee
--- /dev/null
+++ b/src/pylorax/monitor.py
@@ -0,0 +1,191 @@
+# monitor.py
+#
+# Copyright (C) 2011-2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Author(s): Brian C. Lane <bcl at redhat.com>
+#
+import logging
+log = logging.getLogger("livemedia-creator")
+
+import re
+import socket
+import socketserver
+import threading
+import time
+
+class LogRequestHandler(socketserver.BaseRequestHandler):
+ """
+ Handle monitoring and saving the logfiles from the virtual install
+
+ Incoming data is written to self.server.log_path and each line is checked
+ for patterns that would indicate that the installation failed.
+ self.server.log_error is set True when this happens.
+ """
+ def setup(self):
+ """Start writing to self.server.log_path"""
+
+ if self.server.log_path:
+ self.fp = open(self.server.log_path, "w") # pylint: disable=attribute-defined-outside-init
+ else:
+ self.fp = None
+ self.request.settimeout(10)
+
+ def handle(self):
+ """
+ Write incoming data to a logfile and check for errors
+
+ Split incoming data into lines and check for any Tracebacks or other
+ errors that indicate that the install failed.
+
+ Loops until self.server.kill is True
+ """
+ log.info("Processing logs from %s", self.client_address)
+ line = ""
+ while True:
+ if self.server.kill:
+ break
+
+ try:
+ data = str(self.request.recv(4096), "utf8")
+ if self.fp:
+ self.fp.write(data)
+ self.fp.flush()
+
+ # check the data for errors and set error flag
+ # need to assemble it into lines so we can test for the error
+ # string.
+ while data:
+ more = data.split("\n", 1)
+ line += more[0]
+ if len(more) > 1:
+ self.iserror(line)
+ line = ""
+ data = more[1]
+ else:
+ data = None
+
+ except socket.timeout:
+ pass
+ except Exception as e: # pylint: disable=broad-except
+ log.info("log processing killed by exception: %s", e)
+ break
+
+ def finish(self):
+ log.info("Shutting down log processing")
+ self.request.close()
+ if self.fp:
+ self.fp.close()
+
+ def iserror(self, line):
+ """
+ Check a line to see if it contains an error indicating installation failure
+
+ :param str line: log line to check for failure
+
+ If the line contains IGNORED it will be skipped.
+ """
+ if "IGNORED" in line:
+ return
+ simple_tests = ["Traceback (",
+ "Out of memory:",
+ "Call Trace:",
+ "insufficient disk space:",
+ "error populating transaction after",
+ "traceback script(s) have been run",
+ "packaging: Missed: NoSuchPackage"]
+ re_tests = [r"packaging: base repo .* not valid",
+ r"packaging: .* requires .*"]
+ for t in simple_tests:
+ if t in line:
+ self.server.log_error = True
+ self.server.error_line = line
+ return
+ for t in re_tests:
+ if re.search(t, line):
+ self.server.log_error = True
+ self.server.error_line = line
+ return
+
+
+class LogServer(socketserver.TCPServer):
+ """A TCP Server that listens for log data"""
+
+ # Number of seconds to wait for a connection after startup
+ timeout = 60
+
+ def __init__(self, log_path, *args, **kwargs):
+ """
+ Setup the log server
+
+ :param str log_path: Path to the log file to write
+ """
+ self.kill = False
+ self.log_error = False
+ self.error_line = ""
+ self.log_path = log_path
+ self._timeout = kwargs.pop("timeout", None)
+ if self._timeout:
+ self._start_time = time.time()
+ socketserver.TCPServer.__init__(self, *args, **kwargs)
+
+ def log_check(self):
+ """
+ Check to see if an error has been found in the log
+
+ :returns: True if there has been an error
+ :rtype: bool
+ """
+ if self._timeout:
+ taking_too_long = time.time() > self._start_time + (self._timeout * 60)
+ if taking_too_long:
+ log.error("Canceling installation due to timeout")
+ else:
+ taking_too_long = False
+ return self.log_error or taking_too_long
+
+
+class LogMonitor(object):
+ """
+ Setup a server to monitor the logs output by the installation
+
+ This needs to be running before the virt-install runs, it expects
+ there to be a listener on the port used for the virtio log port.
+ """
+ def __init__(self, log_path=None, host="localhost", port=0, timeout=None):
+ """
+ Start a thread to monitor the logs.
+
+ :param str log_path: Path to the logfile to write
+ :param str host: Host to bind to. Default is localhost.
+ :param int port: Port to listen to or 0 to pick a port
+
+ If 0 is passed for the port the dynamically assigned port will be
+ available as self.port
+
+ If log_path isn't set then it only monitors the logs, instead of
+ also writing them to disk.
+ """
+ self.server = LogServer(log_path, (host, port), LogRequestHandler, timeout=timeout)
+ self.host, self.port = self.server.server_address
+ self.log_path = log_path
+ self.server_thread = threading.Thread(target=self.server.handle_request)
+ self.server_thread.daemon = True
+ self.server_thread.start()
+
+ def shutdown(self):
+ """Force shutdown of the monitoring thread"""
+ self.server.kill = True
+ self.server_thread.join()
diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator
index 108871b..482d53b 100755
--- a/src/sbin/livemedia-creator
+++ b/src/sbin/livemedia-creator
@@ -29,15 +29,10 @@ import sys
import uuid
import tempfile
import subprocess
-import socket
-import threading
-import socketserver
-import time
from time import sleep
import shutil
import argparse
import hashlib
-import re
import glob
# Use pykickstart to calculate disk image size
@@ -59,6 +54,7 @@ from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount
from pylorax.imgutils import mksquashfs, mkqcow2, mktar, mkrootfsimg
from pylorax.imgutils import copytree
from pylorax.executils import execWithRedirect, execWithCapture, runcmd
+from pylorax.monitor import LogMonitor
# no-virt mode doesn't need libvirt, so make it optional
try:
@@ -77,171 +73,6 @@ class InstallError(Exception):
pass
-class LogRequestHandler(socketserver.BaseRequestHandler):
- """
- Handle monitoring and saving the logfiles from the virtual install
-
- Incoming data is written to self.server.log_path and each line is checked
- for patterns that would indicate that the installation failed.
- self.server.log_error is set True when this happens.
- """
- def setup(self):
- """Start writing to self.server.log_path"""
-
- if self.server.log_path:
- self.fp = open(self.server.log_path, "w") # pylint: disable=attribute-defined-outside-init
- else:
- self.fp = None
- self.request.settimeout(10)
-
- def handle(self):
- """
- Write incoming data to a logfile and check for errors
-
- Split incoming data into lines and check for any Tracebacks or other
- errors that indicate that the install failed.
-
- Loops until self.server.kill is True
- """
- log.info("Processing logs from %s", self.client_address)
- line = ""
- while True:
- if self.server.kill:
- break
-
- try:
- data = str(self.request.recv(4096), "utf8")
- if self.fp:
- self.fp.write(data)
- self.fp.flush()
-
- # check the data for errors and set error flag
- # need to assemble it into lines so we can test for the error
- # string.
- while data:
- more = data.split("\n", 1)
- line += more[0]
- if len(more) > 1:
- self.iserror(line)
- line = ""
- data = more[1]
- else:
- data = None
-
- except socket.timeout:
- pass
- except Exception as e: # pylint: disable=broad-except
- log.info("log processing killed by exception: %s", e)
- break
-
- def finish(self):
- log.info("Shutting down log processing")
- self.request.close()
- if self.fp:
- self.fp.close()
-
- def iserror(self, line):
- """
- Check a line to see if it contains an error indicating installation failure
-
- :param str line: log line to check for failure
-
- If the line contains IGNORED it will be skipped.
- """
- if "IGNORED" in line:
- return
- simple_tests = ["Traceback (",
- "Out of memory:",
- "Call Trace:",
- "insufficient disk space:",
- "error populating transaction after",
- "traceback script(s) have been run",
- "packaging: Missed: NoSuchPackage"]
- re_tests = [r"packaging: base repo .* not valid",
- r"packaging: .* requires .*"]
- for t in simple_tests:
- if t in line:
- self.server.log_error = True
- self.server.error_line = line
- return
- for t in re_tests:
- if re.search(t, line):
- self.server.log_error = True
- self.server.error_line = line
- return
-
-
-class LogServer(socketserver.TCPServer):
- """A TCP Server that listens for log data"""
-
- # Number of seconds to wait for a connection after startup
- timeout = 60
-
- def __init__(self, log_path, *args, **kwargs):
- """
- Setup the log server
-
- :param str log_path: Path to the log file to write
- """
- self.kill = False
- self.log_error = False
- self.error_line = ""
- self.log_path = log_path
- self._timeout = kwargs.pop("timeout", None)
- if self._timeout:
- self._start_time = time.time()
- socketserver.TCPServer.__init__(self, *args, **kwargs)
-
- def log_check(self):
- """
- Check to see if an error has been found in the log
-
- :returns: True if there has been an error
- :rtype: bool
- """
- if self._timeout:
- taking_too_long = time.time() > self._start_time + (self._timeout * 60)
- if taking_too_long:
- log.error("Canceling installation due to timeout")
- else:
- taking_too_long = False
- return self.log_error or taking_too_long
-
-
-class LogMonitor(object):
- """
- Setup a server to monitor the logs output by the installation
-
- This needs to be running before the virt-install runs, it expects
- there to be a listener on the port used for the virtio log port.
- """
- def __init__(self, log_path=None, host="localhost", port=0, timeout=None):
- """
- Start a thread to monitor the logs.
-
- :param str log_path: Path to the logfile to write
- :param str host: Host to bind to. Default is localhost.
- :param int port: Port to listen to or 0 to pick a port
-
- If 0 is passed for the port the dynamically assigned port will be
- available as self.port
-
- If log_path isn't set then it only monitors the logs, instead of
- also writing them to disk.
- """
- self.server = LogServer(log_path, (host, port), LogRequestHandler, timeout=timeout)
- self.host, self.port = self.server.server_address
- self.log_path = log_path
- self.server_thread = threading.Thread(target=self.server.handle_request)
- self.server_thread.daemon = True
- self.server_thread.start()
-
- def shutdown(self):
- """Force shutdown of the monitoring thread"""
- self.server.kill = True
- self.server_thread.join()
-
-
class IsoMountpoint(object):
"""
Mount the iso and check to make sure the vmlinuz and initrd.img files exist
--
2.2.2
More information about the anaconda-patches
mailing list