epylog scm migrated to git
by Seth Vidal
Hi folks,
I emailed the maintainers of epylog and got permission to take over dev
from all the folks who responded. (read: Konstantin).
So I did. First thing I did was to migrate from subversion to git. This
is not anything against git - it's just that I'm more used to git now
b/c I use it for everything else.
So the git repo is available at:
git clone git://git.fedorahosted.org/epylog.git
and browseable here: http://git.fedorahosted.org/git/?p=epylog.git
I've also checked in some module changes and at least one new module.
I'll be adding a few more things as I get time/inclination :)
patches welcome
-sv
12 years, 8 months
[epylog] add common_unparsed strings module add module config file modify weeder config to run, by default, b
by Seth Vidal
commit 322dee9862e5ecf3e91a49a001866ded4927d759
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Aug 25 11:08:08 2011 -0400
add common_unparsed strings module
add module config file
modify weeder config to run, by default, before common_unparsed strings
etc/modules.d/common_unparsed.conf.in | 12 ++++
etc/modules.d/weeder.conf.in | 2 +-
modules/common_unparsed_mod.py | 115 +++++++++++++++++++++++++++++++++
3 files changed, 128 insertions(+), 1 deletions(-)
---
diff --git a/etc/modules.d/common_unparsed.conf.in b/etc/modules.d/common_unparsed.conf.in
new file mode 100644
index 0000000..71a0620
--- /dev/null
+++ b/etc/modules.d/common_unparsed.conf.in
@@ -0,0 +1,12 @@
+[module]
+desc = Common Unparsed Similar Strings Module
+exec = %%MODULES_DIR%%/common_unparsed_mod.py
+files = /var/log/messages[.#] /var/log/secure[.#]
+enabled = yes
+internal = yes
+outhtml = yes
+priority = 10
+
+[conf]
+# how similar the strings need to be 0-100 - 0 being not at all (bad idea) 100 being almost exactly.
+match_percentage = 95
diff --git a/etc/modules.d/weeder.conf.in b/etc/modules.d/weeder.conf.in
index 591c602..f32756e 100644
--- a/etc/modules.d/weeder.conf.in
+++ b/etc/modules.d/weeder.conf.in
@@ -5,7 +5,7 @@ files = /var/log/messages[.#], /var/log/secure[.#], /var/log/maillog[.#], /var/l
enabled = yes
internal = yes
outhtml = yes
-priority = 10
+priority = 9
[conf]
##
diff --git a/modules/common_unparsed_mod.py b/modules/common_unparsed_mod.py
new file mode 100644
index 0000000..1b1e651
--- /dev/null
+++ b/modules/common_unparsed_mod.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python -tt
+"""
+This module should run after all other modules.
+
+Takes the remaining logs, saves them out to a tmpfile.
+Uses difflib.SequenceMatcher() to return logs which occur most often and are
+at least a set percentage similar. This lets you catch logs which are
+from multiple machines and more or less the same or similar logs (with memory
+or process id offsets) from the same machine.
+
+"""
+
+import sys
+import re
+import difflib
+from operator import itemgetter
+import tempfile
+import os
+import shutil
+
+##
+# This is for testing purposes, so you can invoke this from the
+# modules directory. See also the testing notes at the end of the
+# file.
+#
+sys.path.insert(0, '../py/')
+from epylog import Result, InternalModule
+
+class common_unparsed_mod(InternalModule):
+ def __init__(self, opts, logger):
+ InternalModule.__init__(self)
+ self.logger = logger
+ rc = re.compile
+
+ self.match_percentage = int(opts.get('match_percentage', '95'))
+ self.debug_dump = int(opts.get('debug_dump', '0'))
+ self.regex_map = {rc('^.+$'):self.expand_out_line}
+ self.tmpdir = tempfile.mkdtemp(prefix='epylog-common-unparsed', dir='/var/tmp')
+ self.matchfile = self.tmpdir + '/match_limited'
+ self.matchfile_f = open(self.matchfile, 'w')
+ self.complete = self.tmpdir + '/complete'
+ self.complete_f = open(self.complete, 'w')
+
+
+ ##
+ # Line-matching routines
+ #
+ def expand_out_line(self, linemap):
+ sys, msg, mult = self.get_smm(linemap)
+ matchout = '%s\n' % (msg)
+ com_out = '%s' % (linemap['line'])
+ for i in range(0, int(mult)):
+ self.complete_f.write(com_out)
+ self.matchfile_f.write(matchout)
+
+ #dirty like zebra
+ if not os.path.exists(self.tmpdir + '/returned'):
+ open(self.tmpdir + '/returned', 'w').close()
+ return {'match':0}
+ else:
+ return None
+
+
+ def finalize(self, rs):
+ self.complete_f.close()
+ self.matchfile_f.close()
+ matches = {}
+ full_matches = {}
+ lines_matched = set()
+ where = 0
+ rl = open(self.matchfile, 'r')
+ for l in rl:
+ where += 1
+ sub_where = 0
+ for ol in open(self.matchfile, 'r'):
+ sub_where += 1
+ if sub_where < where: # if we're before it in the file we've already matched it
+ continue
+ if sub_where in lines_matched:
+ continue
+
+ c = difflib.SequenceMatcher(isjunk=lambda x: x in ('1','2','3','4','5','6','7','8','9','0'),a=l, b=ol)
+ rq_ratio = int(c.real_quick_ratio()*100) # do the fast upper bound - find out if we should even glance at it.
+ if rq_ratio < 60:
+ continue
+ ratio = int(c.ratio()*100)
+ if ratio >= self.match_percentage:
+ lines_matched.add(sub_where)
+ if l not in matches:
+ matches[l] = 0
+ full_matches[l] = []
+ matches[l] += 1
+ full_matches[l].append((ol,ratio))
+ if self.debug_dump:
+ fm_db = open(self.tmpdir + '/debug-match', 'w')
+ for k in full_matches:
+ fm_db.write(k)
+ for (v,r) in full_matches[k]:
+ fm_db.write(' %s %s' % (r, v))
+ fm_db.close()
+
+ res = "<table border=0><tr><th>Count</th><th align=left>Log</th></tr>\n\n"
+ for (k,v) in sorted(matches.items(), key=itemgetter(1), reverse=True)[:20]: # take the top 20 most common provided there are more than 1
+ if v > 1:
+ res += "<tr>\n<td bgcolor='#DDDDDD'>%s</td><td>%s</td>\n</tr>" % (v, k)
+ res += "<tr>\n<td colspan=2>\n<h2>Complete messages</h2>\n<pre>\n"
+ res += ''.join(sorted(open(self.complete, 'r').readlines()))
+ res += "</pre>\n</td>\n</tr>\n</table>"
+ return res
+
+
+
+if __name__ == '__main__':
+ from epylog.helpers import ModuleTest
+ ModuleTest(common_unparsed_mod, sys.argv)
12 years, 8 months
[epylog] fix helpers and loggers minor typos which keep the module tests from being able to run
by Seth Vidal
commit 143f1df308dbdef7a0c2db18d93714504a9a0c0c
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Aug 24 17:05:01 2011 -0400
fix helpers and loggers minor typos which keep the module tests from being able to run
py/epylog/helpers.py | 4 ++--
py/epylog/log.py | 1 -
2 files changed, 2 insertions(+), 3 deletions(-)
---
diff --git a/py/epylog/helpers.py b/py/epylog/helpers.py
index 32550e6..81fea8a 100644
--- a/py/epylog/helpers.py
+++ b/py/epylog/helpers.py
@@ -79,7 +79,7 @@ class ModuleTest:
except Exception, e:
msg = "ERROR trying to open file %s: %s" % (filtfile, e)
self._die(msg)
- monthmap = epylog.log.mkmonthmap()
+ monthmap = epylog.log.mkMonthMap()
rs = epylog.Result()
while 1:
line = infh.readline()
@@ -121,7 +121,7 @@ class ModuleTest:
Create a linemap out of a line entry.
"""
try:
- stamp, sys, msg = epylog.log.get_stamp_sys_msg(line, monthmap)
+ stamp, sys, msg = epylog.log.getStampSysMsg(line, monthmap)
except ValueError:
# If we got an empty log line just go on parsing with nothing
# instead of dying
diff --git a/py/epylog/log.py b/py/epylog/log.py
index d648331..ce22b88 100644
--- a/py/epylog/log.py
+++ b/py/epylog/log.py
@@ -33,7 +33,6 @@ import string
import time
import tempfile
-from elementtree import ElementTree
if 'mkdtemp' not in dir(tempfile):
##
12 years, 8 months
[epylog] add rsyncd module
by Seth Vidal
commit b604e4669bd971dfaa57cce81837aac7cf795c75
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Aug 24 17:04:39 2011 -0400
add rsyncd module
etc/modules.d/rsyncd.conf.in | 15 +++
modules/rsyncd_mod.py | 219 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 234 insertions(+), 0 deletions(-)
---
diff --git a/etc/modules.d/rsyncd.conf.in b/etc/modules.d/rsyncd.conf.in
new file mode 100644
index 0000000..a271f13
--- /dev/null
+++ b/etc/modules.d/rsyncd.conf.in
@@ -0,0 +1,15 @@
+[module]
+desc = Rsyncd
+exec = %%MODULES_DIR%%/rsyncd_mod.py
+files = /var/log/messages[.#]
+enabled = no
+internal = yes
+outhtml = yes
+priority = 7
+
+[conf]
+##
+# Report this many "top ranking hosts"
+#
+report_top = 10
+# ignore_hosts = list of dns-resolved host names you want to ignore rsyncs on
diff --git a/modules/rsyncd_mod.py b/modules/rsyncd_mod.py
new file mode 100644
index 0000000..5ec899c
--- /dev/null
+++ b/modules/rsyncd_mod.py
@@ -0,0 +1,219 @@
+#!/usr/bin/python -tt
+"""
+Rsyncd log parsing module for Epylog
+"""
+
+##
+# Copyright (C) 2003 by Duke University
+# Written by Seth Vidal <skvidal at phy.duke.edu>
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+# 02111-1307, USA.
+#
+
+
+import sys
+import re
+
+##
+# This is for testing purposes, so you can invoke this from the
+# modules directory. See also the testing notes at the end of the
+# file.
+#
+sys.path.insert(0, '../py/')
+from epylog import Result, InternalModule
+
+class rsyncd_mod(InternalModule):
+ ##
+ # opts: is a map with extra options set in
+ # [conf] section of the module config, or on the
+ # command line using -o flag to the module.
+ # logger: A logging object. API:
+ # logger.put(loglvl, 'Message')
+ # Only critical stuff needs to go onto lvl 0.
+ # Common output goes to lvl 1.
+ # Others are debug levels.
+ #
+ def __init__(self, opts, logger):
+ ##
+ # Do a "super-init" so the class we are subclassing gets
+ # instantiated.
+ #
+ InternalModule.__init__(self)
+ self.logger = logger
+ ##
+ # Convenience
+ #
+ rc = re.compile
+
+ self.regex_map = {
+ rc('rsyncd\[\d+\]: rsync on'): self.rsync_hosts,
+ rc('rsyncd\[\d+\]: (?:sent|wrote)\s\S*\sbytes'): self.rsync_results
+ }
+ self.topcount = int(opts.get('report_top', 5)) #get report_top, default to 5 if not set
+ ig_s = opts.get('ignore_hosts', '')
+ ig_s.replace(',',' ')
+ self.ignore_hosts = ig_s.split(' ')
+ # dict to store all of our data
+ self.rsync_pid_bytes = {}
+ self.rsync_pid_host = {}
+ self.rsync_host_loc = rc('rsyncd\[(\d+)\]: rsync\son\s(\S*)\sfrom\s.*\((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\)')
+ self.rsync_bytes = rc('rsyncd\[(\d+)\]: (?:sent|wrote)\s(\d+) bytes (?:read|received)\s(\d+) bytes total size (\d+)')
+
+ def rsync_hosts(self, linemap):
+ (sys, msg, multi) = self.get_smm(linemap)
+ pid, loc, ip = self.rsync_host_loc.search(msg).groups()
+ host = self.gethost(ip)
+ if host not in self.ignore_hosts:
+ self.rsync_pid_host[pid] = (host, loc)
+ return {(loc, host): 1}
+
+ def rsync_results(self, linemap):
+ (sys, msg, multi) = self.get_smm(linemap)
+ pid, wbytes, rbytes, tbytes = self.rsync_bytes.search(msg).groups()
+ self.rsync_pid_bytes[pid] = (wbytes, rbytes, tbytes)
+ return {(pid, wbytes): 1}
+
+ def _uniq(self, s):
+ """Return a list of the elements in s, but without duplicates.
+
+ For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3],
+ unique("abcabc") some permutation of ["a", "b", "c"], and
+ unique(([1, 2], [2, 3], [1, 2])) some permutation of
+ [[2, 3], [1, 2]].
+
+ For best speed, all sequence elements should be hashable. Then
+ unique() will usually work in linear time.
+
+ If not possible, the sequence elements should enjoy a total
+ ordering, and if list(s).sort() doesn't raise TypeError it's
+ assumed that they do enjoy a total ordering. Then unique() will
+ usually work in O(N*log2(N)) time.
+
+ If that's not possible either, the sequence elements must support
+ equality-testing. Then unique() will usually work in quadratic
+ time.
+ """
+
+ n = len(s)
+ if n == 0:
+ return []
+
+ # Try using a dict first, as that's the fastest and will usually
+ # work. If it doesn't work, it will usually fail quickly, so it
+ # usually doesn't cost much to *try* it. It requires that all the
+ # sequence elements be hashable, and support equality comparison.
+ u = {}
+ try:
+ for x in s:
+ u[x] = 1
+ except TypeError:
+ del u # move on to the next method
+ else:
+ return u.keys()
+
+ # We can't hash all the elements. Second fastest is to sort,
+ # which brings the equal elements together; then duplicates are
+ # easy to weed out in a single pass.
+ # NOTE: Python's list.sort() was designed to be efficient in the
+ # presence of many duplicate elements. This isn't true of all
+ # sort functions in all languages or libraries, so this approach
+ # is more effective in Python than it may be elsewhere.
+ try:
+ t = list(s)
+ t.sort()
+ except TypeError:
+ del t # move on to the next method
+ else:
+ assert n > 0
+ last = t[0]
+ lasti = i = 1
+ while i < n:
+ if t[i] != last:
+ t[lasti] = last = t[i]
+ lasti += 1
+ i += 1
+ return t[:lasti]
+
+ # Brute force is all that's left.
+ u = []
+ for x in s:
+ if x not in u:
+ u.append(x)
+ return u
+
+ def _sortByVal(self, dict, reverse=0):
+ if type(dict) is not type({}): return []
+ keys = dict.keys()
+ s = map(lambda k: (dict[k], k), keys)
+ s.sort()
+ if reverse: s.reverse()
+ return s
+
+ def finalize(self, resultset):
+ ##
+ # A resultset is a dictionary of all values returned by your
+ # handler functions -- except they are unique and show how many
+ # times each tuple occurs.
+ # See epylog.Result for some convenience methods to use when
+ # processing and analyzing the results.
+ #
+
+ hostloc = {} # key = host, val = [loc, loc, loc]
+ hosttotal = {} # key = host val = totalwbytes
+
+ foo = "<table border=0>\n\t<tr>\n"
+
+ for pid in self.rsync_pid_host.keys():
+ (host, loc) = self.rsync_pid_host[pid]
+ if self.rsync_pid_bytes.has_key(pid):
+ if not hostloc.has_key(host):
+ hostloc[host] = []
+ if not hosttotal.has_key(host):
+ hosttotal[host] = 0L
+ hostloc[host].append(loc)
+ bytes = long(self.rsync_pid_bytes[pid][0])
+ hosttotal[host] += bytes
+
+ for host in hostloc.keys():
+ hostloc[host] = self._uniq(hostloc[host])
+
+ hosts = self._sortByVal(hosttotal, 1)
+ count = 0L
+ for (tot,host) in hosts[:self.topcount]:
+ if count % 2:
+ bgcolor = "#dddddd"
+ else:
+ bgcolor = "#ffffff"
+ count+=1
+ line = '\t\t<td bgcolor=%s valign=\"top\">%s</td>\n' % (bgcolor, host)
+ line = line + '\t\t<td bgcolor=%s valign="top">\n' % bgcolor
+ for loc in hostloc[host]:
+ line = line + '\t\t\t%s<br>\n' % loc
+ line = line + '\t\t</td>\n'
+ size, marker = self.mk_size_unit(hosttotal[host])
+ line = line + '\t\t<td bgcolor=%s valign="top">%s%s</td>\n' % (bgcolor, size, marker)
+ line = line + '\t</tr>\n'
+ foo = foo + line
+ foo = foo + '</table>\n'
+ return foo
+
+##
+# This is useful when testing your module out.
+# Invoke without command-line parameters to learn about the proper
+# invocation.
+#
+if __name__ == '__main__':
+ from epylog.helpers import ModuleTest
+ ModuleTest(rsyncd_mod, sys.argv)
12 years, 8 months
[epylog] Created branch master
by Seth Vidal
The branch 'master' was created pointing to:
7958a91... removed ntp notice, the message is included in the module
12 years, 8 months