Change in vdsm[master]: contrib: Add logdb tool for importing logs to a database
nsoffer at redhat.com
nsoffer at redhat.com
Sun Mar 15 12:21:38 UTC 2015
Nir Soffer has uploaded a new change for review.
Change subject: contrib: Add logdb tool for importing logs to a database
......................................................................
contrib: Add logdb tool for importing logs to a database
Using a database we have more power to dissect big amount of logs and
extract useful statistics.
Examples:
Breakdown of vgs calls every 10 minutes interval:
$ logdb vdsm-all.db vdsm.log.*
$ sqlite3 vdsm-all.db "
select
datetime(strftime('%s',timestamp)/600*600,'unixepoch')
as interval,count()
from messages
where
func = 'cmd' and
text glob '*/sbin/lvm vgs*' and
timestamp > '2015-03-11 05:30:00'
group by interval
"
2015-03-11 05:30:00|2
2015-03-11 05:40:00|5
2015-03-11 05:50:00|8
2015-03-11 06:00:00|16
2015-03-11 06:10:00|2
2015-03-11 06:20:00|2
2015-03-11 06:30:00|2
2015-03-11 06:40:00|2
2015-03-11 06:50:00|2
Show vm xml for vms started on 2015-03-11:
$ sqlite3 vdsm-all.db "
select timestamp,text from messages
where
text glob '*<domain *' and
date(timestamp) = '2015-03-11'
"
Change-Id: If60fc70ff5d76d783da776a9c7941fcdf2309eaa
Signed-off-by: Nir Soffer <nsoffer at redhat.com>
---
A contrib/logdb
1 file changed, 110 insertions(+), 0 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/46/38746/1
diff --git a/contrib/logdb b/contrib/logdb
new file mode 100755
index 0000000..6802afc
--- /dev/null
+++ b/contrib/logdb
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+
+"""
+Usage: logdb dbname logfile ...
+
+Import vdsm log files into log database dbname.
+"""
+
+import sqlite3
+import sys
+
+
+CREATE_TABLE = """
+create table if not exists messages(
+ timestamp datetime,
+ level text,
+ logger text,
+ thread text,
+ module text,
+ lineno int,
+ func text,
+ text text
+);
+"""
+
+DROP_INDEXES = """
+drop index if exists timestamp;
+drop index if exists level;
+drop index if exists thread;
+drop index if exists module;
+drop index if exists func;
+"""
+
+CREATE_INDEXES = """
+create index timestamp on messages (timestamp);
+create index level on messages (level);
+create index thread on messages (thread);
+create index module on messages (module);
+create index func on messages (func);
+"""
+
+
+def main(args):
+ if len(args) < 2:
+ print __doc__
+ sys.exit(2)
+
+ dbname = args.pop(0)
+ with sqlite3.connect(dbname) as con:
+ con.execute(CREATE_TABLE)
+ con.executescript(DROP_INDEXES)
+ for logfile in args:
+ with open(logfile) as f:
+ for msg in parse(f):
+ con.execute("insert into messages values "
+ "(?,?,?,?,?,?,?,?)", msg)
+ con.executescript(CREATE_INDEXES)
+ con.commit()
+
+
+def parse(logfile):
+ msg = None
+ for line in logfile:
+ line = line.rstrip()
+ try:
+ # thread::level::timestamp::module::lineno::logger::(func) message
+ thread, level, timestamp, module, lineno, logger, rest = \
+ line.split("::", 6)
+ except ValueError:
+ # Some message text has multiple lines
+ if msg:
+ msg[-1] += "\n" + line
+ else:
+ # Y-m-d H-M-S,msec -> Y-m-d H-M-S.msec
+ timestamp = timestamp.replace(",", ".")
+ # (func) text
+ try:
+ func, text = rest.split(" ", 1)
+ except ValueError:
+ func, text = rest, ""
+ func = func[1:-1]
+ if msg:
+ yield tuple(msg)
+ msg = [timestamp, level, logger, thread, module, lineno, func,
+ text]
+ if msg:
+ yield tuple(msg)
+
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
--
To view, visit https://gerrit.ovirt.org/38746
To unsubscribe, visit https://gerrit.ovirt.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If60fc70ff5d76d783da776a9c7941fcdf2309eaa
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Nir Soffer <nsoffer at redhat.com>
More information about the vdsm-patches
mailing list