Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=bed... Commit: bedaeebc12fac2e5568249357835f210174f4240 Parent: 8fde4dae1ee517ca1f2ce2c48d006797e59e923e Author: Chris Feist cfeist@redhat.com AuthorDate: Tue Feb 1 14:15:04 2011 -0600 Committer: Chris Feist cfeist@redhat.com CommitterDate: Wed Feb 2 16:04:36 2011 -0600
cman: Added checkquorum script for self fencing
A checkquorum script has been added which when copied to the /etc/watchdog.d directory will cause the machine to node to reboot itself if it has lost quorum for ~60 seconds.
Resolves: rhbz#560700 --- cman/Makefile | 2 +- cman/man/Makefile | 3 +- cman/man/checkquorum.8 | 29 ++++++++++++++ cman/scripts/Makefile | 10 +++++ cman/scripts/checkquorum | 97 ++++++++++++++++++++++++++++++++++++++++++++++ make/install.mk | 4 ++ 6 files changed, 143 insertions(+), 2 deletions(-)
diff --git a/cman/Makefile b/cman/Makefile index ead0baa..1cf8bc9 100644 --- a/cman/Makefile +++ b/cman/Makefile @@ -1,4 +1,4 @@ include ../make/defines.mk include $(OBJDIR)/make/passthrough.mk
-SUBDIRS=lib cman_tool daemon qdisk notifyd init.d man +SUBDIRS=lib cman_tool daemon qdisk notifyd init.d man scripts diff --git a/cman/man/Makefile b/cman/man/Makefile index df20abb..f7fbebf 100644 --- a/cman/man/Makefile +++ b/cman/man/Makefile @@ -5,7 +5,8 @@ MANTARGET= \ qdiskd.8 \ mkqdisk.8 \ cmannotifyd.8 \ - cman_notify.8 + cman_notify.8 \ + checkquorum.8
include ../../make/defines.mk include $(OBJDIR)/make/install.mk diff --git a/cman/man/checkquorum.8 b/cman/man/checkquorum.8 new file mode 100644 index 0000000..96f61f0 --- /dev/null +++ b/cman/man/checkquorum.8 @@ -0,0 +1,29 @@ +.TH "checkquorum" "8" "February 2011" "" "Check Quorum Watchdog Script" +.SH "NAME" +checkquorum - Check Quorum Watchdog Script +.SH "SYNOPSIS" +\fBcheckquorum +.SH "DESCRIPTION" +.PP +The \fBcheckquorum\fP watchdog script, when copied to the +.IR /etc/watchdog.d +directory and after enabling/starting the watchdog daemon causes the node to reboot if quorum is +lost and not regained within a user configurable amount of time (default: 60 seconds). +.SH "OPTIONS" +The checkquorum script includes several options which can be set by editing +the script with a text editor. +.TP +.BR $wait_time +Amount of time in seconds to wait after quorum is lost before trigger a reboot +(Default: 60 seconds). +.TP +.BR $hardreboot +Instantly reboot the machine without cleanly shutting down the system. +Useful when the machine may hang on reboot. Set to 1 to hard reboot the +system, 0 to do a normal reboot. +.SH "NOTES" +\fBcheckquorum\fP should never be called outside of watchdog except for +debugging purposes. + +.SH "SEE ALSO" +watchdog(8) diff --git a/cman/scripts/Makefile b/cman/scripts/Makefile new file mode 100644 index 0000000..b4866c8 --- /dev/null +++ b/cman/scripts/Makefile @@ -0,0 +1,10 @@ +SHAREDIRTEX=checkquorum + +include ../../make/defines.mk +include $(OBJDIR)/make/clean.mk +include $(OBJDIR)/make/install.mk +include $(OBJDIR)/make/uninstall.mk + +all: + +clean: generalclean diff --git a/cman/scripts/checkquorum b/cman/scripts/checkquorum new file mode 100755 index 0000000..43cbc6d --- /dev/null +++ b/cman/scripts/checkquorum @@ -0,0 +1,97 @@ +#!/usr/bin/perl -w +# Quorum detection watchdog script +# +# This script will return -2 if the node had quorum at one point +# and then subsequently lost it +# +# Copyright 2011 Red Hat, Inc. + +# Amount of time in seconds to wait after quorum is lost to fail script +$wait_time = 60; + +# Hard Reboot the system (doesn't cleanly shut down the system) +$hardreboot = 0; + +# Location of temporary file to capture timeouts +$timerfile = "/var/run/cluster/checkquorum-timer"; + +# Enable debug messages (0 to disable, 1 to enable) +$debugval = 0; + +# If command is called attempting to 'repair' we automatically fail +if (($#ARGV != -1) && ($ARGV[0] eq "repair")) { + debug ("Failing on repair\n"); + exit 1; +} + +if (!quorum()) { + if (has_quorum_already_been_formed()) { + debug("Quorum has already existed, node can be self fenced!\n"); + if (-e $timerfile) { + $tf = open (FILE, "$timerfile"); + $time = <FILE>; + close (FILE); + $timediff = time() - $time; + if ($timediff >= $wait_time) { + self_fence() + } else { + $remaining = $wait_time - $timediff; + debug("Time has not exceeded wait time ($remaining seconds remaining).\n"); + } + } else { + debug("Creating timer file...\n"); + $tf = open (FILE, ">$timerfile"); + print FILE time(); + close (FILE); + } + } else { + debug("This is a new startup no self-fencing will occur.\n"); + `rm -f $timerfile`; + } +} else { + debug("Quorum exists, no self-fencing should occur.\n"); + `rm -f $timerfile`; +} + +sub has_quorum_already_been_formed { + $oe = `/usr/sbin/corosync-objctl 2>&1 | grep -E "runtime.totem.pg.mrp.srp.operational_entered|Could not initialize objdb library|Cannot connect to quorum service" `; + if ($oe =~ /^Could not/ || $oe =~ /^Cannot/) { + debug("corosync is not running\n"); + exit 0; + } + $oe =~ s/.*=//; + if ($oe > 1) { + return 1; + } else { + return 0; + } +} + +sub quorum { + $cq = `corosync-quorumtool -s 2>&1 | grep -E "Quorate:|Cannot connect to quorum service"`; + if ($cq =~ /Cannot connect to quorum service/) { + debug("corosync is not running\n"); + exit 0; + } + $cq =~ s/Quorate: *//; + chomp ($cq); + return 1 if ($cq eq "Yes"); + return 0; +} + +sub self_fence { + debug("Self fencing commencing...\n"); + `rm -f $timerfile`; + if ($hardreboot == 1) { + `echo 1 > /proc/sys/kernel/sysrq`; + `echo b > /proc/sysrq-trigger`; + } + exit -2; +} + +sub debug { + $out = pop(@_); + if ($debugval) { + print $out; + } +} diff --git a/make/install.mk b/make/install.mk index 8e1bcd6..823184b 100644 --- a/make/install.mk +++ b/make/install.mk @@ -78,6 +78,10 @@ ifdef PKGCONF install -d ${pkgconfigdir} install -m644 ${PKGCONF} ${pkgconfigdir} endif +ifdef SHAREDIRTEX + install -d ${sharedir} + install -m755 ${SHAREDIRTEX} ${sharedir} +endif ifdef SHAREDIRT install -d ${sharedir} install -m644 ${SHAREDIRT} ${sharedir}
cluster-commits@lists.fedorahosted.org