cluster: RHEL56 - fence_cisco_ucs: New fence agent for Cisco UCS
by Marek Grác
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: f2f0db60469f811f26123eec87cfc09d9002f35d
Parent: 20433b6cab6465b2cdaefb3d383d12da68cab309
Author: Marek 'marx' Grac <mgrac(a)redhat.com>
AuthorDate: Sat Sep 25 00:47:06 2010 +0200
Committer: Marek 'marx' Grac <mgrac(a)redhat.com>
CommitterDate: Sat Sep 25 00:47:06 2010 +0200
fence_cisco_ucs: New fence agent for Cisco UCS
Resolves: rhbz#579859
---
fence/agents/Makefile | 3 +
fence/agents/cisco_ucs/Makefile | 38 ++++++++
fence/agents/cisco_ucs/fence_cisco_ucs.py | 138 +++++++++++++++++++++++++++++
3 files changed, 179 insertions(+), 0 deletions(-)
diff --git a/fence/agents/Makefile b/fence/agents/Makefile
index 4a043ae..0055288 100644
--- a/fence/agents/Makefile
+++ b/fence/agents/Makefile
@@ -22,6 +22,7 @@ all:
${MAKE} -C brocade all
${MAKE} -C bullpap all
${MAKE} -C cisco_mds all
+ ${MAKE} -C cisco_ucs all
# ${MAKE} -C cpint all
${MAKE} -C drac all
${MAKE} -C egenera all
@@ -54,6 +55,7 @@ install: all
${MAKE} -C brocade install
${MAKE} -C bullpap install
${MAKE} -C cisco_mds install
+ ${MAKE} -C cisco_ucs install
# ${MAKE} -C cpint install
${MAKE} -C drac install
${MAKE} -C egenera install
@@ -86,6 +88,7 @@ clean:
${MAKE} -C brocade clean
${MAKE} -C bullpap clean
${MAKE} -C cisco_mds clean
+ ${MAKE} -C cisco_ucs clean
# ${MAKE} -C cpint clean
${MAKE} -C drac clean
${MAKE} -C egenera clean
diff --git a/fence/agents/cisco_ucs/Makefile b/fence/agents/cisco_ucs/Makefile
new file mode 100644
index 0000000..5875ba0
--- /dev/null
+++ b/fence/agents/cisco_ucs/Makefile
@@ -0,0 +1,38 @@
+###############################################################################
+###############################################################################
+##
+## Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
+## Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+##
+## This copyrighted material is made available to anyone wishing to use,
+## modify, copy, or redistribute it subject to the terms and conditions
+## of the GNU General Public License v.2.
+##
+###############################################################################
+###############################################################################
+
+SOURCE= fence_cisco_ucs.py
+TARGET= fence_cisco_ucs
+
+top_srcdir=../..
+include ${top_srcdir}/make/defines.mk
+
+all: $(TARGET)
+
+fence_cisco_ucs: fence_cisco_ucs.py
+ : > $(TARGET)
+ awk "{print}(\$$1 ~ /#BEGIN_VERSION_GENERATION/){exit 0}" $(SOURCE) >> $(TARGET)
+ echo "FENCE_RELEASE_NAME=\"${RELEASE}\";" >> $(TARGET)
+ ${top_srcdir}/scripts/define2var ${top_srcdir}/config/copyright.cf sh REDHAT_COPYRIGHT >> $(TARGET)
+ echo "BUILD_DATE=\"(built `date`)\";" >> $(TARGET)
+ awk -v p=0 "(\$$1 ~ /#END_VERSION_GENERATION/){p = 1} {if(p==1)print}" $(SOURCE) >> $(TARGET)
+ chmod +x $(TARGET)
+
+install: all
+ if [ ! -d ${sbindir} ]; then \
+ install -d ${sbindir}; \
+ fi
+ install -m755 ${TARGET} ${sbindir}
+
+clean:
+ rm -f $(TARGET)
diff --git a/fence/agents/cisco_ucs/fence_cisco_ucs.py b/fence/agents/cisco_ucs/fence_cisco_ucs.py
new file mode 100755
index 0000000..07375b3
--- /dev/null
+++ b/fence/agents/cisco_ucs/fence_cisco_ucs.py
@@ -0,0 +1,138 @@
+#!/usr/bin/python
+
+import sys, re, pexpect, socket
+import pycurl, StringIO
+sys.path.append("@FENCEAGENTSLIBDIR@")
+from fencing import *
+
+#BEGIN_VERSION_GENERATION
+RELEASE_VERSION="New Cisco UCS Agent - test release on steroids"
+REDHAT_COPYRIGHT=""
+BUILD_DATE="March, 2008"
+#END_VERSION_GENERATION
+
+re_cookie = re.compile("<aaaLogin .* outCookie=\"(.*?)\"", re.IGNORECASE);
+re_status = re.compile("<lsPower .*? state=\"(.*?)\"", re.IGNORECASE);
+re_get_dn = re.compile(" dn=\"(.*?)\"", re.IGNORECASE)
+re_get_desc = re.compile(" descr=\"(.*?)\"", re.IGNORECASE)
+
+def get_power_status(conn, options):
+ try:
+ res = send_command(options, "<configResolveDn cookie=\"" + options["cookie"] + "\" inHierarchical=\"false\" dn=\"org-root/ls-" + options["-n"] + "/power\"/>")
+ except pycurl.error, e:
+ sys.stderr.write(e[1] + "\n")
+ fail(EC_TIMED_OUT)
+
+ result = re_status.search(res)
+ if (result == None):
+ fail(EC_STATUS)
+ else:
+ status = result.group(1)
+
+ if (status == "up"):
+ return "on"
+ else:
+ return "off"
+
+def set_power_status(conn, options):
+ action = {
+ 'on' : "up",
+ 'off' : "down"
+ }[options["-o"]]
+
+ try:
+ res = send_command(options, "<configConfMos cookie=\"" + options["cookie"] + "\" inHierarchical=\"no\"><inConfigs><pair key=\"org-root/ls-" + options["-n"] + "/power\"><lsPower dn=\"org-root/ls-" + options["-n"] + "/power\" state=\"" + action + "\" status=\"modified\" /></pair></inConfigs></configConfMos>")
+ except pycurl.error, e:
+ sys.stderr.write(e[1] + "\n")
+ fail(EC_TIMED_OUT)
+
+ return
+
+def get_list(conn, options):
+ outlets = { }
+
+ try:
+ try:
+ res = send_command(options, "<configResolveClass cookie=\"" + options["cookie"] + "\" inHierarchical=\"false\" classId=\"lsServer\"/>")
+ except pycurl.error, e:
+ sys.stderr.write(e[1] + "\n")
+ fail(EC_TIMED_OUT)
+
+ lines = res.split("<lsServer ")
+ for i in range(1, len(lines)):
+ dn = re_get_dn.search(lines[i]).group(1)
+ desc = re_get_desc.search(lines[i]).group(1)
+ outlets[dn] = (desc, None)
+ except AttributeError:
+ return { }
+ except IndexError:
+ return { }
+
+ return outlets
+
+def send_command(opt, command):
+ ## setup correct URL
+ if opt.has_key("-z"):
+ url = "https:"
+ else:
+ url = "http:"
+
+ url += "//" + opt["-a"] + ":" + str(opt["-u"]) + "/nuova"
+
+ ## send command through pycurl
+ c = pycurl.Curl()
+ b = StringIO.StringIO()
+ c.setopt(pycurl.URL, url)
+ c.setopt(pycurl.HTTPHEADER, [ "Content-type: text/xml" ])
+ c.setopt(pycurl.POSTFIELDS, command)
+ c.setopt(pycurl.WRITEFUNCTION, b.write)
+ c.setopt(pycurl.TIMEOUT, int(opt["-Y"]))
+ c.setopt(pycurl.SSL_VERIFYPEER, 0)
+ c.perform()
+ result = b.getvalue()
+
+ if opt["log"] >= LOG_MODE_VERBOSE:
+ opt["debug_fh"].write(command + "\n")
+ opt["debug_fh"].write(result + "\n")
+
+ return result
+
+def main():
+ device_opt = [ "help", "version", "agent", "quiet", "verbose", "debug",
+ "action", "ipaddr", "login", "passwd", "passwd_script",
+ "ssl", "inet4_only", "inet6_only", "ipport", "port",
+ "web", "separator", "power_wait", "power_timeout",
+ "shell_timeout" ]
+
+ atexit.register(atexit_handler)
+
+ options = check_input(device_opt, process_input(device_opt))
+
+ docs = { }
+ docs["shortdesc"] = "Fence agent for Cisco UCS"
+ docs["longdesc"] = "fence_cisco_ucs is an I/O Fencing agent which can be \
+used with Cisco UCS to fence machines."
+ docs["vendorurl"] = "http://www.cisco.com"
+ show_docs(options, docs)
+
+ ### Login
+ res = send_command(options, "<aaaLogin inName=\"" + options["-l"] + "\" inPassword=\"" + options["-p"] + "\" />")
+ result = re_cookie.search(res)
+ if (result == None):
+ ## Cookie is absenting in response
+ fail(EC_LOGIN_DENIED)
+
+ options["cookie"] = result.group(1);
+
+ ##
+ ## Fence operations
+ ####
+ result = fence_action(None, options, set_power_status, get_power_status, get_list)
+
+ ### Logout; we do not care about result as we will end in any case
+ send_command(options, "<aaaLogout inCookie=\"" + options["cookie"] + "\" />")
+
+ sys.exit(result)
+
+if __name__ == "__main__":
+ main()
13 years, 7 months
cluster: RHEL56 - fence_rhevm: New fence agent for RHEV-M REST API
by Marek Grác
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 20433b6cab6465b2cdaefb3d383d12da68cab309
Parent: 1d2a31568a1b502b65bb33358424539911e55d4d
Author: Marek 'marx' Grac <mgrac(a)redhat.com>
AuthorDate: Sat Sep 25 00:25:35 2010 +0200
Committer: Marek 'marx' Grac <mgrac(a)redhat.com>
CommitterDate: Sat Sep 25 00:39:30 2010 +0200
fence_rhevm: New fence agent for RHEV-M REST API
Resolves: rhbz#595385
---
fence/agents/lib/fencing.py.py | 8 ++-
fence/agents/rhevm/Makefile | 38 ++++++++++
fence/agents/rhevm/fence_rhevm.py | 144 +++++++++++++++++++++++++++++++++++++
3 files changed, 189 insertions(+), 1 deletions(-)
diff --git a/fence/agents/lib/fencing.py.py b/fence/agents/lib/fencing.py.py
index 43eafe2..be2a44e 100644
--- a/fence/agents/lib/fencing.py.py
+++ b/fence/agents/lib/fencing.py.py
@@ -79,6 +79,10 @@ all_opt = {
"getopt" : "",
"help" : "",
"order" : 1 },
+ "web" : {
+ "getopt" : "",
+ "help" : "",
+ "order" : 1 },
"action" : {
"getopt" : "o:",
"longopt" : "action",
@@ -346,7 +350,7 @@ all_opt = {
"help" : "--shell-timeout <seconds> Wait X seconds for cmd prompt after issuing command",
"default" : "3",
"required" : "0",
- "shortdesc" : "Wait X seconds for cmd promprt after issuing command",
+ "shortdesc" : "Wait X seconds for cmd prompt after issuing command",
"order" : 200 },
"power_timeout" : {
"getopt" : "g:",
@@ -697,6 +701,8 @@ def check_input(device_opt, opt):
options["-u"] = 22
elif options.has_key("-z"):
options["-u"] = 443
+ elif device_opt.count("web"):
+ options["-u"] = 80
else:
options["-u"] = 23
diff --git a/fence/agents/rhevm/Makefile b/fence/agents/rhevm/Makefile
new file mode 100644
index 0000000..4315a91
--- /dev/null
+++ b/fence/agents/rhevm/Makefile
@@ -0,0 +1,38 @@
+###############################################################################
+###############################################################################
+##
+## Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
+## Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+##
+## This copyrighted material is made available to anyone wishing to use,
+## modify, copy, or redistribute it subject to the terms and conditions
+## of the GNU General Public License v.2.
+##
+###############################################################################
+###############################################################################
+
+SOURCE= fence_rhevm.py
+TARGET= fence_rhevm
+
+top_srcdir=../..
+include ${top_srcdir}/make/defines.mk
+
+all: $(TARGET)
+
+fence_rhevm: $(SOURCE)
+ : > $(TARGET)
+ awk "{print}(\$$1 ~ /#BEGIN_VERSION_GENERATION/){exit 0}" $(SOURCE) >> $(TARGET)
+ echo "FENCE_RELEASE_NAME=\"${RELEASE}\";" >> $(TARGET)
+ ${top_srcdir}/scripts/define2var ${top_srcdir}/config/copyright.cf sh REDHAT_COPYRIGHT >> $(TARGET)
+ echo "BUILD_DATE=\"(built `date`)\";" >> $(TARGET)
+ awk -v p=0 "(\$$1 ~ /#END_VERSION_GENERATION/){p = 1} {if(p==1)print}" $(SOURCE) >> $(TARGET)
+ chmod +x $(TARGET)
+
+install: all
+ if [ ! -d ${sbindir} ]; then \
+ install -d ${sbindir}; \
+ fi
+ install -m755 ${TARGET} ${sbindir}
+
+clean:
+ rm -f $(TARGET)
diff --git a/fence/agents/rhevm/fence_rhevm.py b/fence/agents/rhevm/fence_rhevm.py
new file mode 100755
index 0000000..92272b9
--- /dev/null
+++ b/fence/agents/rhevm/fence_rhevm.py
@@ -0,0 +1,144 @@
+#!/usr/bin/python
+
+import sys, re, pexpect, socket
+import pycurl, StringIO
+sys.path.append("/usr/lib/fence")
+from fencing import *
+
+#BEGIN_VERSION_GENERATION
+FENCE_RELEASE_NAME=""
+REDHAT_COPYRIGHT=""
+BUILD_DATE=""
+#END_VERSION_GENERATION
+
+re_get_id = re.compile("<vm id=\"(.*?)\"", re.IGNORECASE);
+re_status = re.compile("<status>(.*?)</status>", re.IGNORECASE);
+re_get_name = re.compile("<name>(.*?)</name>", re.IGNORECASE);
+
+def get_power_status(conn, options):
+ ### Obtain real ID from name
+ try:
+ res = send_command(options, "vms/?search=name%3D" + options["-n"])
+ except pycurl.error, e:
+ sys.stderr.write(e[1] + "\n")
+ fail(EC_TIMED_OUT)
+
+ result = re_get_id.search(res)
+ if (result == None):
+ # Unable to obtain ID needed to access virtual machine
+ fail(EC_STATUS)
+
+ options["id"] = result.group(1);
+
+ re_status.search(res)
+ result = re_status.search(res)
+ if (result == None):
+ # We were able to parse ID so output is correct
+ # in some cases it is possible that RHEV-M output does not
+ # contain <status> line. We can assume machine is OFF then
+ return "off"
+ else:
+ status = result.group(1)
+
+ if (status == "RUNNING"):
+ return "on"
+ else:
+ return "off"
+
+def set_power_status(conn, options):
+ action = {
+ 'on' : "start",
+ 'off' : "stop"
+ }[options["-o"]]
+
+ url = "vms/" + options["id"] + "/" + action
+ try:
+ res = send_command(options, url, "POST")
+ except pycurl.error, e:
+ sys.stderr.write(e[1] + "\n")
+ fail(EC_TIMED_OUT)
+
+ return
+
+def get_list(conn, options):
+ outlets = { }
+
+ try:
+ try:
+ res = send_command(options, "vms")
+ except pycurl.error, e:
+ sys.stderr.write(e[1] + "\n")
+ fail(EC_TIMED_OUT)
+
+ lines = res.split("<vm ")
+ for i in range(1, len(lines)):
+ name = re_get_name.search(lines[i]).group(1)
+ outlets[name] = ("", None)
+ except AttributeError:
+ return { }
+ except IndexError:
+ return { }
+
+ return outlets
+
+def send_command(opt, command, method = "GET"):
+ ## setup correct URL
+ if opt.has_key("-z"):
+ url = "https:"
+ else:
+ url = "http:"
+
+ url += "//" + opt["-a"] + ":" + str(opt["-u"]) + "/rhevm-api-powershell/" + command
+
+ ## send command through pycurl
+ c = pycurl.Curl()
+ b = StringIO.StringIO()
+ c.setopt(pycurl.URL, url)
+ c.setopt(pycurl.HTTPHEADER, [ "Content-type: application/xml", "Accept: application/xml" ])
+ c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
+ c.setopt(pycurl.USERPWD, opt["-l"] + ":" + opt["-p"])
+ c.setopt(pycurl.TIMEOUT, int(opt["-Y"]))
+ c.setopt(pycurl.SSL_VERIFYPEER, 0)
+
+ if (method == "POST"):
+ c.setopt(pycurl.POSTFIELDS, "<action />")
+
+ c.setopt(pycurl.WRITEFUNCTION, b.write)
+ c.perform()
+ result = b.getvalue()
+
+ if opt["log"] >= LOG_MODE_VERBOSE:
+ opt["debug_fh"].write(command + "\n")
+ opt["debug_fh"].write(result + "\n")
+
+ return result
+
+def main():
+ device_opt = [ "help", "version", "agent", "quiet", "verbose", "debug",
+ "action", "ipaddr", "login", "passwd", "passwd_script",
+ "ssl", "inet4_only", "inet6_only", "ipport", "port",
+ "web", "separator", "power_wait", "power_timeout",
+ "shell_timeout" ]
+
+ atexit.register(atexit_handler)
+
+ all_opt["power_wait"]["default"] = "1"
+
+ options = check_input(device_opt, process_input(device_opt))
+
+ docs = { }
+ docs["shortdesc"] = "Fence agent for RHEV-M REST API"
+ docs["longdesc"] = "fence_rhevm is an I/O Fencing agent which can be \
+used with RHEV-M REST API to fence virtual machines."
+ docs["vendorurl"] = "http://www.redhat.com"
+ show_docs(options, docs)
+
+ ##
+ ## Fence operations
+ ####
+ result = fence_action(None, options, set_power_status, get_power_status, get_list)
+
+ sys.exit(result)
+
+if __name__ == "__main__":
+ main()
13 years, 7 months
cluster: RHEL56 - resource-agents: Support ext4 in fs.sh
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 1d2a31568a1b502b65bb33358424539911e55d4d
Parent: f2bfc93101e06cba918c2bb0c11ab6d668788019
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Wed Sep 22 10:10:58 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 18:26:26 2010 -0400
resource-agents: Support ext4 in fs.sh
Resolves: rhbz#636550
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/resources/fs.sh | 8 +++++---
1 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/rgmanager/src/resources/fs.sh b/rgmanager/src/resources/fs.sh
index a2148f4..ce3e983 100755
--- a/rgmanager/src/resources/fs.sh
+++ b/rgmanager/src/resources/fs.sh
@@ -384,7 +384,7 @@ verify_fstype()
[ -z "$OCF_RESKEY_fstype" ] && return 0
case $OCF_RESKEY_fstype in
- ext2|ext3|jfs|xfs|reiserfs|vfat|tmpfs|vxfs)
+ ext2|ext3|ext4|jfs|xfs|reiserfs|vfat|tmpfs|vxfs)
return 0
;;
*)
@@ -417,7 +417,7 @@ verify_options()
esac
case $OCF_RESKEY_fstype in
- ext2|ext3)
+ ext2|ext3|ext4)
case $o in
bsddf|minixdf|check|check=*|nocheck|debug)
continue
@@ -433,7 +433,8 @@ verify_options()
;;
esac
- if [ "$OCF_RESKEY_fstype" = "ext3" ]; then
+ if [ "$OCF_RESKEY_fstype" = "ext3" ] ||
+ [ "$OCF_RESKEY_fstype" = "ext4" ] ; then
case $0 in
noload|data=*)
continue
@@ -908,6 +909,7 @@ Cannot mount $dev on $mp, the device or mount point is already in use!"
case "$fstype" in
reiserfs) typeset fsck_needed="" ;;
ext3) typeset fsck_needed="" ;;
+ ext4) typeset fsck_needed="" ;;
jfs) typeset fsck_needed="" ;;
xfs) typeset fsck_needed="" ;;
vxfs) typeset fsck_needed="" ;;
13 years, 7 months
cluster: RHEL56 - cman: Make qdiskd heuristics time out
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: f2bfc93101e06cba918c2bb0c11ab6d668788019
Parent: badfcf9fd75c65285c1a231d989346098ea1c42c
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Tue Sep 21 13:45:20 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 18:25:00 2010 -0400
cman: Make qdiskd heuristics time out
Qdiskd heuristics were previously expected to enforce
their own timeouts. This patch makes qdiskd count
any heuristic which has taken longer than (interval*(tko-1))
as failed, since that heuristic is not being reliable.
A side effect is that now qdiskd will also automatically
calculate interval and tko counts for all heuristics,
obviating the need for administrators to do this manually.
Resolves: rhbz#589266
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
cman/man/qdisk.5 | 8 ++--
cman/qdisk/main.c | 6 +++-
cman/qdisk/score.c | 92 ++++++++++++++++++++++++++++++++++++++++++++--------
cman/qdisk/score.h | 7 +++-
4 files changed, 92 insertions(+), 21 deletions(-)
diff --git a/cman/man/qdisk.5 b/cman/man/qdisk.5
index 0bca22a..100952e 100644
--- a/cman/man/qdisk.5
+++ b/cman/man/qdisk.5
@@ -189,7 +189,7 @@ master will only grant a node membership if:
.in 12
(a) CMAN believes the node to be online, and
-.br
+.bi
(b) that node has made enough consecutive, timely writes
.in 16
to the quorum disk, and
@@ -409,15 +409,15 @@ for heuristics. The default score for each heuristic is 1.
\fIinterval\fP\fB="\fP2\fB"/>\fP
.in 12
This is the frequency (in seconds) at which we poll the heuristic. The
-default interval for every heuristic is 2 seconds.
+default interval is determined by the qdiskd timeout.
.in 0
.in 9
\fItko\fP\fB="\fP1\fB"/>\fP
.in 12
After this many failed attempts to run the heuristic, it is considered DOWN,
-and its score is removed. The default tko for each heuristic is 1, which
-may be inadequate for things such as 'ping'.
+and its score is removed. The default tko for each heuristic is determined
+by the qdiskd timeout.
.in 8
\fB/>\fP
.in 0
diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index 81c6545..153b190 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -1468,7 +1468,11 @@ get_config_data(char *cluster_name, qd_ctx *ctx, struct h_data *h, int maxh,
free(val);
}
- *cfh = configure_heuristics(ccsfd, h, maxh);
+ /* Heuristics need to report in 1 cycle before we need to
+ * report in so we can get their score.
+ */
+ *cfh = configure_heuristics(ccsfd, h, maxh,
+ ctx->qc_interval * (ctx->qc_tko - 1));
if (*cfh) {
if (ctx->qc_flags & RF_MASTER_WINS) {
diff --git a/cman/qdisk/score.c b/cman/qdisk/score.c
index 3f508a2..2e8993d 100644
--- a/cman/qdisk/score.c
+++ b/cman/qdisk/score.c
@@ -96,22 +96,25 @@ restore_signals(void)
Spin off a user-defined heuristic
*/
static int
-fork_heuristic(struct h_data *h)
+fork_heuristic(struct h_data *h, struct timespec *now)
{
int pid;
char *argv[4];
- time_t now;
if (h->childpid) {
errno = EINPROGRESS;
return -1;
}
- now = time(NULL);
- if (now < h->nextrun)
+ if (now->tv_sec < h->nextrun.tv_sec ||
+ now->tv_nsec < h->nextrun.tv_nsec)
return 0;
- h->nextrun = now + h->interval;
+ h->nextrun.tv_sec = now->tv_sec + h->interval;
+ h->nextrun.tv_nsec = now->tv_nsec;
+
+ h->failtime.tv_sec = now->tv_sec + h->maxtime;
+ h->failtime.tv_nsec = now->tv_nsec;
pid = fork();
if (pid < 0)
@@ -181,7 +184,7 @@ total_score(struct h_data *h, int max, int *score, int *maxscore)
Check for response from a user-defined heuristic / script
*/
static int
-check_heuristic(struct h_data *h, int block)
+check_heuristic(struct h_data *h, int block, struct timespec *now)
{
int ret;
int status;
@@ -191,14 +194,40 @@ check_heuristic(struct h_data *h, int block)
return 0;
ret = waitpid(h->childpid, &status, block?0:WNOHANG);
- if (!block && ret == 0)
+ if (!block && ret == 0) {
/* No children exited */
+
+ /* no timeout */
+ if (!h->maxtime)
+ return 0;
+
+ /* If we overran our timeout, the heuristic is dead */
+ if (now->tv_sec > h->failtime.tv_sec ||
+ (now->tv_sec == h->failtime.tv_sec &&
+ now->tv_nsec > h->failtime.tv_nsec)) {
+ h->misses = h->tko;
+ h->failed = ETIMEDOUT;
+ if (h->available) {
+ clulog(LOG_INFO, "Heuristic: '%s' DOWN - "
+ "Exceeded timeout of %d seconds\n",
+ h->program, h->maxtime);
+ h->available = 0;
+ }
+ }
+
return 0;
+ }
h->childpid = 0;
if (ret < 0 && errno == ECHILD)
/* wrong child? */
goto miss;
+
+ /* Timed out previously; this run must be ignored. */
+ if (h->failed) {
+ h->failed = 0;
+ goto miss;
+ }
if (!WIFEXITED(status)) {
ret = 0;
goto miss;
@@ -207,7 +236,7 @@ check_heuristic(struct h_data *h, int block)
ret = 0;
goto miss;
}
-
+
/* Returned 0 and was not killed */
if (!h->available) {
h->available = 1;
@@ -241,10 +270,12 @@ miss:
static int
fork_heuristics(struct h_data *h, int max)
{
+ struct timespec now;
int x;
+ clock_gettime(CLOCK_MONOTONIC, &now);
for (x = 0; x < max; x++)
- fork_heuristic(&h[x]);
+ fork_heuristic(&h[x], &now);
return 0;
}
@@ -255,19 +286,49 @@ fork_heuristics(struct h_data *h, int max)
static int
check_heuristics(struct h_data *h, int max, int block)
{
+ struct timespec now;
int x;
+ clock_gettime(CLOCK_MONOTONIC, &now);
for (x = 0; x < max; x++)
- check_heuristic(&h[x], block);
+ check_heuristic(&h[x], block, &now);
return 0;
}
+/*
+ * absmax should be qdiskd (interval * (tko-1))
+ */
+static void
+auto_heuristic_timing(int *interval, int *tko, int absmax)
+{
+ if (!interval || ! tko)
+ return;
+
+ if (absmax < 3)
+ return;
+
+ if (absmax <= 4) {
+ *interval = 1;
+ } else if (absmax <= 22) {
+ *interval = 2;
+ } else if (absmax <= 39) {
+ *interval = 3;
+ } else if (absmax <= 50) {
+ *interval = 4;
+ } else {
+ *interval = 5;
+ }
+
+ *tko = absmax / (*interval);
+}
+
+
/**
Read configuration data from CCS into the array provided
*/
int
-configure_heuristics(int ccsfd, struct h_data *h, int max)
+configure_heuristics(int ccsfd, struct h_data *h, int max, int maxtime)
{
int x = 0;
char *val;
@@ -280,11 +341,14 @@ configure_heuristics(int ccsfd, struct h_data *h, int max)
h[x].program = NULL;
h[x].available = 0;
h[x].misses = 0;
- h[x].interval = 2;
- h[x].tko = 1;
+ auto_heuristic_timing(&h[x].interval, &h[x].tko, maxtime);
+ h[x].maxtime = maxtime;
h[x].score = 1;
h[x].childpid = 0;
- h[x].nextrun = 0;
+ h[x].nextrun.tv_sec = 0;
+ h[x].nextrun.tv_nsec = 0;
+ h[x].failtime.tv_sec = 0;
+ h[x].failtime.tv_nsec = 0;
/* Get program */
snprintf(query, sizeof(query),
diff --git a/cman/qdisk/score.h b/cman/qdisk/score.h
index 95b53d6..9db35a4 100644
--- a/cman/qdisk/score.h
+++ b/cman/qdisk/score.h
@@ -30,19 +30,22 @@
struct h_data {
char * program;
+ struct timespec nextrun;
+ struct timespec failtime;
int score;
int available;
int tko;
int interval;
+ int maxtime;
int misses;
+ int failed;
pid_t childpid;
- time_t nextrun;
};
/*
Grab score data from CCSD
*/
-int configure_heuristics(int ccsfd, struct h_data *hp, int max);
+int configure_heuristics(int ccsfd, struct h_data *hp, int max, int maxtime);
/*
Stop the thread which runs the scoring applets.
13 years, 7 months
cluster: STABLE3 - cman: Make qdiskd heuristics time out
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: c51005e99964c0020511cb953b7eeacb8bc1bbdf
Parent: 6dc5ed853217fa44014d29f309b2bc90a2fa3f19
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Tue Sep 21 13:45:20 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 18:22:16 2010 -0400
cman: Make qdiskd heuristics time out
Qdiskd heuristics were previously expected to enforce
their own timeouts. This patch makes qdiskd count
any heuristic which has taken longer than (interval*(tko-1))
as failed, since that heuristic is not being reliable.
A side effect is that now qdiskd will also automatically
calculate interval and tko counts for all heuristics,
obviating the need for administrators to do this manually.
Resolves: rhbz#636243
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
cman/man/qdisk.5 | 8 ++--
cman/qdisk/main.c | 6 +++-
cman/qdisk/score.c | 92 ++++++++++++++++++++++++++++++++++++++++++++--------
cman/qdisk/score.h | 7 +++-
4 files changed, 92 insertions(+), 21 deletions(-)
diff --git a/cman/man/qdisk.5 b/cman/man/qdisk.5
index efa3638..4070f48 100644
--- a/cman/man/qdisk.5
+++ b/cman/man/qdisk.5
@@ -189,7 +189,7 @@ master will only grant a node membership if:
.in 12
(a) CMAN believes the node to be online, and
-.br
+.bi
(b) that node has made enough consecutive, timely writes
.in 16
to the quorum disk, and
@@ -448,15 +448,15 @@ for heuristics. The default score for each heuristic is 1.
\fIinterval\fP\fB="\fP2\fB"\fP
.in 12
This is the frequency (in seconds) at which we poll the heuristic. The
-default interval for every heuristic is 2 seconds.
+default interval is determined by the qdiskd timeout.
.in 0
.in 9
\fItko\fP\fB="\fP1\fB"\fP
.in 12
After this many failed attempts to run the heuristic, it is considered DOWN,
-and its score is removed. The default tko for each heuristic is 1, which
-may be inadequate for things such as 'ping'.
+and its score is removed. The default tko for each heuristic is determined
+by the qdiskd timeout.
.in 8
\fB/>\fP
.in 0
diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index 4bb2b16..6a9b821 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -1843,7 +1843,11 @@ get_config_data(qd_ctx *ctx, struct h_data *h, int maxh, int *cfh)
goto out;
}
- *cfh = configure_heuristics(ccsfd, h, maxh);
+ /* Heuristics need to report in 1 cycle before we need to
+ * report in so we can get their score.
+ */
+ *cfh = configure_heuristics(ccsfd, h, maxh,
+ ctx->qc_interval * (ctx->qc_tko - 1));
if (*cfh) {
if (ctx->qc_flags & RF_MASTER_WINS) {
diff --git a/cman/qdisk/score.c b/cman/qdisk/score.c
index 81ff700..572464d 100644
--- a/cman/qdisk/score.c
+++ b/cman/qdisk/score.c
@@ -75,22 +75,25 @@ restore_signals(void)
Spin off a user-defined heuristic
*/
static int
-fork_heuristic(struct h_data *h)
+fork_heuristic(struct h_data *h, struct timespec *now)
{
int pid;
char *argv[4];
- time_t now;
if (h->childpid) {
errno = EINPROGRESS;
return -1;
}
- now = time(NULL);
- if (now < h->nextrun)
+ if (now->tv_sec < h->nextrun.tv_sec ||
+ now->tv_nsec < h->nextrun.tv_nsec)
return 0;
- h->nextrun = now + h->interval;
+ h->nextrun.tv_sec = now->tv_sec + h->interval;
+ h->nextrun.tv_nsec = now->tv_nsec;
+
+ h->failtime.tv_sec = now->tv_sec + h->maxtime;
+ h->failtime.tv_nsec = now->tv_nsec;
pid = fork();
if (pid < 0)
@@ -162,7 +165,7 @@ total_score(struct h_data *h, int max, int *score, int *maxscore)
Check for response from a user-defined heuristic / script
*/
static int
-check_heuristic(struct h_data *h, int block)
+check_heuristic(struct h_data *h, int block, struct timespec *now)
{
int ret;
int status;
@@ -172,14 +175,40 @@ check_heuristic(struct h_data *h, int block)
return 0;
ret = waitpid(h->childpid, &status, block?0:WNOHANG);
- if (!block && ret == 0)
+ if (!block && ret == 0) {
/* No children exited */
+
+ /* no timeout */
+ if (!h->maxtime)
+ return 0;
+
+ /* If we overran our timeout, the heuristic is dead */
+ if (now->tv_sec > h->failtime.tv_sec ||
+ (now->tv_sec == h->failtime.tv_sec &&
+ now->tv_nsec > h->failtime.tv_nsec)) {
+ h->misses = h->tko;
+ h->failed = ETIMEDOUT;
+ if (h->available) {
+ logt_print(LOG_INFO, "Heuristic: '%s' DOWN - "
+ "Exceeded timeout of %d seconds\n",
+ h->program, h->maxtime);
+ h->available = 0;
+ }
+ }
+
return 0;
+ }
h->childpid = 0;
if (ret < 0 && errno == ECHILD)
/* wrong child? */
goto miss;
+
+ /* Timed out previously; this run must be ignored. */
+ if (h->failed) {
+ h->failed = 0;
+ goto miss;
+ }
if (!WIFEXITED(status)) {
ret = 0;
goto miss;
@@ -188,7 +217,7 @@ check_heuristic(struct h_data *h, int block)
ret = 0;
goto miss;
}
-
+
/* Returned 0 and was not killed */
if (!h->available) {
h->available = 1;
@@ -222,10 +251,12 @@ miss:
static int
fork_heuristics(struct h_data *h, int max)
{
+ struct timespec now;
int x;
+ clock_gettime(CLOCK_MONOTONIC, &now);
for (x = 0; x < max; x++)
- fork_heuristic(&h[x]);
+ fork_heuristic(&h[x], &now);
return 0;
}
@@ -236,19 +267,49 @@ fork_heuristics(struct h_data *h, int max)
static int
check_heuristics(struct h_data *h, int max, int block)
{
+ struct timespec now;
int x;
+ clock_gettime(CLOCK_MONOTONIC, &now);
for (x = 0; x < max; x++)
- check_heuristic(&h[x], block);
+ check_heuristic(&h[x], block, &now);
return 0;
}
+/*
+ * absmax should be qdiskd (interval * (tko-1))
+ */
+static void
+auto_heuristic_timing(int *interval, int *tko, int absmax)
+{
+ if (!interval || ! tko)
+ return;
+
+ if (absmax < 3)
+ return;
+
+ if (absmax <= 4) {
+ *interval = 1;
+ } else if (absmax <= 22) {
+ *interval = 2;
+ } else if (absmax <= 39) {
+ *interval = 3;
+ } else if (absmax <= 50) {
+ *interval = 4;
+ } else {
+ *interval = 5;
+ }
+
+ *tko = absmax / (*interval);
+}
+
+
/**
Read configuration data from CCS into the array provided
*/
int
-configure_heuristics(int ccsfd, struct h_data *h, int max)
+configure_heuristics(int ccsfd, struct h_data *h, int max, int maxtime)
{
int x = 0;
char *val;
@@ -261,11 +322,14 @@ configure_heuristics(int ccsfd, struct h_data *h, int max)
h[x].program = NULL;
h[x].available = 0;
h[x].misses = 0;
- h[x].interval = 2;
- h[x].tko = 1;
+ auto_heuristic_timing(&h[x].interval, &h[x].tko, maxtime);
+ h[x].maxtime = maxtime;
h[x].score = 1;
h[x].childpid = 0;
- h[x].nextrun = 0;
+ h[x].nextrun.tv_sec = 0;
+ h[x].nextrun.tv_nsec = 0;
+ h[x].failtime.tv_sec = 0;
+ h[x].failtime.tv_nsec = 0;
/* Get program */
snprintf(query, sizeof(query),
diff --git a/cman/qdisk/score.h b/cman/qdisk/score.h
index 77e155b..beff31b 100644
--- a/cman/qdisk/score.h
+++ b/cman/qdisk/score.h
@@ -10,19 +10,22 @@
struct h_data {
char * program;
+ struct timespec nextrun;
+ struct timespec failtime;
int score;
int available;
int tko;
int interval;
+ int maxtime;
int misses;
+ int failed;
pid_t childpid;
- time_t nextrun;
};
/*
Grab score data from CCSD
*/
-int configure_heuristics(int ccsfd, struct h_data *hp, int max);
+int configure_heuristics(int ccsfd, struct h_data *hp, int max, int maxtime);
/*
Start the thread which runs the scoring applets
13 years, 7 months
cluster: RHEL56 - rgmanager: Fix corner case in critical/non-critical handling
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: badfcf9fd75c65285c1a231d989346098ea1c42c
Parent: 4c2014fc4ecd96357881019848a77d25cbe1121b
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Fri Sep 24 16:45:29 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 17:15:23 2010 -0400
rgmanager: Fix corner case in critical/non-critical handling
If a subtree had failed and you removed
__independent_subtree from the configuration for that
subtree as part of a reconfiguration operation, the
subtree would fail to stop, causing the service to enter
the failed state.
Resolves: rhbz#605733
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/restree.c | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index eca28f3..951dbda 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1636,8 +1636,8 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
assert(node->rn_resource->r_incarnations >= 0);
if (node->rn_resource->r_incarnations > 0)
--node->rn_resource->r_incarnations;
- } else if (node->rn_state != RES_DISABLED &&
- !(node->rn_flags & RF_QUIESCE)) {
+ } else if (rv != 0 && (node->rn_state != RES_DISABLED &&
+ !(node->rn_flags & RF_QUIESCE))) {
node->rn_state = RES_FAILED;
pthread_mutex_unlock(&node->rn_resource->r_mutex);
return SFL_FAILURE;
13 years, 7 months
cluster: RHEL56 - rgmanager: clean up independent subtree restart handling
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 4c2014fc4ecd96357881019848a77d25cbe1121b
Parent: b05534b22a95d15be95035448a20f6fa321a9a10
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Tue Sep 14 10:10:05 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 16:16:03 2010 -0400
rgmanager: clean up independent subtree restart handling
Resolves: rhbz#605733
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/restree.c | 28 ++++++++++++++++------------
rgmanager/src/daemons/rg_state.c | 11 +++++++----
2 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index 60743a9..eca28f3 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -549,9 +549,6 @@ assign_restart_policy(resource_t *curres, resource_node_t *parent,
if (max_restarts <= 0)
max_restarts = 0;
free(val);
-
- if (!max_restarts)
- return;
}
snprintf(tok, sizeof(tok), "%s/@__restart_expire_time", base);
@@ -561,11 +558,13 @@ assign_restart_policy(resource_t *curres, resource_node_t *parent,
if (conf_get(tok, &val) == 0) {
#endif
restart_expire_time = (time_t)expand_time(val);
- if ((int64_t)restart_expire_time < 0)
+ if ((int64_t)restart_expire_time <= 0)
restart_expire_time = 0;
free(val);
}
+ if (restart_expire_time == 0 || max_restarts == 0)
+ return;
goto out_assign;
}
@@ -1551,7 +1550,8 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
RF_NEEDSTOP | RF_QUIESCE, 0);
restart_clear(node->rn_restart_counter);
return SFL_RECOVERABLE|SFL_PARTIAL;
- } else if (!rte) {
+ } else if (!rte ||
+ !node->rn_restart_counter) {
restart_add(node->rn_restart_counter);
mark_nodes(node, RES_FAILED,
RF_NEEDSTART | RF_NEEDSTOP, 0);
@@ -1592,7 +1592,7 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
mark_nodes(node, RES_FAILED, RF_NEEDSTOP | RF_QUIESCE, 0);
rv |= SFL_PARTIAL;
- } else if (!rte) {
+ } else if (!rte || !node->rn_restart_counter) {
restart_add(node->rn_restart_counter);
mark_nodes(node, RES_FAILED,
RF_NEEDSTOP | RF_NEEDSTART, 0);
@@ -1613,18 +1613,21 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
node->rn_flags &= ~RF_NEEDSTOP;
rv |= res_exec(node, op, NULL, 0);
- if (node->rn_flags & RF_QUIESCE) {
+ if (node->rn_flags & (RF_NEEDSTART|RF_QUIESCE) &&
+ node->rn_flags & RF_INDEPENDENT &&
+ node->rn_flags & RF_NON_CRITICAL) {
/* Non-critical resources = do not fail
* service if the resource fails to stop
*/
if (rv & SFL_FAILURE) {
- clulog(LOG_WARNING, "Failure to stop %s:%s"
+ clulog(LOG_WARNING, "Failed to stop subtree %s:%s"
" during non-critical recovery "
"operation\n",
node->rn_resource->r_rule->rr_type,
primary_attr_value(
node->rn_resource));
- rv &= ~SFL_FAILURE;
+ rv = SFL_PARTIAL;
+ node->rn_flags |= RF_QUIESCE;
}
}
@@ -1633,7 +1636,8 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
assert(node->rn_resource->r_incarnations >= 0);
if (node->rn_resource->r_incarnations > 0)
--node->rn_resource->r_incarnations;
- } else if (node->rn_state != RES_DISABLED) {
+ } else if (node->rn_state != RES_DISABLED &&
+ !(node->rn_flags & RF_QUIESCE)) {
node->rn_state = RES_FAILED;
pthread_mutex_unlock(&node->rn_resource->r_mutex);
return SFL_FAILURE;
@@ -1641,8 +1645,8 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
pthread_mutex_unlock(&node->rn_resource->r_mutex);
if (node->rn_flags & RF_QUIESCE) {
- node->rn_flags &= ~RF_QUIESCE;
- node->rn_state = RES_DISABLED;
+ mark_nodes(node, RES_DISABLED, 0,
+ RF_NEEDSTART|RF_QUIESCE);
} else {
node->rn_state = RES_STOPPED;
}
diff --git a/rgmanager/src/daemons/rg_state.c b/rgmanager/src/daemons/rg_state.c
index 8af6c7e..de44bf7 100644
--- a/rgmanager/src/daemons/rg_state.c
+++ b/rgmanager/src/daemons/rg_state.c
@@ -1308,17 +1308,20 @@ handle_started_status(char *svcName, int ret,
"Attempting inline recovery\n", svcName);
ret2 = group_op(svcName, RG_CONDSTOP);
- if (!(ret2 & SFL_FAILURE)) {
- ret2 = group_op(svcName, RG_CONDSTART);
+ if (ret2 & SFL_PARTIAL) {
+ ret |= SFL_PARTIAL;
+ ret2 &= ~SFL_PARTIAL;
}
- if (ret2) {
+ if (!(ret2 & SFL_FAILURE)) {
+ ret2 = group_op(svcName, RG_CONDSTART);
+ } else {
clulog(LOG_WARNING, "Inline recovery of %s failed\n",
svcName);
return ret;
}
- clulog(LOG_NOTICE, "Inline recovery of %s succeeded\n",
+ clulog(LOG_NOTICE, "Inline recovery of %s complete\n",
svcName);
if (ret & SFL_PARTIAL) {
clulog(LOG_NOTICE, "Note: Some non-critical "
13 years, 7 months
cluster: RHEL56 - rgmanager: Add independent subtree restart thresholds
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: b05534b22a95d15be95035448a20f6fa321a9a10
Parent: 2a7f8f06f369fe255a5eb5e9cd6b2b2d577ed7c5
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Fri Sep 10 14:45:15 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 16:16:03 2010 -0400
rgmanager: Add independent subtree restart thresholds
__max_restarts => number of restarts to tolerate
__restart_expire_time => in the amount of time given here
__independent_subtree="1":
- after threshold is exceeded for the subtree, restart the
service
- if no threshold, never restart the service
__independent_subtree="2"
- after threshold is exceeded for the subtree, stop the
subtree
- if no threshold, stop the subtree on first failure
Resolves: rhbz#605733
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/restree.c | 97 ++++++++++++++++++++++++++++++++-------
1 files changed, 80 insertions(+), 17 deletions(-)
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index 0494cdc..60743a9 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -522,20 +522,54 @@ res_exec(resource_node_t *node, int op, const char *arg, int depth)
static inline void
assign_restart_policy(resource_t *curres, resource_node_t *parent,
- resource_node_t *node)
+ resource_node_t *node, int ccsfd, char *base)
{
char *val;
int max_restarts = 0;
time_t restart_expire_time = 0;
+ char tok[1024];
node->rn_restart_counter = NULL;
if (!curres || !node)
return;
- if (parent) /* Non-parents don't get one for now */
+ if (parent &&
+ !(node->rn_flags & RF_INDEPENDENT))
return;
- val = res_attr_value(curres, "max_restarts");
+ if (node->rn_flags & RF_INDEPENDENT) {
+ /* per-resource-node failures / expire times */
+ snprintf(tok, sizeof(tok), "%s/@__max_restarts", base);
+#ifndef NO_CCS
+ if (ccs_get(ccsfd, tok, &val) == 0) {
+#else
+ if (conf_get(tok, &val) == 0) {
+#endif
+ max_restarts = atoi(val);
+ if (max_restarts <= 0)
+ max_restarts = 0;
+ free(val);
+
+ if (!max_restarts)
+ return;
+ }
+
+ snprintf(tok, sizeof(tok), "%s/@__restart_expire_time", base);
+#ifndef NO_CCS
+ if (ccs_get(ccsfd, tok, &val) == 0) {
+#else
+ if (conf_get(tok, &val) == 0) {
+#endif
+ restart_expire_time = (time_t)expand_time(val);
+ if ((int64_t)restart_expire_time < 0)
+ restart_expire_time = 0;
+ free(val);
+ }
+
+ goto out_assign;
+ }
+
+ val = (char *)res_attr_value(curres, "max_restarts");
if (!val)
return;
max_restarts = atoi(val);
@@ -548,6 +582,7 @@ assign_restart_policy(resource_t *curres, resource_node_t *parent,
return;
}
+out_assign:
node->rn_restart_counter = restart_init(restart_expire_time,
max_restarts);
}
@@ -637,20 +672,25 @@ do_load_resource(int ccsfd, char *base,
node->rn_state = RES_STOPPED;
node->rn_flags = 0;
node->rn_actions = (resource_act_t *)act_dup(curres->r_actions);
- assign_restart_policy(curres, parent, node);
- snprintf(tok, sizeof(tok), "%s/@__independent_subtree", base);
+
+ if (parent) {
+ /* Independent subtree / non-critical for top-level is
+ * not useful and can interfere with restart thresholds for
+ * non critical resources */
+ snprintf(tok, sizeof(tok), "%s/@__independent_subtree", base);
#ifndef NO_CCS
- if (ccs_get(ccsfd, tok, &ref) == 0) {
+ if (ccs_get(ccsfd, tok, &ref) == 0) {
#else
- if (conf_get(tok, &ref) == 0) {
+ if (conf_get(tok, &ref) == 0) {
#endif
- if (atoi(ref) == 1 || strcasecmp(ref, "yes") == 0)
- node->rn_flags |= RF_INDEPENDENT;
- if (atoi(ref) == 2 || strcasecmp(ref, "non-critical") == 0) {
- curres->r_flags |= RF_NON_CRITICAL;
+ if (atoi(ref) == 1 || strcasecmp(ref, "yes") == 0)
+ node->rn_flags |= RF_INDEPENDENT;
+ if (atoi(ref) == 2 || strcasecmp(ref, "non-critical") == 0) {
+ curres->r_flags |= RF_NON_CRITICAL;
+ }
+ free(ref);
}
- free(ref);
}
snprintf(tok, sizeof(tok), "%s/@__enforce_timeouts", base);
@@ -708,8 +748,11 @@ do_load_resource(int ccsfd, char *base,
* resource is non-critical
*/
node->rn_flags |= RF_NON_CRITICAL | RF_INDEPENDENT;
+
}
+ assign_restart_policy(curres, parent, node, ccsfd, base);
+
*newnode = node;
list_insert(tree, node);
@@ -1377,7 +1420,7 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
char *type, void *__attribute__((unused))ret, int realop,
resource_node_t *node)
{
- int rv = 0, me, op;
+ int rv = 0, me, op, rte = 0;
/* Restore default operation. */
op = realop;
@@ -1443,6 +1486,10 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
/* Nothing to do - children are also disabled */
return 0;
+ if ((realop == RS_START || realop == RS_CONVALESCE) &&
+ node->rn_flags & RF_INDEPENDENT)
+ restart_clear(node->rn_restart_counter);
+
pthread_mutex_lock(&node->rn_resource->r_mutex);
if (node->rn_flags & RF_RECONFIG &&
@@ -1496,14 +1543,22 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
resources of this node must be restarted,
but siblings of this node are not affected. */
if (node->rn_flags & RF_INDEPENDENT) {
- if (node->rn_flags & RF_NON_CRITICAL) {
+
+ rte = restart_threshold_exceeded(node->rn_restart_counter);
+ if ((node->rn_flags & RF_NON_CRITICAL) && (rte ||
+ !node->rn_restart_counter)) {
mark_nodes(node, RES_FAILED,
RF_NEEDSTOP | RF_QUIESCE, 0);
+ restart_clear(node->rn_restart_counter);
return SFL_RECOVERABLE|SFL_PARTIAL;
- } else {
+ } else if (!rte) {
+ restart_add(node->rn_restart_counter);
mark_nodes(node, RES_FAILED,
RF_NEEDSTART | RF_NEEDSTOP, 0);
return SFL_RECOVERABLE;
+ } else {
+ restart_clear(node->rn_restart_counter);
+ return SFL_FAILURE;
}
}
@@ -1527,15 +1582,23 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
the resource tree. */
if (op == RS_STATUS && (rv & (SFL_FAILURE|SFL_RECOVERABLE)) &&
(node->rn_flags & RF_INDEPENDENT)) {
+
+ rte = restart_threshold_exceeded(node->rn_restart_counter);
+
rv = SFL_RECOVERABLE;
- if (node->rn_flags & RF_NON_CRITICAL) {
+ if ((node->rn_flags & RF_NON_CRITICAL) && (rte ||
+ !node->rn_restart_counter)) {
/* if non-critical, just stop */
mark_nodes(node, RES_FAILED, RF_NEEDSTOP | RF_QUIESCE, 0);
rv |= SFL_PARTIAL;
- } else {
+ } else if (!rte) {
+ restart_add(node->rn_restart_counter);
mark_nodes(node, RES_FAILED,
RF_NEEDSTOP | RF_NEEDSTART, 0);
+ } else {
+ restart_clear(node->rn_restart_counter);
+ rv = SFL_FAILURE;
}
}
}
13 years, 7 months
cluster: RHEL56 - rgmanager: Add -c option to man page & clusvcadm -h
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 2a7f8f06f369fe255a5eb5e9cd6b2b2d577ed7c5
Parent: 390e19b31e8783b4c80b6e044ec06978bdb20c1e
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Fri Sep 10 11:49:10 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 16:16:03 2010 -0400
rgmanager: Add -c option to man page & clusvcadm -h
Resolves: rhbz#605733
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/man/clusvcadm.8 | 9 +++++++++
rgmanager/src/utils/clusvcadm.c | 4 ++++
2 files changed, 13 insertions(+), 0 deletions(-)
diff --git a/rgmanager/man/clusvcadm.8 b/rgmanager/man/clusvcadm.8
index d1435ef..d5e09eb 100644
--- a/rgmanager/man/clusvcadm.8
+++ b/rgmanager/man/clusvcadm.8
@@ -33,6 +33,9 @@ clusvcadm \- Cluster User Service Administration Utility
.B [\-s
.I <service>
.B ]
+.B [\-c
+.I <service>
+.B ]
.B [\-v]
.SH "DESCRIPTION"
@@ -117,6 +120,12 @@ Stops the service named
service
until a member transition or until it is enabled again.
+.IP "\-c <service>"
+Attempt to convalesce the named
+.I
+service
+by restarting failed, non-critical components.
+
.IP \-v
Display version information and exit.
diff --git a/rgmanager/src/utils/clusvcadm.c b/rgmanager/src/utils/clusvcadm.c
index 1bc5d9e..c876c82 100644
--- a/rgmanager/src/utils/clusvcadm.c
+++ b/rgmanager/src/utils/clusvcadm.c
@@ -186,6 +186,10 @@ printf(" %s -u Unlock local resource group manager.\n"
name, pad, pad, " ", pad, pad, " ");
printf(" %s -Z <group> Freeze group in place\n", name);
printf(" %s -U <group> Unfreeze/thaw group\n", name);
+printf(" %s -c <group> Convalesce (repair, fix) resource group.\n"
+ " %-*.*s Attempts to start failed, non-critical \n"
+ " %-*.*s resources within a resource group.\n",
+ name, pad, pad, " ", pad, pad, " ");
}
13 years, 7 months
cluster: RHEL56 - rgmanager: Support convalesce w/ central_processing
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 390e19b31e8783b4c80b6e044ec06978bdb20c1e
Parent: 39919bc11779ca18490548199e977c8a93c627be
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Thu Sep 9 10:58:17 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 16:16:03 2010 -0400
rgmanager: Support convalesce w/ central_processing
Resolves: rhbz#605733
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/include/event.h | 1 +
rgmanager/src/daemons/service_op.c | 84 +++++++++++++++++++++++
rgmanager/src/daemons/slang_event.c | 18 +++++-
rgmanager/src/resources/default_event_script.sl | 4 +
4 files changed, 106 insertions(+), 1 deletions(-)
diff --git a/rgmanager/include/event.h b/rgmanager/include/event.h
index 378ee4c..7e628d8 100644
--- a/rgmanager/include/event.h
+++ b/rgmanager/include/event.h
@@ -141,6 +141,7 @@ void set_transition_throttling(int nsecs);
int service_op_start(char *svcName, int *target_list, int target_list_len,
int *new_owner);
int service_op_stop(char *svcName, int do_disable, int event_type);
+int service_op_convalesce(char *svcName);
#endif
diff --git a/rgmanager/src/daemons/service_op.c b/rgmanager/src/daemons/service_op.c
index 8e1cc91..6b0db24 100644
--- a/rgmanager/src/daemons/service_op.c
+++ b/rgmanager/src/daemons/service_op.c
@@ -206,6 +206,90 @@ service_op_stop(char *svcName, int do_disable, int event_type)
}
+int
+service_op_convalesce(char *svcName)
+{
+ SmMessageSt msg;
+ int msg_ret;
+ msgctx_t ctx;
+ rg_state_t svcStatus;
+ int msgtarget = my_id();
+
+ /* Build the message header */
+ msg.sm_hdr.gh_magic = GENERIC_HDR_MAGIC;
+ msg.sm_hdr.gh_command = RG_ACTION_REQUEST;
+ msg.sm_hdr.gh_arg1 = RG_ACTION_MASTER;
+ msg.sm_hdr.gh_length = sizeof (SmMessageSt);
+
+ msg.sm_data.d_action = RG_CONVALESCE;
+
+ strncpy(msg.sm_data.d_svcName, svcName,
+ sizeof(msg.sm_data.d_svcName));
+ msg.sm_data.d_ret = 0;
+ msg.sm_data.d_svcOwner = 0;
+
+ /* Open a connection to the local node - it will decide what to
+ do in this case. XXX inefficient; should queue requests
+ locally and immediately forward requests otherwise */
+
+ if (get_service_state_internal(svcName, &svcStatus) < 0)
+ return RG_EFAIL;
+ if (svcStatus.rs_owner > 0) {
+ if (member_online(svcStatus.rs_owner)) {
+ msgtarget = svcStatus.rs_owner;
+ } else {
+ /* If the owner is not online,
+ mark the service as 'stopped' but
+ otherwise, do nothing.
+ */
+ return svc_stop(svcName, RG_STOP);
+ }
+ }
+
+ if (msg_open(MSG_CLUSTER, msgtarget, RG_PORT, &ctx, 2 * cluster_timeout)< 0) {
+ clulog(LOG_ERR,
+ "#58: Failed opening connection to member #%d\n",
+ my_id());
+ return -1;
+ }
+
+ /* Encode */
+ swab_SmMessageSt(&msg);
+
+ /* Send stop message to the other node */
+ if (msg_send(&ctx, &msg, sizeof (SmMessageSt)) <
+ (int)sizeof (SmMessageSt)) {
+ clulog(LOG_ERR, "Failed to send complete message\n");
+ msg_close(&ctx);
+ return -1;
+ }
+
+ /* Check the response */
+ do {
+ msg_ret = msg_receive(&ctx, &msg,
+ sizeof (SmMessageSt), 10);
+ if ((msg_ret == -1 && errno != ETIMEDOUT) ||
+ (msg_ret > 0)) {
+ break;
+ }
+ } while(1);
+
+ if (msg_ret != sizeof (SmMessageSt)) {
+ clulog(LOG_WARNING, "Strange response size: %d vs %d\n",
+ msg_ret, (int)sizeof(SmMessageSt));
+ return 0; /* XXX really UNKNOWN */
+ }
+
+ /* Got a valid response from other node. */
+ msg_close(&ctx);
+
+ /* Decode */
+ swab_SmMessageSt(&msg);
+
+ return msg.sm_data.d_ret;
+}
+
+
/*
TODO
service_op_migrate()
diff --git a/rgmanager/src/daemons/slang_event.c b/rgmanager/src/daemons/slang_event.c
index 086ac6b..08974ad 100644
--- a/rgmanager/src/daemons/slang_event.c
+++ b/rgmanager/src/daemons/slang_event.c
@@ -102,7 +102,8 @@ static int
_user_restart = RG_RESTART,
_user_migrate = RG_MIGRATE,
_user_freeze = RG_FREEZE,
- _user_unfreeze = RG_UNFREEZE;
+ _user_unfreeze = RG_UNFREEZE,
+ _user_convalesce = RG_CONVALESCE;
SLang_Intrin_Var_Type rgmanager_vars[] =
@@ -155,6 +156,7 @@ SLang_Intrin_Var_Type rgmanager_vars[] =
MAKE_VARIABLE("USER_MIGRATE", &_user_migrate, SLANG_INT_TYPE, 1),
MAKE_VARIABLE("USER_FREEZE", &_user_freeze, SLANG_INT_TYPE, 1),
MAKE_VARIABLE("USER_UNFREEZE", &_user_unfreeze,SLANG_INT_TYPE, 1),
+ MAKE_VARIABLE("USER_CONVALESCE",&_user_convalesce,SLANG_INT_TYPE, 1),
/* Errors */
MAKE_VARIABLE("rg_error", &_rg_err, SLANG_INT_TYPE, 1),
@@ -469,6 +471,18 @@ sl_service_property(char *svcName, char *prop)
/**
usage:
+ service_convalesce(name);
+ */
+static int
+sl_convalesce_service(const char *svcname)
+{
+ return service_op_convalesce(svcname);
+}
+
+
+/**
+ usage:
+
stop_service(name, disable_flag);
*/
int
@@ -999,6 +1013,8 @@ SLang_Intrin_Fun_Type rgmanager_slang[] =
SLANG_VOID_TYPE),
MAKE_INTRINSIC_S("service_domain_info", sl_domain_info, SLANG_VOID_TYPE),
MAKE_INTRINSIC_0("service_stop", sl_stop_service, SLANG_INT_TYPE),
+ MAKE_INTRINSIC_S((char *)"service_convalesce", sl_convalesce_service,
+ SLANG_INT_TYPE),
MAKE_INTRINSIC_0("service_start", sl_start_service, SLANG_INT_TYPE),
MAKE_INTRINSIC_S("service_status", sl_service_status,
SLANG_VOID_TYPE),
diff --git a/rgmanager/src/resources/default_event_script.sl b/rgmanager/src/resources/default_event_script.sl
index 25347eb..da6bc1f 100644
--- a/rgmanager/src/resources/default_event_script.sl
+++ b/rgmanager/src/resources/default_event_script.sl
@@ -593,6 +593,10 @@ define default_user_event_handler()
ret = service_unfreeze(service_name);
+ } else if (user_request == USER_CONVALESCE) {
+
+ ret = service_convalesce(service_name);
+
}
%
13 years, 7 months