cluster: RHEL4 - qdisk: Implement I/O timeout for read
Lon Hohberger
lon at fedoraproject.org
Thu Oct 21 22:11:28 UTC 2010
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=bf3e09fcbcf0b661df5b45febd825569d31ac74d
Commit: bf3e09fcbcf0b661df5b45febd825569d31ac74d
Parent: d586efd354c0df2507a293d2fd9f472610928006
Author: Eduardo Damato <edamato at redhat.com>
AuthorDate: Tue Sep 29 10:06:26 2009 -0400
Committer: Lon Hohberger <lhh at redhat.com>
CommitterDate: Thu Oct 21 17:31:11 2010 -0400
qdisk: Implement I/O timeout for read
This patch creates a timer for last successful read and
reboots the system if last successful read was more than
interval*tko ago.
Resolves: rhbz#510611
Part 2/2
Signed-off-by: Eduardo Damato <edamato at redhat.com>
Signed-off-by: Lon Hohberger <lhh at redhat.com>
---
cman/qdisk/main.c | 39 +++++++++++++++++++++++++++++++--------
1 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index 8b79916..10ec337 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -138,10 +138,10 @@ check_self(qd_ctx *ctx, status_block_t *sb)
or has not updated their timestamp recently. See check_transitions as
well.
*/
-void
+int
read_node_blocks(qd_ctx *ctx, node_info_t *ni, int max)
{
- int x;
+ int x, errors = 0;
status_block_t *sb;
for (x = 0; x < max; x++) {
@@ -153,6 +153,7 @@ read_node_blocks(qd_ctx *ctx, node_info_t *ni, int max)
sb, sizeof(*sb)) < 0) {
clulog(LOG_WARNING,"Error reading node ID block %d\n",
x+1);
+ errors++;
continue;
}
swab_status_block_t(sb);
@@ -188,6 +189,8 @@ read_node_blocks(qd_ctx *ctx, node_info_t *ni, int max)
ni[x].ni_seen++;
ni[x].ni_last_seen = sb->ps_timestamp;
}
+
+ return errors;
}
@@ -879,7 +882,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
int low_id, bid_pending = 0, score, score_max, score_req,
upgrade = 0, count;
memb_mask_t mask, master_mask;
- struct timeval maxtime, oldtime, newtime, diff, sleeptime, interval, lastok;
+ struct timeval maxtime, oldtime, newtime, diff, sleeptime, interval, rd_lastok, wr_lastok;
ctx->qc_status = S_NONE;
@@ -889,8 +892,11 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
interval.tv_usec = 0;
interval.tv_sec = ctx->qc_interval;
- lastok.tv_usec = 0;
- lastok.tv_sec = 0;
+ rd_lastok.tv_usec = 0;
+ rd_lastok.tv_sec = 0;
+
+ wr_lastok.tv_usec = 0;
+ wr_lastok.tv_sec = 0;
get_my_score(&score, &score_max);
if (score_max < ctx->qc_scoremin) {
@@ -905,7 +911,8 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
get_time(&oldtime, (ctx->qc_flags&RF_UPTIME));
/* Read everyone else's status */
- read_node_blocks(ctx, ni, max);
+ if ( read_node_blocks(ctx, ni, max) == 0 )
+ get_time(&rd_lastok, ctx->qc_flags&RF_UPTIME);
/* Check for node transitions */
check_transitions(ctx, ni, max, mask);
@@ -1077,7 +1084,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
&msg, mask, master_mask) != 0) {
clulog(LOG_ERR, "Error writing to quorum disk\n");
} else {
- get_time(&lastok, ctx->qc_flags&RF_UPTIME);
+ get_time(&wr_lastok, ctx->qc_flags&RF_UPTIME);
}
/* write out our local status */
@@ -1090,7 +1097,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
/*
* Reboot if the last successful hearbeat was longer ago than interval*TKO_COUNT
*/
- _diff_tv(&diff, &lastok, &newtime);
+ _diff_tv(&diff, &wr_lastok, &newtime);
if (_cmp_tv(&maxtime, &diff) == 1 &&
ctx->qc_flags & RF_IOTIMEOUT) {
clulog(LOG_EMERG, "Failed to send a heartbeat within "
@@ -1102,6 +1109,22 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
if (!(ctx->qc_flags & RF_DEBUG))
reboot(RB_AUTOBOOT);
}
+
+ /*
+ * Reboot if the last successful hearbeat was longer ago than interval*TKO_COUNT
+ */
+ _diff_tv(&diff, &rd_lastok, &newtime);
+ if (_cmp_tv(&maxtime, &diff) == 1 &&
+ ctx->qc_flags & RF_IOTIMEOUT) {
+ clulog(LOG_EMERG, "Failed to read from qdisk within "
+ "%d second%s (%d.%06d) - REBOOTING\n",
+ (int)maxtime.tv_sec,
+ maxtime.tv_sec==1?"":"s",
+ (int)diff.tv_sec,
+ (int)diff.tv_usec);
+ if (!(ctx->qc_flags & RF_DEBUG))
+ reboot(RB_AUTOBOOT);
+ }
/*
* Reboot if we didn't send a heartbeat in interval*TKO_COUNT
More information about the cluster-commits
mailing list