cluster: RHEL4 - qdisk: Implement I/O timeout for read

Lon Hohberger lon at fedoraproject.org
Thu Oct 21 22:11:28 UTC 2010


Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=bf3e09fcbcf0b661df5b45febd825569d31ac74d
Commit:        bf3e09fcbcf0b661df5b45febd825569d31ac74d
Parent:        d586efd354c0df2507a293d2fd9f472610928006
Author:        Eduardo Damato <edamato at redhat.com>
AuthorDate:    Tue Sep 29 10:06:26 2009 -0400
Committer:     Lon Hohberger <lhh at redhat.com>
CommitterDate: Thu Oct 21 17:31:11 2010 -0400

qdisk: Implement I/O timeout for read

This patch creates a timer for last successful read and
reboots the system if last successful read was more than
interval*tko ago.

Resolves: rhbz#510611

Part 2/2

Signed-off-by: Eduardo Damato <edamato at redhat.com>
Signed-off-by: Lon Hohberger <lhh at redhat.com>
---
 cman/qdisk/main.c |   39 +++++++++++++++++++++++++++++++--------
 1 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index 8b79916..10ec337 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -138,10 +138,10 @@ check_self(qd_ctx *ctx, status_block_t *sb)
   or has not updated their timestamp recently.  See check_transitions as
   well.
  */
-void
+int
 read_node_blocks(qd_ctx *ctx, node_info_t *ni, int max)
 {
-	int x;
+	int x, errors = 0;
 	status_block_t *sb;
 
 	for (x = 0; x < max; x++) {
@@ -153,6 +153,7 @@ read_node_blocks(qd_ctx *ctx, node_info_t *ni, int max)
 			       sb, sizeof(*sb)) < 0) {
 			clulog(LOG_WARNING,"Error reading node ID block %d\n",
 			       x+1);
+			errors++;
 			continue;
 		}
 		swab_status_block_t(sb);
@@ -188,6 +189,8 @@ read_node_blocks(qd_ctx *ctx, node_info_t *ni, int max)
 		ni[x].ni_seen++;
 		ni[x].ni_last_seen = sb->ps_timestamp;
 	}
+
+	return errors;
 }
 
 
@@ -879,7 +882,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 	int low_id, bid_pending = 0, score, score_max, score_req,
 	    upgrade = 0, count;
 	memb_mask_t mask, master_mask;
-	struct timeval maxtime, oldtime, newtime, diff, sleeptime, interval, lastok;
+	struct timeval maxtime, oldtime, newtime, diff, sleeptime, interval, rd_lastok, wr_lastok;
 
 	ctx->qc_status = S_NONE;
 	
@@ -889,8 +892,11 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 	interval.tv_usec = 0;
 	interval.tv_sec = ctx->qc_interval;
 	
-	lastok.tv_usec = 0;
-	lastok.tv_sec = 0;
+	rd_lastok.tv_usec = 0;
+	rd_lastok.tv_sec = 0;
+	
+	wr_lastok.tv_usec = 0;
+	wr_lastok.tv_sec = 0;
 	
 	get_my_score(&score, &score_max);
 	if (score_max < ctx->qc_scoremin) {
@@ -905,7 +911,8 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 		get_time(&oldtime, (ctx->qc_flags&RF_UPTIME));
 		
 		/* Read everyone else's status */
-		read_node_blocks(ctx, ni, max);
+		if ( read_node_blocks(ctx, ni, max) == 0 ) 
+			get_time(&rd_lastok, ctx->qc_flags&RF_UPTIME);
 
 		/* Check for node transitions */
 		check_transitions(ctx, ni, max, mask);
@@ -1077,7 +1084,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 				    &msg, mask, master_mask) != 0) {
 			clulog(LOG_ERR, "Error writing to quorum disk\n");
  		} else {
- 			get_time(&lastok, ctx->qc_flags&RF_UPTIME);
+ 			get_time(&wr_lastok, ctx->qc_flags&RF_UPTIME);
 		}
 
 		/* write out our local status */
@@ -1090,7 +1097,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
  		/*
 		 * Reboot if the last successful hearbeat was longer ago than interval*TKO_COUNT
 		 */
-		_diff_tv(&diff, &lastok, &newtime);
+		_diff_tv(&diff, &wr_lastok, &newtime);
 		if (_cmp_tv(&maxtime, &diff) == 1 &&
 		    ctx->qc_flags & RF_IOTIMEOUT) {
 			clulog(LOG_EMERG, "Failed to send a heartbeat within "
@@ -1102,6 +1109,22 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 			if (!(ctx->qc_flags & RF_DEBUG)) 
 				reboot(RB_AUTOBOOT);
 		}
+
+ 		/*
+		 * Reboot if the last successful hearbeat was longer ago than interval*TKO_COUNT
+		 */
+		_diff_tv(&diff, &rd_lastok, &newtime);
+		if (_cmp_tv(&maxtime, &diff) == 1 &&
+		    ctx->qc_flags & RF_IOTIMEOUT) {
+			clulog(LOG_EMERG, "Failed to read from qdisk within "
+			       "%d second%s (%d.%06d) - REBOOTING\n",
+			       (int)maxtime.tv_sec,
+			       maxtime.tv_sec==1?"":"s",
+			       (int)diff.tv_sec,
+			       (int)diff.tv_usec);
+			if (!(ctx->qc_flags & RF_DEBUG)) 
+				reboot(RB_AUTOBOOT);
+		}
 	
 		/*
 		 * Reboot if we didn't send a heartbeat in interval*TKO_COUNT


More information about the cluster-commits mailing list