cluster: STABLE3 - qdisk: Autoconfigure default timings

Lon Hohberger lon at fedoraproject.org
Fri Feb 19 17:21:50 UTC 2010


Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=704fd5bb382ff452fc8403b9a2322e73b918619d
Commit:        704fd5bb382ff452fc8403b9a2322e73b918619d
Parent:        e3fdd916da5ad54e37bbc457f336fb482dbe3b14
Author:        Lon Hohberger <lhh at redhat.com>
AuthorDate:    Fri Feb 19 09:45:17 2010 -0500
Committer:     Lon Hohberger <lhh at redhat.com>
CommitterDate: Fri Feb 19 11:56:47 2010 -0500

qdisk: Autoconfigure default timings

Qdiskd has historically had bad default timings in STABLE3
and STABLE2.  This patch makes qdiskd scale automatically
with the Totem token timeout.

Signed-off-by: Lon Hohberger <lhh at redhat.com>
---
 cman/man/qdisk.5  |   30 +++++++++++++++++++++++++++---
 cman/qdisk/disk.h |    2 +-
 cman/qdisk/main.c |   45 ++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/cman/man/qdisk.5 b/cman/man/qdisk.5
index f578e92..1a74470 100644
--- a/cman/man/qdisk.5
+++ b/cman/man/qdisk.5
@@ -80,7 +80,8 @@ capacity.  If CMAN is not running, qdisk will wait for it.
 
 * CMAN's eviction timeout should be at least 2x the quorum daemon's
 to give the quorum daemon adequate time to converge on a master during a
-failure + load spike situation.
+failure + load spike situation.  See section 3.3.1 for specific
+details.
 
 * For 'all-but-one' failure operation, the total number of votes assigned
 to the quorum device should be equal to or greater than the total number
@@ -211,6 +212,7 @@ This is the frequency of read/write cycles, in seconds.
 \fItko\fP\fB="\fP10\fB"\fP
 .in 12
 This is the number of cycles a node must miss in order to be declared dead.
+The default for this number is dependent on the configured token timeout.
 
 .in 9
 \fItko_up\fP\fB="\fPX\fB"\fP
@@ -289,7 +291,7 @@ This option requires careful tuning of the CMAN timeout, the qdiskd
 timeout, and CMAN's quorum_dev_poll value.  As a rule of thumb,
 CMAN's quorum_dev_poll value should be equal to Totem's token timeout
 and qdiskd's timeout (interval*tko) should be less than half of
-Totem's token timeout.
+Totem's token timeout.  See section 3.3.1 for more information.
 
 This option only takes effect if there are no heuristics
 configured.  Usage of this option in configurations with more than
@@ -372,7 +374,7 @@ label.  This is useful in configurations where the block device name
 differs on a per-node basis.
 
 .in 9
-\fIcman_label\fP\fB="\fPmylabel\fB"/>\fP
+\fIcman_label\fP\fB="\fPmylabel\fB"\fP
 .in 12
 This overrides the label advertised to CMAN if present.  If specified,
 the quorum daemon will register with this name instead of the actual
@@ -391,6 +393,28 @@ qdiskd is running.  This option is ignored if io_timeout is set to 1.
 \fB/>\fP
 .in 0
 
+.SH "3.3.1.  Quorum Disk Timings"
+Qdiskd should not be used in environments requiring failure detection
+times of less than approximately 10 seconds.
+
+Qdiskd will attempt to automatically configure timings based on the
+totem timeout and the TKO.  If configuring manually, Totem's token
+timeout \fBmust\fP be set to a value at least 1 interval greater than
+the the following function:
+
+  interval * (tko + master_wait + upgrade_wait)
+
+So, if you have an interval of 2, a tko of 7, master_wait of 2 and
+upgrade_wait of 2, the token timeout should be at least 24 seconds
+(24000 msec).
+
+It is recommended to have at least 3 intervals to reduce the risk of 
+quorum loss during heavy I/O load.  As a rule of thumb, using a totem
+timeout more than 2x of qdiskd's timeout will result in good behavior.
+
+An improper timing configuration will cause CMAN to give up on qdiskd,
+causing a temporary loss of quorum during master transition.
+
 .SH "3.2.  The <heuristic> tag"
 This tag is a child of the <quorumd> tag.  Heuristics may not be changed
 while qdiskd is running.
diff --git a/cman/qdisk/disk.h b/cman/qdisk/disk.h
index c5b3d18..8678ca7 100644
--- a/cman/qdisk/disk.h
+++ b/cman/qdisk/disk.h
@@ -246,7 +246,7 @@ typedef struct {
 	int qc_max_error_cycles;
 	int qc_master;		/* Master?! */
 	int qc_config;
-	int qc_pad;
+	int qc_token_timeout;
 	disk_node_state_t qc_disk_status;
 	disk_node_state_t qc_status;
 	run_flag_t qc_flags;
diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index eb3ab3c..85a0563 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -24,6 +24,7 @@
 #include <ccs.h>
 #include <liblogthread.h>
 #include "score.h"
+#include "../daemon/cman.h"
 #include <sys/syslog.h>
 
 #define LOG_DAEMON_NAME  "qdiskd"
@@ -1472,6 +1473,7 @@ get_static_config_data(qd_ctx *ctx, int ccsfd)
 {
 	char *val = NULL;
 	char query[256];
+	int qdisk_fo;
 
 	if (ccsfd < 0)
 		return -1;
@@ -1486,14 +1488,36 @@ get_static_config_data(qd_ctx *ctx, int ccsfd)
 		if (ctx->qc_interval < 1)
 			ctx->qc_interval = 1;
 	}
+
+	snprintf(query, sizeof(query), "/cluster/totem/@token");
+	if (ccs_get(ccsfd, query, &val) == 0) {
+		ctx->qc_token_timeout = atoi(val);
+		free(val);
+		if (ctx->qc_token_timeout < 10000) {
+			logt_print(LOG_DEBUG, "Token timeout %d is too fast "
+				   "to use with qdiskd!\n",
+				   ctx->qc_token_timeout);
+		} 
+	} else {
+		ctx->qc_token_timeout = DEFAULT_TOKEN_TIMEOUT;
+	}
 		
 	/* Get tko */
 	snprintf(query, sizeof(query), "/cluster/quorumd/@tko");
 	if (ccs_get(ccsfd, query, &val) == 0) {
 		ctx->qc_tko = atoi(val);
 		free(val);
-		if (ctx->qc_tko < 3)
-			ctx->qc_tko = 3;
+	} else {
+		ctx->qc_tko = ((ctx->qc_token_timeout / 1000) -
+			       ctx->qc_interval) / 2;
+		logt_print(LOG_DEBUG, "Auto-configured TKO as %d based on "
+			   "token=%d interval=%d\n", ctx->qc_tko,
+			   ctx->qc_token_timeout, ctx->qc_interval);
+	}
+
+	if (ctx->qc_tko < 4) {
+		logt_print(LOG_ERR, "Quorum disk TKO (%d) is too low!\n",
+			   ctx->qc_tko);
 	}
 
 	/* Get up-tko (transition off->online) */
@@ -1527,7 +1551,22 @@ get_static_config_data(qd_ctx *ctx, int ccsfd)
 	}
 	if (ctx->qc_master_wait <= ctx->qc_tko_up)
 		ctx->qc_master_wait = ctx->qc_tko_up + 1;
-		
+
+	qdisk_fo = ctx->qc_interval * (ctx->qc_master_wait +
+				ctx->qc_upgrade_wait +
+				ctx->qc_tko) * 1000;
+	if (qdisk_fo >= ctx->qc_token_timeout) {
+		logt_print(LOG_WARNING, "Quorum disk timings are too slow for "
+			   "configured token timeout\n");
+		logt_print(LOG_WARNING, " * Totem Token timeout: %dms\n",
+			   ctx->qc_token_timeout);
+		logt_print(LOG_WARNING, " * Min. Master recovery time: %dms\n",
+			   qdisk_fo);
+		logt_print(LOG_WARNING,
+			   "Please set token timeout to at least %dms\n",
+			   qdisk_fo + (ctx->qc_interval * 1000));
+	}
+
 	/* Get votes */
 	snprintf(query, sizeof(query), "/cluster/quorumd/@votes");
 	if (ccs_get(ccsfd, query, &val) == 0) {


More information about the cluster-commits mailing list