dlm: master - dlm_controld: wait for ringid

David Teigland teigland at fedoraproject.org
Fri Jan 6 20:07:53 UTC 2012


Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=02850b642b64ec70c1bdbb411eb653624c135384
Commit:        02850b642b64ec70c1bdbb411eb653624c135384
Parent:        1498ce20c091845100456fd1f235658f6056d140
Author:        David Teigland <teigland at redhat.com>
AuthorDate:    Fri Jan 6 14:06:31 2012 -0600
Committer:     David Teigland <teigland at redhat.com>
CommitterDate: Fri Jan 6 14:06:31 2012 -0600

dlm_controld: wait for ringid

Ensure we don't process a nodedown confchg before getting
the corresponding ringid cb.

Improve some debugging.

Signed-off-by: David Teigland <teigland at redhat.com>
---
 dlm_controld/cpg.c        |   61 +++++++++++++++++++++++++++++++++++++++++++-
 dlm_controld/dlm_daemon.h |    1 +
 dlm_controld/member.c     |   16 +++++------
 3 files changed, 67 insertions(+), 11 deletions(-)

diff --git a/dlm_controld/cpg.c b/dlm_controld/cpg.c
index 183879e..4f63fb0 100644
--- a/dlm_controld/cpg.c
+++ b/dlm_controld/cpg.c
@@ -564,6 +564,27 @@ static void node_history_start(struct lockspace *ls, int nodeid)
 
 static int check_ringid_done(struct lockspace *ls)
 {
+	/* If we've received a confchg due to a nodedown, but not
+	   the corresponding ringid callback, then we should wait
+	   for the ringid callback.  Once we have both conf and ring
+	   callbacks, we can compare cpg/quorum ringids.
+	   
+	   Otherwise, there's a possible problem if we receive a
+	   confchg before both ringid callback and quorum callback.
+	   Then we'd get through this function by comparing the old,
+	   matching ringids.
+
+	   (We seem to usually get the quorum callback before any cpg
+	   callbacks, in which case we wouldn't need cpg_ringid_wait,
+	   but that's probably not guaranteed.) */
+
+	if (ls->cpg_ringid_wait) {
+		log_group(ls, "check_ringid wait cluster %u cpg %u:%llu",
+			  cluster_ringid_seq, ls->cpg_ringid.nodeid,
+			  (unsigned long long)ls->cpg_ringid.seq);
+		return 0;
+	}
+
 	if (cluster_ringid_seq != (uint32_t)ls->cpg_ringid.seq) {
 		log_group(ls, "check_ringid cluster %u cpg %u:%llu",
 			  cluster_ringid_seq, ls->cpg_ringid.nodeid,
@@ -1434,6 +1455,24 @@ void process_lockspace_changes(void)
 	}
 }
 
+static const char *reason_str(int reason)
+{
+	switch (reason) {
+	case CPG_REASON_JOIN:
+		return "join";
+	case CPG_REASON_LEAVE:
+		return "leave";
+	case CPG_REASON_NODEDOWN:
+		return "nodedown";
+	case CPG_REASON_NODEUP:
+		return "nodeup";
+	case CPG_REASON_PROCDOWN:
+		return "procdown";
+	default:
+		return "unknown";
+	};
+}
+
 static int add_change(struct lockspace *ls,
 		      const struct cpg_address *member_list,
 		      size_t member_list_entries,
@@ -1486,6 +1525,9 @@ static int add_change(struct lockspace *ls,
 		}
 		list_add_tail(&memb->list, &cg->removed);
 
+		if (left_list[i].reason == CPG_REASON_NODEDOWN)
+			ls->cpg_ringid_wait = 1;
+
 		if (memb->failed) {
 			node_history_lockspace_fail(ls, memb->nodeid, cg,
 						    left_list[i].reason, now);
@@ -1493,8 +1535,8 @@ static int add_change(struct lockspace *ls,
 			node_history_lockspace_left(ls, memb->nodeid, cg, now);
 		}
 
-		log_group(ls, "add_change cg %u remove nodeid %d reason %d",
-			  cg->seq, memb->nodeid, left_list[i].reason);
+		log_group(ls, "add_change cg %u remove nodeid %d reason %s",
+			  cg->seq, memb->nodeid, reason_str(left_list[i].reason));
 
 		if (left_list[i].reason == CPG_REASON_PROCDOWN)
 			kick_node_from_cluster(memb->nodeid);
@@ -1829,6 +1871,7 @@ static void totem_cb(cpg_handle_t handle,
 
 	ls->cpg_ringid.nodeid = ring_id.nodeid;
 	ls->cpg_ringid.seq = ring_id.seq;
+	ls->cpg_ringid_wait = 0;
 
 	apply_changes(ls);
 }
@@ -2450,12 +2493,26 @@ static void confchg_cb_daemon(cpg_handle_t handle,
 {
 	struct node_daemon *node;
 	uint64_t now = monotime();
+	int nodedown = 0, procdown = 0, leave = 0;
 	int i;
 
 	log_config(group_name, member_list, member_list_entries,
 		   left_list, left_list_entries,
 		   joined_list, joined_list_entries);
 
+	for (i = 0; i < left_list_entries; i++) {
+		if (left_list[i].reason == CPG_REASON_NODEDOWN)
+			nodedown++;
+		else if (left_list[i].reason == CPG_REASON_PROCDOWN)
+			procdown++;
+		else if (left_list[i].reason == CPG_REASON_LEAVE)
+			leave++;
+	}
+
+	if (nodedown || procdown || leave)
+		log_debug("%s left nodedown %d procdown %d leave %d",
+			  group_name->value, nodedown, procdown, leave);
+
 	if (joined_list_entries)
 		send_protocol(&our_protocol);
 
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 2d27caa..f1ea8ea 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -226,6 +226,7 @@ struct lockspace {
 
 	cpg_handle_t		cpg_handle;
 	struct cpg_ring_id	cpg_ringid;
+	int			cpg_ringid_wait;
 	int			cpg_client;
 	int			cpg_fd;
 	int			joining;
diff --git a/dlm_controld/member.c b/dlm_controld/member.c
index d39e541..58ef70e 100644
--- a/dlm_controld/member.c
+++ b/dlm_controld/member.c
@@ -243,8 +243,7 @@ void kick_node_from_cluster(int nodeid)
 		corosync_cfg_try_shutdown(ch,
 				COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
 	} else {
-		log_error("telling corosync to remove nodeid %d from cluster",
-			  nodeid);
+		log_error("tell corosync to remove nodeid %d from cluster", nodeid);
 		corosync_cfg_kill_node(ch, nodeid, "dlm_controld");
 	}
 }
@@ -253,13 +252,12 @@ static void shutdown_callback(corosync_cfg_handle_t h,
 			      corosync_cfg_shutdown_flags_t flags)
 {
 	if (flags & COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) {
-		if (list_empty(&lockspaces))
-			corosync_cfg_replyto_shutdown(ch,
-					COROSYNC_CFG_SHUTDOWN_FLAG_YES);
-		else {
-			log_debug("no to corosync shutdown");
-			corosync_cfg_replyto_shutdown(ch,
-					COROSYNC_CFG_SHUTDOWN_FLAG_NO);
+		if (list_empty(&lockspaces)) {
+			log_debug("shutdown request yes");
+			corosync_cfg_replyto_shutdown(ch, COROSYNC_CFG_SHUTDOWN_FLAG_YES);
+		} else {
+			log_debug("shutdown request no");
+			corosync_cfg_replyto_shutdown(ch, COROSYNC_CFG_SHUTDOWN_FLAG_NO);
 		}
 	}
 }


More information about the cluster-commits mailing list