dlm: master - dlm_controld: even partition merges

David Teigland teigland at fedoraproject.org
Wed Mar 6 17:57:38 UTC 2013


Gitweb:        http://git.fedorahosted.org/git/?p=dlm.git;a=commitdiff;h=a83408ad8ee91a3f5d0416dcc3b7e8c50c47c7b9
Commit:        a83408ad8ee91a3f5d0416dcc3b7e8c50c47c7b9
Parent:        ee11f54d5448447505239fdff4a68b3922d97f4e
Author:        David Teigland <teigland at redhat.com>
AuthorDate:    Tue Mar 5 13:36:38 2013 -0600
Committer:     David Teigland <teigland at redhat.com>
CommitterDate: Tue Mar 5 13:36:38 2013 -0600

dlm_controld: even partition merges

When a cluster partitions and merges evenly,
e.g. 2/2, both sides were killing/fencing each
other.  In this case, wait for stateful merged
nodes to be cleared before fencing.  The side
with the low nodeid will kill the stateful nodes
(from its perspective).

Signed-off-by: David Teigland <teigland at redhat.com>
---
 dlm_controld/daemon_cpg.c |   82 ++++++++++++++++++++++++++++++++++++++++++++-
 dlm_controld/fence.c      |    2 +-
 2 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/dlm_controld/daemon_cpg.c b/dlm_controld/daemon_cpg.c
index b3c543d..8c4cff2 100644
--- a/dlm_controld/daemon_cpg.c
+++ b/dlm_controld/daemon_cpg.c
@@ -72,6 +72,7 @@ struct node_daemon {
 	int need_fence_clear;
 	int need_fencing;
 	int delay_fencing;
+	int stateful_merge;
 	int fence_pid;
 	int fence_pid_wait;
 	int fence_result_wait;
@@ -657,6 +658,20 @@ static void fence_pid_cancel(int nodeid, int pid)
 		  nodeid, pid, rv, result);
 }
 
+static void kick_stateful_merge_members(void)
+{
+	struct node_daemon *node;
+
+	list_for_each_entry(node, &daemon_nodes, list) {
+		if (!node->killed && node->stateful_merge) {
+			log_error("daemon node %d kill stateful merge member",
+				  node->nodeid);
+			kick_node_from_cluster(node->nodeid);
+			node->killed = 1;
+		}
+	}
+}
+
 /*
  * fence_in_progress_unknown (fipu)
  *
@@ -746,7 +761,8 @@ static void fence_pid_cancel(int nodeid, int pid)
 static void daemon_fence_work(void)
 {
 	struct node_daemon *node, *safe;
-	int rv, nodeid, pid, need, low, actor, result;
+	int gone_count = 0, part_count = 0, merge_count = 0, clean_count = 0;
+	int rv, nodeid, pid, need, low = 0, actor, result;
 	int retry = 0;
 	uint32_t flags;
 
@@ -774,6 +790,67 @@ static void daemon_fence_work(void)
 	}
 
 	/*
+	 * Count different types of nodes
+	 * gone: node not a member
+	 * part: member we've not received a proto message from
+	 * merge: member we received a stateful proto message from
+	 * clean: member we received a clean/new proto message from
+	 *
+	 * A node always views itself as a clean member, not a merge member.
+	 */
+
+	list_for_each_entry(node, &daemon_nodes, list) {
+		if (!node->daemon_member) {
+			gone_count++;
+		} else {
+			if (!low || node->nodeid < low)
+				low = node->nodeid;
+
+			if (node->stateful_merge)
+				merge_count++;
+			else if (!node->proto.daemon_max[0])
+				part_count++;
+			else
+				clean_count++;
+		}
+	}
+
+	/*
+	 * Wait for stateful merged members to be removed before moving
+	 * on to fencing.  Kill stateful merged members to clear them.
+	 * This section is only relevant to non-two-node, even splits.
+	 *
+	 * With two node splits, they race to fence each other and
+	 * whichever fences successfully then kills corosync on the other
+	 * (in the case where corosync is still running on the fenced node).
+	 *
+	 * With an odd split, the partition that maintained quorum will
+	 * kill stateful merged nodes when their proto message is received.
+	 *
+	 * With an even split, e.g. 2/2, we don't want both sets to
+	 * be fencing each other right after merge, when both sides
+	 * have quorum again and see the other side as statefully merged.
+	 * So, delay fencing until the stateful nodes are cleared on one
+	 * side (by way of the low nodeid killing stateful merged members).
+	 *
+	 * When there are 3 or more partitions that merge, none may see
+	 * enough clean nodes, so the cluster would be stuck here waiting
+	 * for someone to manually reset/restart enough nodes to produce
+	 * sufficient clean nodes (>= merged).
+	 */
+
+	if (!cluster_two_node && merge_count) {
+		log_retry(retry_fencing, "fence work wait to clear merge %d clean %d part %d gone %d",
+			  merge_count, clean_count, part_count, gone_count);
+
+		if ((clean_count >= merge_count) && !part_count && (low == our_nodeid))
+			kick_stateful_merge_members();
+
+		retry = 1;
+		goto out;
+	}
+
+	/*
 	 * startup fencing
 	 */
 
@@ -1608,6 +1685,8 @@ static void receive_protocol(struct dlm_header *hd, int len)
 			  (unsigned long long)cluster_quorate_monotime,
 			  node->killed);
 
+		node->stateful_merge = 1;
+
 		if (cluster_quorate && node->daemon_rem_time &&
 		    cluster_quorate_monotime < node->daemon_rem_time) {
 			if (!node->killed) {
@@ -1960,6 +2039,7 @@ static void confchg_cb_daemon(cpg_handle_t handle,
 			node->daemon_member = 0;
 			node->daemon_rem_time = now;
 			node->killed = 0;
+			node->stateful_merge = 0;
 
 			/* If we never accepted a valid proto from this node,
 			   then it never fully joined and there's no need to
diff --git a/dlm_controld/fence.c b/dlm_controld/fence.c
index 0094b3a..e4c67e9 100644
--- a/dlm_controld/fence.c
+++ b/dlm_controld/fence.c
@@ -138,7 +138,7 @@ int fence_result(int nodeid, int pid, int *result)
 	if (rv < 0) {
 		/* shouldn't happen */
 		log_error("fence result %d pid %d waitpid %d errno %d",
-			  pid, nodeid, rv, errno);
+			  nodeid, pid, rv, errno);
 		return rv;
 	}
 


More information about the cluster-commits mailing list