gfs2-utils: master - gfs_controld: watch cluster membership

David Teigland teigland at fedoraproject.org
Thu Oct 15 18:59:59 UTC 2009


Gitweb:        http://git.fedorahosted.org/git/gfs2-utils.git?p=gfs2-utils.git;a=commitdiff;h=c87787214e61794d55fb424bf6b35a206c5f410a
Commit:        c87787214e61794d55fb424bf6b35a206c5f410a
Parent:        8428fc6366ab3948099258548a325c7d34f182fb
Author:        David Teigland <teigland at redhat.com>
AuthorDate:    Thu Oct 15 12:56:34 2009 -0500
Committer:     David Teigland <teigland at redhat.com>
CommitterDate: Thu Oct 15 12:56:34 2009 -0500

gfs_controld: watch cluster membership

Keep track of cluster membership changes so we can call the
recently added node_history_cluster_{add,remove}.
Copied from dlm_controld.

Signed-off-by: David Teigland <teigland at redhat.com>
---
 group/gfs_controld/Makefile.am   |    4 +-
 group/gfs_controld/gfs_daemon.h  |    4 +
 group/gfs_controld/main.c        |    8 ++
 group/gfs_controld/member_cman.c |  131 +++++++++++++++++++++++++++++++++++---
 4 files changed, 136 insertions(+), 11 deletions(-)

diff --git a/group/gfs_controld/Makefile.am b/group/gfs_controld/Makefile.am
index 6e20244..f8d646d 100644
--- a/group/gfs_controld/Makefile.am
+++ b/group/gfs_controld/Makefile.am
@@ -17,8 +17,8 @@ gfs_controld_CPPFLAGS	= -I$(top_srcdir)/group/libgfscontrol \
 
 gfs_controld_CFLAGS	= $(ccs_CFLAGS) $(logt_CFLAGS) $(dlmcontrol_CFLAGS) \
 			  $(fenced_CFLAGS) $(sackpt_CFLAGS) $(cpg_CFLAGS) \
-			  $(cfg_CFLAGS)
+			  $(cfg_CFLAGS) $(quorum_CFLAGS)
 
 gfs_controld_LDFLAGS	= $(ccs_LIBS) $(logt_LIBS) $(dlmcontrol_LIBS) \
 			  $(cpg_LIBS) $(cfg_LIBS) $(sackpt_LIBS) \
-			  $(fenced_LIBS) -lpthread
+			  $(fenced_LIBS) $(quorum_LIBS) -lpthread
diff --git a/group/gfs_controld/gfs_daemon.h b/group/gfs_controld/gfs_daemon.h
index 1d89497..fdfc603 100644
--- a/group/gfs_controld/gfs_daemon.h
+++ b/group/gfs_controld/gfs_daemon.h
@@ -212,6 +212,10 @@ void process_connection(int ci);
 void cluster_dead(int ci);
 
 /* member_cman.c */
+int setup_cluster(void);
+void close_cluster(void);
+void process_cluster(int ci);
+void update_cluster(void);
 int setup_cluster_cfg(void);
 void close_cluster_cfg(void);
 void process_cluster_cfg(int ci);
diff --git a/group/gfs_controld/main.c b/group/gfs_controld/main.c
index 2b5a87e..e52f04b 100644
--- a/group/gfs_controld/main.c
+++ b/group/gfs_controld/main.c
@@ -1049,6 +1049,13 @@ static void loop(void)
 		goto out;
 	client_add(rv, process_cluster_cfg, cluster_dead);
 
+	rv = setup_cluster();
+	if (rv < 0)
+		goto out;
+	client_add(rv, process_cluster, cluster_dead);
+
+	update_cluster();
+
 	rv = setup_ccs();
 	if (rv < 0)
 		goto out;
@@ -1135,6 +1142,7 @@ static void loop(void)
 	close_cpg_daemon();
 	close_logging();
 	close_ccs();
+	close_cluster();
 	close_cluster_cfg();
 
 	if (!list_empty(&mountgroups))
diff --git a/group/gfs_controld/member_cman.c b/group/gfs_controld/member_cman.c
index 16631d3..1c377f8 100644
--- a/group/gfs_controld/member_cman.c
+++ b/group/gfs_controld/member_cman.c
@@ -2,8 +2,129 @@
 #include "config.h"
 #include <corosync/corotypes.h>
 #include <corosync/cfg.h>
+#include <corosync/quorum.h>
 
-static corosync_cfg_handle_t ch;
+static corosync_cfg_handle_t	ch;
+static quorum_handle_t		qh;
+static uint32_t			old_nodes[MAX_NODES];
+static int			old_node_count;
+static uint32_t			quorum_nodes[MAX_NODES];
+static int			quorum_node_count;
+
+static int is_member(uint32_t *node_list, int count, uint32_t nodeid)
+{
+	int i;
+
+	for (i = 0; i < count; i++) {
+		if (node_list[i] == nodeid)
+			return 1;
+	}
+	return 0;
+}
+
+static int is_old_member(uint32_t nodeid)
+{
+	return is_member(old_nodes, old_node_count, nodeid);
+}
+
+int is_cluster_member(uint32_t nodeid)
+{
+	return is_member(quorum_nodes, quorum_node_count, nodeid);
+}
+
+static void quorum_callback(quorum_handle_t h, uint32_t quorate,
+			    uint64_t ring_seq, uint32_t node_list_entries,
+			    uint32_t *node_list)
+{
+	int i;
+
+	old_node_count = quorum_node_count;
+	memcpy(&old_nodes, &quorum_nodes, sizeof(old_nodes));
+
+	quorum_node_count = 0;
+	memset(&quorum_nodes, 0, sizeof(quorum_nodes));
+
+	for (i = 0; i < node_list_entries; i++)
+		quorum_nodes[quorum_node_count++] = node_list[i];
+
+	for (i = 0; i < old_node_count; i++) {
+		if (!is_cluster_member(old_nodes[i])) {
+			log_debug("cluster node %u removed", old_nodes[i]);
+			node_history_cluster_remove(old_nodes[i]);
+		}
+	}
+
+	for (i = 0; i < quorum_node_count; i++) {
+		if (!is_old_member(quorum_nodes[i])) {
+			log_debug("cluster node %u added", quorum_nodes[i]);
+			node_history_cluster_add(quorum_nodes[i]);
+		}
+	}
+}
+
+static quorum_callbacks_t quorum_callbacks =
+{
+	.quorum_notify_fn = quorum_callback,
+};
+
+void process_cluster(int ci)
+{
+	cs_error_t err;
+
+	err = quorum_dispatch(qh, CS_DISPATCH_ALL);
+	if (err != CS_OK)
+		cluster_dead(0);
+}
+
+/* Force re-read of quorum nodes */
+void update_cluster(void)
+{
+	cs_error_t err;
+
+	err = quorum_dispatch(qh, CS_DISPATCH_ONE);
+	if (err != CS_OK)
+		cluster_dead(0);
+}
+
+int setup_cluster(void)
+{
+	cs_error_t err;
+	int fd;
+
+	err = quorum_initialize(&qh, &quorum_callbacks);
+	if (err != CS_OK) {
+		log_error("quorum init error %d", err);
+		return -1;
+	}
+
+	err = quorum_fd_get(qh, &fd);
+	if (err != CS_OK) {
+		log_error("quorum fd_get error %d", err);
+		goto fail;
+	}
+
+	err = quorum_trackstart(qh, CS_TRACK_CURRENT);
+	if (err != CS_OK) {
+		log_error("quorum trackstart error %d", err);
+		goto fail;
+	}
+
+	old_node_count = 0;
+	memset(&old_nodes, 0, sizeof(old_nodes));
+	quorum_node_count = 0;
+	memset(&quorum_nodes, 0, sizeof(quorum_nodes));
+
+	return fd;
+ fail:
+	quorum_finalize(qh);
+	return -1;
+}
+
+void close_cluster(void)
+{
+	quorum_trackstop(qh);
+	quorum_finalize(qh);
+}
 
 void kick_node_from_cluster(int nodeid)
 {
@@ -84,11 +205,3 @@ void close_cluster_cfg(void)
 	corosync_cfg_finalize(ch);
 }
 
-/* what's the replacement for this?
- *	case CMAN_REASON_CONFIG_UPDATE:
- *		setup_logging();
- *		setup_ccs();
- *		break;
- *	}
- */
-


More information about the cluster-commits mailing list