gfs2-utils: master - gfs2-utils: Retire gfs_controld

Andrew Price andyp at fedoraproject.org
Fri May 31 11:15:54 UTC 2013


Gitweb:        http://git.fedorahosted.org/git/?p=gfs2-utils.git;a=commitdiff;h=fbfb14664cbaaafab2bcf33707728191d382d52a
Commit:        fbfb14664cbaaafab2bcf33707728191d382d52a
Parent:        8ed638b27a21dfa1776f55709592a453974c5d90
Author:        Andrew Price <anprice at redhat.com>
AuthorDate:    Thu May 30 14:27:02 2013 +0100
Committer:     Andrew Price <anprice at redhat.com>
CommitterDate: Thu May 30 14:27:02 2013 +0100

gfs2-utils: Retire gfs_controld

Since linux v3.3 gfs_controld has not been required and it has been
disabled by default in gfs2-utils for some time. This patch removes the
group/ directory containing gfs_control* and related files and removes
them from the build system and .gitignore accordingly.

tunegfs2 also #includes linux_endian.h which was in group/include/ so it
has been moved into gfs2/include/ for the time being.

It also fixes some other build system issues causing 'make distcheck' to
fail.

Signed-off-by: Andrew Price <anprice at redhat.com>
---
 .gitignore                          |    2 -
 Makefile.am                         |   18 +-
 configure.ac                        |   58 -
 gfs2/include/Makefile.am            |    2 +-
 gfs2/include/linux_endian.h         |   68 +
 gfs2/init.d/gfs2-cluster            |   94 -
 gfs2/system/gfs2-cluster.service    |   12 -
 group/Makefile.am                   |    3 -
 group/gfs_control/Makefile.am       |    9 -
 group/gfs_control/main.c            |  465 -----
 group/gfs_control/target.mk         |    3 -
 group/gfs_controld/Makefile.am      |   24 -
 group/gfs_controld/config.c         |  157 --
 group/gfs_controld/config.h         |   14 -
 group/gfs_controld/cpg-new.c        | 3600 -----------------------------------
 group/gfs_controld/crc.c            |   72 -
 group/gfs_controld/gfs_controld.h   |   37 -
 group/gfs_controld/gfs_daemon.h     |  241 ---
 group/gfs_controld/logging.c        |   65 -
 group/gfs_controld/main.c           | 1496 ---------------
 group/gfs_controld/member_cman.c    |  207 --
 group/gfs_controld/target.mk        |    3 -
 group/gfs_controld/util.c           |  266 ---
 group/include/Makefile.am           |    3 -
 group/include/linux_endian.h        |   68 -
 group/include/list.h                |  336 ----
 group/libgfscontrol/Makefile.am     |    9 -
 group/libgfscontrol/libgfscontrol.h |  122 --
 group/libgfscontrol/main.c          |  436 -----
 group/libgfscontrol/target.mk       |    3 -
 group/man/Makefile.am               |    3 -
 group/man/gfs_controld.8            |  122 --
 group/man/target.mk                 |    3 -
 tests/Makefile.am                   |    1 +
 34 files changed, 72 insertions(+), 7950 deletions(-)

diff --git a/.gitignore b/.gitignore
index f8f8a82..3fa77a0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,8 +44,6 @@ gfs2/tune/tunegfs2
 tests/check_libgfs2
 tests/testvol
 tests/tests.log
-group/gfs_control/gfs_control
-group/gfs_controld/gfs_controld
 ABOUT-NLS
 po/Makevars.template
 po/POTFILES
diff --git a/Makefile.am b/Makefile.am
index 1eea44c..963c060 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,4 +1,4 @@
-EXTRA_DIST		= autogen.sh
+EXTRA_DIST		= autogen.sh README.build
 
 AUTOMAKE_OPTIONS	= foreign
 
@@ -12,21 +12,7 @@ noinst_HEADERS		= make/copyright.cf
 
 ACLOCAL_AMFLAGS		= -I m4
 
-if BUILD_GFS_CONTROLD
-DIR_GROUP		= group
-endif
-
-SUBDIRS			= po $(DIR_GROUP) gfs2 doc tests
-
-install-exec-local:
-			$(INSTALL) -d $(DESTDIR)/$(LOGDIR)
-			$(INSTALL) -d $(DESTDIR)/$(CLUSTERVARRUN)
-			$(INSTALL) -d $(DESTDIR)/$(CLUSTERVARLIB)
-
-uninstall-local:
-			rmdir $(DESTDIR)/$(LOGDIR) || :;
-			rmdir $(DESTDIR)/$(CLUSTERVARRUN) || :;
-			rmdir $(DESTDIR)/$(CLUSTERVARLIB) || :;
+SUBDIRS			= po gfs2 doc tests
 
 maintainer-clean-local:
 	rm -rf m4
diff --git a/configure.ac b/configure.ac
index c8e52a3..51c42b1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -9,7 +9,6 @@ LT_PREREQ([2.2.6])
 LT_INIT
 
 AC_CONFIG_MACRO_DIR([m4])
-AC_CONFIG_SRCDIR([group/gfs_controld/config.c])
 AC_CONFIG_HEADERS([make/clusterautoconfig.h])
 
 AC_CANONICAL_HOST
@@ -89,36 +88,6 @@ AC_ARG_ENABLE([debug],
 	[  --enable-debug          enable debug build. ],
 	[ default="no" ])
 
-AC_ARG_ENABLE([gfs_controld],
-	[  --enable-gfs_controld   build gfs_controld. ],
-	[ default="no" ])
-
-AC_ARG_WITH([syslogfacility],
-	[  --syslogfacility=FACILITY
-                          cluster default syslog facility. ],
-	[ SYSLOGFACILITY="$withval" ],
-	[ SYSLOGFACILITY="LOG_LOCAL4" ])
-
-AC_ARG_WITH([sysloglevel],
-	[  --sysloglevel=LEVEL
-                          cluster default syslog level. ],
-	[ SYSLOGLEVEL="$withval" ],
-	[ SYSLOGLEVEL="LOG_INFO" ])
-
-# gfs_controld isn't required in the latest versions of cluster
-AM_CONDITIONAL([BUILD_GFS_CONTROLD], [test "x$enable_gfs_controld" = "xyes"])
-AS_IF([test "x$enable_gfs_controld" = "xyes"], [
-	PKG_CHECK_MODULES([corosync],[corosync])
-	PKG_CHECK_MODULES([cpg],[libcpg])
-	PKG_CHECK_MODULES([sackpt],[libSaCkpt])
-	PKG_CHECK_MODULES([logt],[liblogthread])
-	PKG_CHECK_MODULES([ccs],[libccs])
-	PKG_CHECK_MODULES([cfg],[libcfg])
-	PKG_CHECK_MODULES([fenced],[libfenced])
-	PKG_CHECK_MODULES([dlmcontrol],[libdlmcontrol])
-	PKG_CHECK_MODULES([quorum],[libquorum])
-])
-
 # We use the Check framework for unit tests
 PKG_CHECK_MODULES([check], [check >= 0.9.8],
                   [have_check=yes],
@@ -176,27 +145,6 @@ AC_FUNC_MALLOC
 AC_FUNC_REALLOC
 AC_CHECK_FUNCS([ftruncate gettimeofday memset realpath rmdir select setlocale socket strcasecmp strchr strdup strerror strstr])
 
-## random vars
-
-LOGDIR=${localstatedir}/log/cluster
-CLUSTERVARRUN=${localstatedir}/run/cluster
-
-## do subst
-
-AC_SUBST([LOGDIR])
-AC_DEFINE_UNQUOTED([LOGDIR], "$(eval echo ${LOGDIR})",
-		   [Default logging directory])
-
-AC_SUBST([CLUSTERVARRUN])
-AC_DEFINE_UNQUOTED([CLUSTERVARRUN], "$(eval echo ${CLUSTERVARRUN})",
-		   [Default cluster var/run directory])
-
-AC_DEFINE_UNQUOTED([SYSLOGFACILITY], $(eval echo ${SYSLOGFACILITY}),
-		   [Default syslog facility])
-
-AC_DEFINE_UNQUOTED([SYSLOGLEVEL], $(eval echo ${SYSLOGLEVEL}),
-		   [Default syslog level])
-
 ## *FLAGS handling
 
 ENV_CFLAGS="$CFLAGS"
@@ -255,12 +203,6 @@ CPPFLAGS="-I\$(top_builddir)/make -I\$(top_srcdir)/make \
 LDFLAGS="$ENV_LDFLAGS"
 
 AC_CONFIG_FILES([Makefile
-		 group/Makefile
-		 group/libgfscontrol/Makefile
-		 group/gfs_control/Makefile
-		 group/gfs_controld/Makefile
-		 group/man/Makefile
-		 group/include/Makefile
 		 gfs2/Makefile
 		 gfs2/include/Makefile
 		 gfs2/libgfs2/Makefile
diff --git a/gfs2/include/Makefile.am b/gfs2/include/Makefile.am
index fff2953..a017cb3 100644
--- a/gfs2/include/Makefile.am
+++ b/gfs2/include/Makefile.am
@@ -1,3 +1,3 @@
 MAINTAINERCLEANFILES	= Makefile.in
 
-noinst_HEADERS		= osi_list.h
+noinst_HEADERS		= osi_list.h osi_tree.h linux_endian.h
diff --git a/gfs2/include/linux_endian.h b/gfs2/include/linux_endian.h
new file mode 100644
index 0000000..43089d2
--- /dev/null
+++ b/gfs2/include/linux_endian.h
@@ -0,0 +1,68 @@
+#ifndef __LINUX_ENDIAN_DOT_H__
+#define __LINUX_ENDIAN_DOT_H__
+
+
+#include <endian.h>
+#include <byteswap.h>
+
+
+/*  I'm not sure which versions of alpha glibc/gcc are broken,
+    so fix all of them.  */
+#ifdef __alpha__
+#undef bswap_64
+static __inline__ unsigned long bswap_64(unsigned long x)
+{
+  unsigned int h = x >> 32;
+  unsigned int l = x;
+
+  h = bswap_32(h);
+  l = bswap_32(l);
+
+  return ((unsigned long)l << 32) | h;
+}
+#endif  /*  __alpha__  */
+
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+
+#define be16_to_cpu(x) (x)
+#define be32_to_cpu(x) (x)
+#define be64_to_cpu(x) (x)
+
+#define cpu_to_be16(x) (x)
+#define cpu_to_be32(x) (x)
+#define cpu_to_be64(x) (x)
+
+#define le16_to_cpu(x) (bswap_16((x)))
+#define le32_to_cpu(x) (bswap_32((x)))
+#define le64_to_cpu(x) (bswap_64((x)))
+
+#define cpu_to_le16(x) (bswap_16((x)))
+#define cpu_to_le32(x) (bswap_32((x)))
+#define cpu_to_le64(x) (bswap_64((x)))
+
+#endif  /*  __BYTE_ORDER == __BIG_ENDIAN  */
+
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+
+#define be16_to_cpu(x) (bswap_16((x)))
+#define be32_to_cpu(x) (bswap_32((x)))
+#define be64_to_cpu(x) (bswap_64((x)))
+
+#define cpu_to_be16(x) (bswap_16((x)))
+#define cpu_to_be32(x) (bswap_32((x)))
+#define cpu_to_be64(x) (bswap_64((x))) 
+
+#define le16_to_cpu(x) (x)
+#define le32_to_cpu(x) (x)
+#define le64_to_cpu(x) (x)
+
+#define cpu_to_le16(x) (x)
+#define cpu_to_le32(x) (x)
+#define cpu_to_le64(x) (x)
+
+#endif  /*  __BYTE_ORDER == __LITTLE_ENDIAN  */
+
+
+#endif  /*  __LINUX_ENDIAN_DOT_H__  */
diff --git a/gfs2/init.d/gfs2-cluster b/gfs2/init.d/gfs2-cluster
deleted file mode 100644
index d0f1c0e..0000000
--- a/gfs2/init.d/gfs2-cluster
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/sh
-#
-# gfs_controld - GFS2 cluster filesystem daemon
-#
-# chkconfig: - 23 76
-# description: Starts and stops gfs_controld for dlm based GFS2 filesystems
-#
-
-
-### BEGIN INIT INFO
-# Provides: gfs_controld
-# Required-Start: $network cman
-# Required-Stop: $network cman
-# Should-Start:
-# Should-Stop:
-# Default-Start:
-# Default-Stop:
-# Short-Description: Starts and stops gfs_controld
-# Description: Starts and stops gfs_controld for dlm based GFS2 filesystems
-### END INIT INFO
-
-. /etc/rc.d/init.d/functions
-
-prog="gfs_controld"
-lockfile="/var/lock/subsys/$prog"
-exec="/usr/sbin/$prog"
-
-start() {
-	[ -x $exec ] || exit 5
-	echo -n $"Starting $prog: "
-	daemon $prog
-	retval=$?
-	echo
-	[ $retval -eq 0 ] && touch $lockfile
-	return $retval
-}
-
-stop() {
-	echo -n $"Stopping $prog: "
-	killproc $prog
-	retval=$?
-	echo
-	[ $retval -eq 0 ] && rm -f $lockfile
-}
-
-restart() {
-	stop
-	start
-}
-
-reload() {
-	restart
-}
-
-rh_status() {
-	status $prog
-}
-
-rh_status_q() {
-	rh_status >/dev/null 2>&1
-}
-
-case "$1" in
-	start)
-		rh_status_q && exit 0
-		$1
-		;;
-	stop)
-		rh_status_q || exit 0
-		$1
-		;;
-	restart)
-		$1
-		;;
-	reload)
-		rh_status_q || exit 7
-		$1
-		;;
-	force-reload)
-		force_reload
-		;;
-	status)
-		rh_status
-		;;
-	condrestart|try-restart)
-		rh_status_q || exit 0
-		restart
-		;;
-	*)
-		echo $"Usage $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
-		exit 2
-esac
-exit $?
-
diff --git a/gfs2/system/gfs2-cluster.service b/gfs2/system/gfs2-cluster.service
deleted file mode 100644
index d6bc0ca..0000000
--- a/gfs2/system/gfs2-cluster.service
+++ /dev/null
@@ -1,12 +0,0 @@
-[Unit]
-Description=GFS2 Control Daemon
-After=syslog.target network.target cman.service
-Before=remote-fs-pre.target
-Wants=remote-fs-pre.target
-
-[Service]
-Type=forking
-ExecStart=/usr/sbin/gfs_controld $OPTIONS
-
-[Install]
-WantedBy=multi-user.target
diff --git a/group/Makefile.am b/group/Makefile.am
deleted file mode 100644
index 5b7d0f8..0000000
--- a/group/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-MAINTAINERCLEANFILES	= Makefile.in
-
-SUBDIRS			= libgfscontrol gfs_control gfs_controld man include
diff --git a/group/gfs_control/Makefile.am b/group/gfs_control/Makefile.am
deleted file mode 100644
index dd75f6f..0000000
--- a/group/gfs_control/Makefile.am
+++ /dev/null
@@ -1,9 +0,0 @@
-MAINTAINERCLEANFILES	= Makefile.in
-
-sbin_PROGRAMS		= gfs_control
-
-gfs_control_SOURCES	= main.c
-
-gfs_control_CPPFLAGS	= -I$(top_srcdir)/group/libgfscontrol
-
-gfs_control_LDADD	= $(top_builddir)/group/libgfscontrol/libgfscontrol.la
diff --git a/group/gfs_control/main.c b/group/gfs_control/main.c
deleted file mode 100644
index dff32e3..0000000
--- a/group/gfs_control/main.c
+++ /dev/null
@@ -1,465 +0,0 @@
-#include "clusterautoconfig.h"
-
-#include <sys/types.h>
-#include <sys/un.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <stddef.h>
-#include <fcntl.h>
-#include <string.h>
-#include <errno.h>
-#include <limits.h>
-#include <netinet/in.h>
-
-#include "libgfscontrol.h"
-
-#define OPTION_STRING			"nhV"
-
-#define OP_LIST				1
-#define OP_DUMP				2
-#define OP_PLOCKS			3
-#define OP_JOIN				4
-#define OP_LEAVE			5
-#define OP_JOINLEAVE			6
-
-static char *fsname;
-static int operation;
-static int opt_ind;
-static int ls_all_nodes;
-
-#define MAX_MG 128
-#define MAX_NODES 128
-
-struct gfsc_mountgroup mgs[MAX_MG];
-struct gfsc_node nodes[MAX_NODES];
-
-
-static void print_usage(const char *prog_name)
-{
-	printf("Usage:\n");
-	printf("\n");
-	printf("%s [options] [ls|dump|plocks]\n", prog_name);
-	printf("\n");
-	printf("Options:\n");
-	printf("  -n               Show all node information in ls\n");
-	printf("  -h               Print this help, then exit\n");
-	printf("  -V               Print program version information, then exit\n");
-	printf("\n");
-}
-
-static void decode_arguments(int argc, char **argv)
-{
-	int cont = 1;
-	int optchar;
-	int need_fsname;
-
-	while (cont) {
-		optchar = getopt(argc, argv, OPTION_STRING);
-
-		switch (optchar) {
-		case 'n':
-			ls_all_nodes = 1;
-			break;
-
-		case 'h':
-			print_usage(argv[0]);
-			exit(EXIT_SUCCESS);
-			break;
-
-		case 'V':
-			printf("%s %s (built %s %s)\n",
-				argv[0], VERSION, __DATE__, __TIME__);
-			/* printf("%s\n", REDHAT_COPYRIGHT); */
-			exit(EXIT_SUCCESS);
-			break;
-
-		case ':':
-		case '?':
-			fprintf(stderr, "Please use '-h' for usage.\n");
-			exit(EXIT_FAILURE);
-			break;
-
-		case EOF:
-			cont = 0;
-			break;
-
-		default:
-			fprintf(stderr, "unknown option: %c\n", optchar);
-			exit(EXIT_FAILURE);
-			break;
-		};
-	}
-
-	need_fsname = 1;
-
-	while (optind < argc) {
-
-		if (!strncmp(argv[optind], "leave", 5) &&
-			   (strlen(argv[optind]) == 5)) {
-			operation = OP_LEAVE;
-			opt_ind = optind + 1;
-			break;
-		} else if (!strncmp(argv[optind], "ls", 2) &&
-			   (strlen(argv[optind]) == 2)) {
-			operation = OP_LIST;
-			opt_ind = optind + 1;
-			need_fsname = 0;
-			break;
-		} else if (!strncmp(argv[optind], "dump", 4) &&
-			   (strlen(argv[optind]) == 4)) {
-			operation = OP_DUMP;
-			opt_ind = optind + 1;
-			need_fsname = 0;
-			break;
-		} else if (!strncmp(argv[optind], "plocks", 6) &&
-			   (strlen(argv[optind]) == 6)) {
-			operation = OP_PLOCKS;
-			opt_ind = optind + 1;
-			break;
-		}
-
-		optind++;
-	}
-
-	if (!operation || !opt_ind) {
-		print_usage(argv[0]);
-		exit(EXIT_FAILURE);
-	}
-
-	if (optind < argc - 1)
-		fsname = argv[opt_ind];
-	else if (need_fsname) {
-		fprintf(stderr, "fs name required\n");
-		exit(EXIT_FAILURE);
-	}
-}
-
-static int do_write(int fd, void *buf, size_t count)
-{
-	int rv, off = 0;
-
- retry:
-	rv = write(fd, (char *)buf + off, count);
-	if (rv == -1 && errno == EINTR)
-		goto retry;
-	if (rv < 0)
-		return rv;
-
-	if (rv != count) {
-		count -= rv;
-		off += rv;
-		goto retry;
-	}
-	return 0;
-}
-
-static void do_leave(char *table)
-{
-	struct gfsc_mount_args ma;
-	int rv;
-
-	memset(&ma, 0, sizeof(ma));
-
-	strncpy(ma.table, table, sizeof(ma.table));
-
-	rv = gfsc_fs_leave(&ma, 0);
-	if (rv < 0)
-		fprintf(stderr, "gfs_controld leave error %d\n", rv);
-}
-
-static char *mg_flags_str(uint32_t flags)
-{
-	static char str[128];
-	int i = 0;
-
-	memset(str, 0, sizeof(str));
-
-	if (flags & GFSC_MF_LOCAL_RECOVERY_BUSY) {
-		i++;
-		strcat(str, "recover");
-	}
-	if (flags & GFSC_MF_FIRST_RECOVERY_NEEDED) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "need_first");
-	}
-	if (flags & GFSC_MF_KERNEL_MOUNT_ERROR) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "error");
-	}
-	if (flags & GFSC_MF_KERNEL_MOUNT_DONE) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "mounted");
-	}
-	if (flags & GFSC_MF_KERNEL_STOPPED) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "blocked");
-	}
-	if (flags & GFSC_MF_LEAVING) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "leave");
-	}
-	if (flags & GFSC_MF_JOINING) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "join");
-	}
-
-	/* leave this one out, it will often be set and we don't need
-	   to see it */
-	/*
-	if (flags & GFSC_MF_FIRST_RECOVERY_MSG)
-		strcat(str, "first_recovery_msg ");
-	*/
-	return str;
-}
-
-static char *node_mount_str(uint32_t flags)
-{
-	static char str[128];
-	int i = 0;
-
-	memset(str, 0, sizeof(str));
-
-	if (flags & GFSC_NF_KERNEL_MOUNT_DONE) {
-		i++;
-		strcat(str, "done");
-	}
-	if (flags & GFSC_NF_KERNEL_MOUNT_ERROR) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "error");
-	}
-	if (flags & GFSC_NF_READONLY) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "ro");
-	}
-	if (flags & GFSC_NF_SPECTATOR) {
-		strcat(str, i++ ? "," : "");
-		strcat(str, "spect");
-	}
-
-	if (!i)
-		strcat(str, "none");
-
-	/* ignoring CHECK_DLM */
-
-	return str;
-}
-
-static int member_int(struct gfsc_node *n)
-{
-	if (n->flags & GFSC_NF_DISALLOWED)
-		return -1;
-	if (n->flags & GFSC_NF_MEMBER)
-		return 1;
-	return 0;
-}
-
-static const char *condition_str(int cond)
-{
-	switch (cond) {
-	case 0:
-		return "";
-	case 1:
-		return "kernel_mount_done";
-	case 2:
-		return "notify_nodeid";
-	case 3:
-		return "poll_dlm";
-	case 4:
-		return "pending";
-	default:
-		return "unknown";
-	}
-}
-
-static void show_nodeids(int count, struct gfsc_node *nodes_in)
-{
-	struct gfsc_node *n = nodes_in;
-	int i;
-
-	for (i = 0; i < count; i++) {
-		printf("%d ", n->nodeid);
-		n++;
-	}
-	printf("\n");
-}
-
-static int node_compare(const void *va, const void *vb)
-{
-	const struct gfsc_node *a = va;
-	const struct gfsc_node *b = vb;
-
-	return a->nodeid - b->nodeid;
-}
-
-static void show_mg(struct gfsc_mountgroup *mg)
-{
-	int rv, node_count;
-
-	printf("name          %s\n", mg->name);
-	printf("id            0x%08x\n", mg->global_id);
-	printf("flags         0x%08x %s\n",
-		mg->flags, mg_flags_str(mg->flags));
-	printf("change        member %d joined %d remove %d failed %d seq %d,%d\n",
-		mg->cg_prev.member_count, mg->cg_prev.joined_count,
-		mg->cg_prev.remove_count, mg->cg_prev.failed_count,
-	        mg->cg_prev.combined_seq, mg->cg_prev.seq);
-
-	node_count = 0;
-	memset(&nodes, 0, sizeof(nodes));
-	rv = gfsc_mountgroup_nodes(mg->name, GFSC_NODES_MEMBERS,
-				   MAX_NODES, &node_count, nodes);
-	if (rv < 0) {
-		printf("members       error\n");
-		goto next;
-	}
-	qsort(nodes, node_count, sizeof(struct gfsc_node), node_compare);
-
-	printf("members       ");
-	show_nodeids(node_count, nodes);
-
- next:
-	if (!mg->cg_next.seq)
-		return;
-
-	printf("new change    member %d joined %d remove %d failed %d seq %d,%d\n",
-		mg->cg_next.member_count, mg->cg_next.joined_count,
-		mg->cg_next.remove_count, mg->cg_next.failed_count,
-		mg->cg_next.combined_seq, mg->cg_next.seq);
-
-	printf("new status    wait_messages %d wait_condition %d %s\n",
-		mg->cg_next.wait_messages, mg->cg_next.wait_condition,
-		condition_str(mg->cg_next.wait_condition));
-
-	node_count = 0;
-	memset(&nodes, 0, sizeof(nodes));
-	rv = gfsc_mountgroup_nodes(mg->name, GFSC_NODES_NEXT,
-				   MAX_NODES, &node_count, nodes);
-	if (rv < 0) {
-		printf("new members   error\n");
-		return;
-	}
-	qsort(nodes, node_count, sizeof(struct gfsc_node), node_compare);
-
-	printf("new members ");
-	show_nodeids(node_count, nodes);
-}
-
-static void show_all_nodes(int count, struct gfsc_node *nodes_in)
-{
-	struct gfsc_node *n = nodes_in;
-	int i;
-
-	for (i = 0; i < count; i++) {
-		printf("nodeid %d jid %d member %d failed %d start %d seq_add %u seq_rem %u mount %s\n",
-			n->nodeid,
-			n->jid,
-			member_int(n),
-			n->failed_reason,
-			(n->flags & GFSC_NF_START) ? 1 : 0,
-			n->added_seq,
-			n->removed_seq,
-			node_mount_str(n->flags));
-		n++;
-	}
-}
-
-static void do_list(char *name)
-{
-	struct gfsc_mountgroup *mg;
-	int node_count;
-	int mg_count;
-	int rv;
-	int i;
-
-	memset(mgs, 0, sizeof(mgs));
-
-	if (name) {
-		mg_count = 1;
-		rv = gfsc_mountgroup_info(name, mgs);
-	} else {
-		rv = gfsc_mountgroups(MAX_MG, &mg_count, mgs);
-	}
-
-	if (rv < 0)
-		exit(EXIT_FAILURE); /* gfs_controld probably not running */
-
-	if (mg_count)
-		printf("gfs mountgroups\n");
-
-	for (i = 0; i < mg_count; i++) {
-		mg = &mgs[i];
-
-		show_mg(mg);
-
-		if (!ls_all_nodes)
-			goto next;
-
-		node_count = 0;
-		memset(&nodes, 0, sizeof(nodes));
-
-		rv = gfsc_mountgroup_nodes(mg->name, GFSC_NODES_ALL,
-					   MAX_NODES, &node_count, nodes);
-		if (rv < 0) {
-			printf("all nodes error %d %d\n", rv, errno);
-			goto next;
-		}
-
-		qsort(nodes, node_count, sizeof(struct gfsc_node),node_compare);
-
-		printf("all nodes\n");
-		show_all_nodes(node_count, nodes);
- next:
-		printf("\n");
-	}
-}
-
-static void do_plocks(char *name)
-{
-	char buf[GFSC_DUMP_SIZE];
-
-	memset(buf, 0, sizeof(buf));
-
-	gfsc_dump_plocks(name, buf);
-
-	do_write(STDOUT_FILENO, buf, strlen(buf));
-}
-
-static void do_dump(void)
-{
-	char buf[GFSC_DUMP_SIZE];
-
-	memset(buf, 0, sizeof(buf));
-
-	gfsc_dump_debug(buf);
-
-	do_write(STDOUT_FILENO, buf, strlen(buf));
-}
-
-int main(int argc, char **argv)
-{
-	decode_arguments(argc, argv);
-
-	switch (operation) {
-
-	case OP_LEAVE:
-		do_leave(fsname);
-		break;
-
-	case OP_LIST:
-		do_list(fsname);
-		break;
-
-	case OP_DUMP:
-		do_dump();
-		break;
-
-	case OP_PLOCKS:
-		do_plocks(fsname);
-		break;
-	}
-	return 0;
-}
-
diff --git a/group/gfs_control/target.mk b/group/gfs_control/target.mk
deleted file mode 100644
index ff65253..0000000
--- a/group/gfs_control/target.mk
+++ /dev/null
@@ -1,3 +0,0 @@
-
-$(eval $(call make-binary,/usr/sbin/gfs_control,group/libgfscontrol/libgfscontrol.a))
-
diff --git a/group/gfs_controld/Makefile.am b/group/gfs_controld/Makefile.am
deleted file mode 100644
index f8d646d..0000000
--- a/group/gfs_controld/Makefile.am
+++ /dev/null
@@ -1,24 +0,0 @@
-MAINTAINERCLEANFILES	= Makefile.in
-
-sbin_PROGRAMS		= gfs_controld
-
-noinst_HEADERS		= config.h gfs_controld.h gfs_daemon.h
-
-gfs_controld_SOURCES	= config.c \
-			  cpg-new.c \
-			  crc.c \
-			  logging.c \
-			  main.c \
-			  member_cman.c \
-			  util.c
-
-gfs_controld_CPPFLAGS	= -I$(top_srcdir)/group/libgfscontrol \
-			  -I$(top_srcdir)/group/include
-
-gfs_controld_CFLAGS	= $(ccs_CFLAGS) $(logt_CFLAGS) $(dlmcontrol_CFLAGS) \
-			  $(fenced_CFLAGS) $(sackpt_CFLAGS) $(cpg_CFLAGS) \
-			  $(cfg_CFLAGS) $(quorum_CFLAGS)
-
-gfs_controld_LDFLAGS	= $(ccs_LIBS) $(logt_LIBS) $(dlmcontrol_LIBS) \
-			  $(cpg_LIBS) $(cfg_LIBS) $(sackpt_LIBS) \
-			  $(fenced_LIBS) $(quorum_LIBS) -lpthread
diff --git a/group/gfs_controld/config.c b/group/gfs_controld/config.c
deleted file mode 100644
index 3131189..0000000
--- a/group/gfs_controld/config.c
+++ /dev/null
@@ -1,157 +0,0 @@
-#include "gfs_daemon.h"
-
-#include <sys/types.h>
-#include <asm/types.h>
-#include <sys/uio.h>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <net/if.h>
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <limits.h>
-#include <unistd.h>
-#include <dirent.h>
-
-#include "config.h"
-#include "ccs.h"
-
-int ccs_handle;
-
-/* was a config value set on command line?, 0 or 1. */
-
-int optd_debug_logfile;
-int optd_enable_withdraw;
-
-/* actual config value from command line, cluster.conf, or default. */
-
-int cfgd_debug_logfile		= DEFAULT_DEBUG_LOGFILE;
-int cfgd_enable_withdraw	= DEFAULT_ENABLE_WITHDRAW;
-
-void read_ccs_name(const char *path, char *name)
-{
-	char *str;
-	int error;
-
-	error = ccs_get(ccs_handle, path, &str);
-	if (error || !str)
-		return;
-
-	strcpy(name, str);
-
-	free(str);
-}
-
-void read_ccs_yesno(const char *path, int *yes, int *no)
-{
-	char *str;
-	int error;
-
-	*yes = 0;
-	*no = 0;
-
-	error = ccs_get(ccs_handle, path, &str);
-	if (error || !str)
-		return;
-
-	if (!strcmp(str, "yes"))
-		*yes = 1;
-
-	else if (!strcmp(str, "no"))
-		*no = 1;
-
-	free(str);
-}
-
-int read_ccs_int(const char *path, int *config_val)
-{
-	char *str;
-	int val;
-	int error;
-
-	error = ccs_get(ccs_handle, path, &str);
-	if (error || !str)
-		return -1;
-
-	val = atoi(str);
-
-	if (val < 0) {
-		log_error("ignore invalid value %d for %s", val, path);
-		return -1;
-	}
-
-	*config_val = val;
-	log_debug("%s is %u", path, val);
-	free(str);
-	return 0;
-}
-
-#define LOCKSPACE_NODIR "/cluster/dlm/lockspace[@name=\"%s\"]/@nodir"
-
-void read_ccs_nodir(struct mountgroup *mg, char *buf)
-{
-	char path[PATH_MAX];
-	char *str;
-	int val;
-	int error;
-
-	memset(path, 0, PATH_MAX);
-	sprintf(path, LOCKSPACE_NODIR, mg->name);
-
-	error = ccs_get(ccs_handle, path, &str);
-	if (error || !str)
-		return;
-
-	val = atoi(str);
-
-	if (val < 0) {
-		log_error("ignore invalid value %d for %s", val, path);
-		return;
-	}
-
-	snprintf(buf, 32, ":nodir=%d", val);
-
-	log_debug("%s is %u", path, val);
-	free(str);
-}
-
-#define ENABLE_WITHDRAW_PATH "/cluster/gfs_controld/@enable_withdraw"
-
-int setup_ccs(void)
-{
-	int cd;
-
-	if (ccs_handle)
-		return 0;
-
-	cd = ccs_connect();
-	if (cd < 0) {
-		log_error("ccs_connect error %d %d", cd, errno);
-		return -1;
-	}
-	ccs_handle = cd;
-
-	if (!optd_enable_withdraw)
-		read_ccs_int(ENABLE_WITHDRAW_PATH, &cfgd_enable_withdraw);
-
-	read_ccs_name("/cluster/@name", clustername);
-	log_debug("cluster name \"%s\"", clustername);
-
-	return 0;
-}
-
-void close_ccs(void)
-{
-	ccs_disconnect(ccs_handle);
-}
-
diff --git a/group/gfs_controld/config.h b/group/gfs_controld/config.h
deleted file mode 100644
index bc4788d..0000000
--- a/group/gfs_controld/config.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#ifndef __CONFIG_DOT_H__
-#define __CONFIG_DOT_H__
-
-#define DEFAULT_DEBUG_LOGFILE 0
-#define DEFAULT_ENABLE_WITHDRAW 1
-
-extern int optd_debug_logfile;
-extern int optd_enable_withdraw;
-
-extern int cfgd_debug_logfile;
-extern int cfgd_enable_withdraw;
-
-#endif
-
diff --git a/group/gfs_controld/cpg-new.c b/group/gfs_controld/cpg-new.c
deleted file mode 100644
index 8cf2b00..0000000
--- a/group/gfs_controld/cpg-new.c
+++ /dev/null
@@ -1,3600 +0,0 @@
-#include "gfs_daemon.h"
-#include "config.h"
-#include "libdlmcontrol.h"
-
-#define MAX_JOURNALS 256
-
-/* gfs_header types */
-enum {
-	GFS_MSG_PROTOCOL		= 1,
-	GFS_MSG_START			= 2,
-	GFS_MSG_MOUNT_DONE		= 3,
-	GFS_MSG_FIRST_RECOVERY_DONE	= 4,
-	GFS_MSG_RECOVERY_RESULT		= 5,
-	GFS_MSG_REMOUNT			= 6,
-	GFS_MSG_WITHDRAW		= 7,
-	GFS_MSG_WITHDRAW_ACK		= 8,
-};
-
-/* gfs_header flags */
-#define GFS_MFLG_JOINING   1  /* accompanies start, we are joining */
-
-struct gfs_header {
-	uint16_t version[3];	/* daemon_run protocol */
-	uint16_t type;          /* GFS_MSG_ */
-	uint32_t nodeid;        /* sender */
-	uint32_t to_nodeid;     /* recipient, 0 for all */
-	uint32_t global_id;     /* global unique id for this lockspace */
-	uint32_t flags;         /* GFS_MFLG_ */
-	uint32_t msgdata;       /* in-header payload depends on MSG type */
-	uint32_t pad1;
-	uint64_t pad2;
-};
-
-struct protocol_version {
-	uint16_t major;
-	uint16_t minor;
-	uint16_t patch;
-	uint16_t flags;
-};
-
-struct protocol {
-	union {
-		struct protocol_version dm_ver;
-		uint16_t		daemon_max[4];
-	};
-	union {
-		struct protocol_version km_ver;
-		uint16_t		kernel_max[4];
-	};
-	union {
-		struct protocol_version dr_ver;
-		uint16_t		daemon_run[4];
-	};
-	union {
-		struct protocol_version kr_ver;
-		uint16_t		kernel_run[4];
-	};
-};
-
-/* mg_info and id_info: for syncing state in start message */
-
-struct mg_info {
-	uint32_t mg_info_size;
-	uint32_t id_info_size;
-	uint32_t id_info_count;
-
-	uint32_t started_count;
-
-	int member_count;
-	int joined_count;
-	int remove_count;
-	int failed_count;
-
-	int first_recovery_needed;
-	int first_recovery_master;
-};
-
-#define IDI_NODEID_IS_MEMBER	0x00000001
-#define IDI_JID_NEEDS_RECOVERY	0x00000002
-#define IDI_MOUNT_DONE		0x00000008
-#define IDI_MOUNT_ERROR		0x00000010
-#define IDI_MOUNT_RO		0x00000020
-#define IDI_MOUNT_SPECTATOR	0x00000040
-
-struct id_info {
-	int nodeid;
-	int jid;
-	uint32_t flags;
-};
-
-#define JID_NONE -1
-
-struct journal {
-	struct list_head list;
-	int jid;
-	int nodeid;
-	int failed_nodeid;
-	int needs_recovery;
-
-	int local_recovery_busy;
-	int local_recovery_done;
-	int local_recovery_result;
-	int failed_recovery_count;
-};
-
-struct node {
-	struct list_head list;
-	int nodeid;
-	int jid;
-	int ro;
-	int spectator;
-	int kernel_mount_done;
-	int kernel_mount_error;
-
-	int check_dlm;
-	int dlm_notify_callback;
-	int dlm_notify_result;
-
-	int failed_reason;
-	uint32_t added_seq;
-	uint32_t removed_seq;
-	uint64_t add_time;
-
-	int withdraw;
-	int send_withdraw_ack;
-
-	uint64_t cluster_add_time;
-	uint64_t cluster_remove_time;
-
-	struct protocol proto;
-};
-
-struct member {
-	struct list_head list;
-	int nodeid;
-	int start;   /* 1 if we received a start message for this change */
-	int added;   /* 1 if added by this change */
-	int failed;  /* 1 if failed in this change */
-	int disallowed;
-	char *start_msg; /* full copy of the start message from this node */
-	struct mg_info *mg_info; /* shortcut into started_msg */
-};
-
-/* One of these change structs is created for every confchg a cpg gets. */
-
-#define CGST_WAIT_CONDITIONS 1
-#define CGST_WAIT_MESSAGES   2
-
-struct change {
-	struct list_head list;
-	struct list_head members;
-	struct list_head removed; /* nodes removed by this change */
-	struct list_head saved_messages; /* saved messages */
-	int member_count;
-	int joined_count;
-	int remove_count;
-	int failed_count;
-	int state;
-	int we_joined;
-	uint32_t seq; /* used as a reference for debugging, and for queries */
-	uint32_t combined_seq; /* for queries */
-	uint64_t create_time;
-};
-
-struct save_msg {
-	struct list_head list;
-	int len;
-	char buf[0];
-};
-
-static int dlmcontrol_fd;
-static int cpg_fd_daemon;
-static struct protocol our_protocol;
-static struct list_head daemon_nodes;
-static struct cpg_address daemon_member[MAX_NODES];
-static int daemon_member_count;
-
-/* 
-   cpg confchg's arrive telling us that mountgroup members have
-   joined/left/failed.  A "change" struct is created for each confchg,
-   and added to the mg->changes list.
-
-   apply_changes()
-   ---------------
-
-   <a new node won't know whether first_recovery_needed or not, but it also
-    won't have any conditions to wait for, so a new node will go directly to
-    sending out start message regardless>
-
-   if first_recovery_needed,
-   (or new, where new is not having completed a start barrier yet)
-   all nodes: skip wait conditions
-   all nodes: send start message
-
-   else !first_recovery_needed,
-   all nodes: if failures in changes, wait for conditions:
-              local mount to complete if in progress, stop_kernel, dlm_notified
-   all nodes: send start message
-
-   <new changes that arrive result in going back to beginning; start messages
-    from this aborted start cycle will be ignored>
-
-   all nodes: wait for all start messages
-
-   <once all start messages are received, new changes will be handled in a
-    new batch after all current changes are cleared at end of sync_state>
-
-   if start cycle / start barrier completes (start messages received from
-   all nodes without being interrupted by a change), go on to sync_state
-   which puts all members (as defined by the most recent change) in sync.
-
-   "old nodes" are nodes that have completed a start cycle before (have
-   a non-zero started_count), and "new nodes" are nodes that have not
-   completed a start cycle before (they are being added by one of the
-   changes in this start cycle)
-
-   sync_state()
-   ------------
-
-   if old nodes have first_recovery_needed, or all nodes are new
-   all nodes: mg->first_recovery_needed = 1
-   all nodes: mg->first_recovery_master = prev or new low nodeid
-   new nodes: instantiate existing state to match old nodes
-   old nodes: update state per the changes in the completed start cycle
-   all nodes: assign jids to new members
-   all nodes: clear all change structs
-
-   else !first_recovery_needed,
-   new nodes: instantiate existing state to match old nodes
-   old nodes: update state per the changes in the completed start cycle
-   all nodes: assign jids to new members
-   all nodes: clear all change structs
-
-   <new changes that arrive from here on result in going back to the top>
-
-   apply_recovery()
-   ----------------
-
-   if first_recovery_needed,
-   master:    tells mount to run with first=1 (if not already)
-   all nodes: wait for first_recovery_done message
-   master:    sends first_recovery_done message when mount is done
-   all nodes: mg->first_recovery_needed = 0
-   all nodes: start kernel / tell mount.gfs to mount(2) (master already did)
-   all nodes: send message with result of kernel mount
-
-   else !first_recovery_needed,
-   all nodes: if there are no journals to recover, goto start kernel
-   old nodes: tell kernel to recover jids, send message with each result
-   all nodes: wait for all recoveries to be done
-   all nodes: start kernel
-   new nodes: tell mount.gfs to mount(2)
-   new nodes: send message with result of kernel mount
-
-   [If no one can recover some journal(s), all will be left waiting, unstarted.
-    A new change from a new mount will result in things going back to the top,
-    and hopefully the new node will be successful at doing the journal
-    recoveries when it comes through the apply_recovery() section, which
-    would let everyone start again.]
-*/
-
-static void apply_changes_recovery(struct mountgroup *mg);
-static void send_withdraw_acks(struct mountgroup *mg);
-
-static void log_config(const struct cpg_name *group_name,
-		       const struct cpg_address *member_list,
-		       size_t member_list_entries,
-		       const struct cpg_address *left_list,
-		       size_t left_list_entries,
-		       const struct cpg_address *joined_list,
-		       size_t joined_list_entries)
-{
-	char m_buf[128];
-	char j_buf[32];
-	char l_buf[32];
-	size_t i, len, pos;
-	int ret;
-
-	memset(m_buf, 0, sizeof(m_buf));
-	memset(j_buf, 0, sizeof(j_buf));
-	memset(l_buf, 0, sizeof(l_buf));
-
-	len = sizeof(m_buf);
-	pos = 0;
-	for (i = 0; i < member_list_entries; i++) {
-		ret = snprintf(m_buf + pos, len - pos, " %d",
-			       member_list[i].nodeid);
-		if (ret >= len - pos)
-			break;
-		pos += ret;
-	}
-
-	len = sizeof(j_buf);
-	pos = 0;
-	for (i = 0; i < joined_list_entries; i++) {
-		ret = snprintf(j_buf + pos, len - pos, " %d",
-			       joined_list[i].nodeid);
-		if (ret >= len - pos)
-			break;
-		pos += ret;
-	}
-
-	len = sizeof(l_buf);
-	pos = 0;
-	for (i = 0; i < left_list_entries; i++) {
-		ret = snprintf(l_buf + pos, len - pos, " %d",
-			       left_list[i].nodeid);
-		if (ret >= len - pos)
-			break;
-		pos += ret;
-	}
-
-	log_debug("%s conf %zu %zu %zu memb%s join%s left%s", group_name->value,
-		  member_list_entries, joined_list_entries, left_list_entries,
-		  m_buf, j_buf, l_buf);
-}
-
-static const char *msg_name(int type)
-{
-	switch (type) {
-	case GFS_MSG_PROTOCOL:
-		return "protocol";
-	case GFS_MSG_START:
-		return "start";
-	case GFS_MSG_MOUNT_DONE:
-		return "mount_done";
-	case GFS_MSG_FIRST_RECOVERY_DONE:
-		return "first_recovery_done";
-	case GFS_MSG_RECOVERY_RESULT:
-		return "recovery_result";
-	case GFS_MSG_REMOUNT:
-		return "remount";
-	case GFS_MSG_WITHDRAW:
-		return "withdraw";
-	case GFS_MSG_WITHDRAW_ACK:
-		return "withdraw_ack";
-	default:
-		return "unknown";
-	}
-}
-
-static int _send_message(cpg_handle_t h, void *buf, int len, int type)
-{
-	struct iovec iov;
-	cpg_error_t error;
-	int retries = 0;
-
-	iov.iov_base = buf;
-	iov.iov_len = len;
-
- retry:
-	error = cpg_mcast_joined(h, CPG_TYPE_AGREED, &iov, 1);
-	if (error == CPG_ERR_TRY_AGAIN) {
-		retries++;
-		usleep(1000);
-		if (!(retries % 100))
-			log_error("cpg_mcast_joined retry %d %s",
-				   retries, msg_name(type));
-		goto retry;
-	}
-	if (error != CPG_OK) {
-		log_error("cpg_mcast_joined error %d handle %llx %s",
-			  error, (unsigned long long)h, msg_name(type));
-		return -1;
-	}
-
-	if (retries)
-		log_debug("cpg_mcast_joined retried %d %s",
-			  retries, msg_name(type));
-
-	return 0;
-}
-
-/* header fields caller needs to set: type, to_nodeid, flags, msgdata */
-
-static void gfs_send_message(struct mountgroup *mg, char *buf, int len)
-{
-	struct gfs_header *hd = (struct gfs_header *) buf;
-	int type = hd->type;
-
-	hd->version[0]  = cpu_to_le16(our_protocol.daemon_run[0]);
-	hd->version[1]  = cpu_to_le16(our_protocol.daemon_run[1]);
-	hd->version[2]  = cpu_to_le16(our_protocol.daemon_run[2]);
-	hd->type	= cpu_to_le16(hd->type);
-	hd->nodeid      = cpu_to_le32(our_nodeid);
-	hd->to_nodeid   = cpu_to_le32(hd->to_nodeid);
-	hd->global_id   = cpu_to_le32(mg->id);
-	hd->flags       = cpu_to_le32(hd->flags);
-	hd->msgdata     = cpu_to_le32(hd->msgdata);
-
-	_send_message(mg->cpg_handle, buf, len, type);
-}
-
-static struct member *find_memb(struct change *cg, int nodeid)
-{
-	struct member *memb;
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (memb->nodeid == nodeid)
-			return memb;
-	}
-	return NULL;
-}
-
-static struct mountgroup *find_mg_handle(cpg_handle_t h)
-{
-	struct mountgroup *mg;
-
-	list_for_each_entry(mg, &mountgroups, list) {
-		if (mg->cpg_handle == h)
-			return mg;
-	}
-	return NULL;
-}
-
-static struct mountgroup *find_mg_ci(int ci)
-{
-	struct mountgroup *mg;
-
-	list_for_each_entry(mg, &mountgroups, list) {
-		if (mg->cpg_client == ci)
-			return mg;
-	}
-	return NULL;
-}
-
-static struct journal *find_journal(struct mountgroup *mg, int jid)
-{
-	struct journal *j;
-
-	list_for_each_entry(j, &mg->journals, list) {
-		if (j->jid == jid)
-			return j;
-	}
-	return NULL;
-}
-
-static struct journal *find_journal_by_nodeid(struct mountgroup *mg, int nodeid)
-{
-	struct journal *j;
-
-	list_for_each_entry(j, &mg->journals, list) {
-		if (j->nodeid == nodeid)
-			return j;
-	}
-	return NULL;
-}
-
-static void free_cg(struct change *cg)
-{
-	struct member *memb, *safe;
-	struct save_msg *sm, *sm2;
-
-	list_for_each_entry_safe(memb, safe, &cg->members, list) {
-		list_del(&memb->list);
-		if (memb->start_msg)
-			free(memb->start_msg);
-		free(memb);
-	}
-	list_for_each_entry_safe(memb, safe, &cg->removed, list) {
-		list_del(&memb->list);
-		if (memb->start_msg)
-			free(memb->start_msg);
-		free(memb);
-	}
-	list_for_each_entry_safe(sm, sm2, &cg->saved_messages, list) {
-		list_del(&sm->list);
-		free(sm);
-	}
-	free(cg);
-}
-
-void free_mg(struct mountgroup *mg)
-{
-	struct change *cg, *cg_safe;
-	struct node *node, *node_safe;
-
-	list_for_each_entry_safe(cg, cg_safe, &mg->changes, list) {
-		list_del(&cg->list);
-		free_cg(cg);
-	}
-
-	if (mg->started_change)
-		free_cg(mg->started_change);
-
-	list_for_each_entry_safe(node, node_safe, &mg->node_history, list) {
-		list_del(&node->list);
-		free(node);
-	}
-
-	free(mg);
-}
-
-static struct node *get_node_history(struct mountgroup *mg, int nodeid)
-{
-	struct node *node;
-
-	list_for_each_entry(node, &mg->node_history, list) {
-		if (node->nodeid == nodeid)
-			return node;
-	}
-	return NULL;
-}
-
-static void node_history_init(struct mountgroup *mg, int nodeid,
-			      struct change *cg)
-{
-	struct node *node;
-
-	node = get_node_history(mg, nodeid);
-	if (node) {
-		list_del(&node->list);
-		goto out;
-	}
-
-	node = malloc(sizeof(struct node));
-	if (!node) {
-		log_error("node_history_init no mem");
-		return;
-	}
- out:
-	memset(node, 0, sizeof(struct node));
-
-	node->nodeid = nodeid;
-	node->add_time = 0;
-	list_add_tail(&node->list, &mg->node_history);
-
-	if (cg)
-		node->added_seq = cg->seq;	/* for queries */
-}
-
-void node_history_cluster_add(int nodeid)
-{
-	struct mountgroup *mg;
-	struct node *node;
-
-	list_for_each_entry(mg, &mountgroups, list) {
-		node_history_init(mg, nodeid, NULL);
-
-		node = get_node_history(mg, nodeid);
-		if (!node) {
-			log_error("node_history_cluster_add no nodeid %d",
-				  nodeid);
-			return;
-		}
-
-		node->cluster_add_time = time(NULL);
-	}
-}
-
-void node_history_cluster_remove(int nodeid)
-{
-	struct mountgroup *mg;
-	struct node *node;
-
-	list_for_each_entry(mg, &mountgroups, list) {
-		node = get_node_history(mg, nodeid);
-		if (!node) {
-			log_error("node_history_cluster_remove no nodeid %d",
-				  nodeid);
-			return;
-		}
-
-		node->cluster_remove_time = time(NULL);
-	}
-}
-
-static void node_history_start(struct mountgroup *mg, int nodeid)
-{
-	struct node *node;
-	
-	node = get_node_history(mg, nodeid);
-	if (!node) {
-		log_error("node_history_start no nodeid %d", nodeid);
-		return;
-	}
-
-	node->add_time = time(NULL);
-}
-
-static void node_history_left(struct mountgroup *mg, int nodeid,
-			      struct change *cg)
-{
-	struct node *node;
-
-	node = get_node_history(mg, nodeid);
-	if (!node) {
-		log_error("node_history_left no nodeid %d", nodeid);
-		return;
-	}
-
-	node->add_time = 0;
-	node->removed_seq = cg->seq;	/* for queries */
-}
-
-static void node_history_fail(struct mountgroup *mg, int nodeid,
-			      struct change *cg, int reason)
-{
-	struct node *node;
-
-	node = get_node_history(mg, nodeid);
-	if (!node) {
-		log_error("node_history_fail no nodeid %d", nodeid);
-		return;
-	}
-
-	node->check_dlm = 1;
-
-	node->removed_seq = cg->seq;	/* for queries */
-	node->failed_reason = reason;	/* for queries */
-}
-
-static int is_added(struct mountgroup *mg, int nodeid)
-{
-	struct change *cg;
-	struct member *memb;
-
-	list_for_each_entry(cg, &mg->changes, list) {
-		memb = find_memb(cg, nodeid);
-		if (memb && memb->added)
-			return 1;
-	}
-	return 0;
-}
-
-static int is_withdraw(struct mountgroup *mg, int nodeid)
-{
-	struct node *node;
-
-	node = get_node_history(mg, nodeid);
-	if (!node) {
-		log_error("is_withdraw no nodeid %d", nodeid);
-		return 0;
-	}
-	return node->withdraw;
-}
-
-static int journals_need_recovery(struct mountgroup *mg)
-{
-	struct change *cg;
-	struct journal *j;
-	struct member *memb;
-	int count = 0;
-
-	list_for_each_entry(j, &mg->journals, list)
-		if (j->needs_recovery)
-			count++;
-
-	list_for_each_entry(cg, &mg->changes, list) {
-		list_for_each_entry(memb, &cg->removed, list) {
-			if (!memb->failed && !is_withdraw(mg, memb->nodeid))
-				continue;
-			/* check whether this node had a journal assigned? */
-			count++;
-		}
-	}
-
-	return count;
-}
-
-/* find a start message from an old node to use; it doesn't matter which old
-   node we take the start message from, they should all be the same */
-
-static int get_id_list(struct mountgroup *mg, struct id_info **ids,
-		       int *count, int *size)
-{
-	struct change *cg;
-	struct member *memb;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (!memb->mg_info->started_count)
-			continue;
-
-		*count = memb->mg_info->id_info_count;
-		*size = memb->mg_info->id_info_size;
-		*ids = (struct id_info *)(memb->start_msg +
-					  sizeof(struct gfs_header) +
-					  memb->mg_info->mg_info_size);
-		return 0;
-	}
-	return -1;
-}
-
-static struct id_info *get_id_struct(struct id_info *ids, int count, int size,
-				     int nodeid)
-{
-	struct id_info *id = ids;
-	int i;
-
-	for (i = 0; i < count; i++) {
-		if (id->nodeid == nodeid)
-			return id;
-		id = (struct id_info *)((char *)id + size);
-	}
-	return NULL;
-}
-
-static void start_kernel(struct mountgroup *mg)
-{
-	struct change *cg = mg->started_change;
-
-	if (!mg->kernel_stopped) {
-		log_error("start_kernel cg %u not stopped", cg->seq);
-		return;
-	}
-
-	log_group(mg, "start_kernel cg %u member_count %d",
-		  cg->seq, cg->member_count);
-
-	set_sysfs(mg, "block", 0);
-	mg->kernel_stopped = 0;
-
-	if (mg->joining) {
-		client_reply_join_full(mg, 0);
-		mg->joining = 0;
-		mg->mount_client_notified = 1;
-	}
-}
-
-static void stop_kernel(struct mountgroup *mg)
-{
-	if (!mg->kernel_stopped) {
-		log_group(mg, "stop_kernel");
-		set_sysfs(mg, "block", 1);
-		mg->kernel_stopped = 1;
-	}
-}
-
-void process_dlmcontrol(int ci)
-{
-	struct mountgroup *mg;
-	struct node *node;
-	char name[GFS_MOUNTGROUP_LEN+1];
-	int rv, type, nodeid, result;
-
-	memset(name, 0, sizeof(name));
-
-	rv = dlmc_fs_result(dlmcontrol_fd, name, &type, &nodeid, &result);
-	if (rv) {
-		log_error("process_dlmcontrol dlmc_fs_result %d", rv);
-		return;
-	}
-
-	mg = find_mg(name);
-	if (!mg) {
-		log_error("process_dlmcontrol no mg %s", name);
-		return;
-	}
-
-	if (type == DLMC_RESULT_NOTIFIED) {
-		log_group(mg, "process_dlmcontrol notified nodeid %d result %d",
-			  nodeid, result);
-
-		node = get_node_history(mg, nodeid);
-		if (!node) {
-			/* shouldn't happen */
-			log_error("process_dlmcontrol no nodeid %d", nodeid);
-			return;
-		}
-
-		if (mg->dlm_notify_nodeid != nodeid) {
-			/* shouldn't happen */
-			log_error("process_dlmcontrol node %d expected %d",
-				  nodeid, mg->dlm_notify_nodeid);
-			return;
-		}
-
-		mg->dlm_notify_nodeid = 0;
-		node->dlm_notify_callback = 1;
-		node->dlm_notify_result = result;
-
-	} else if (type == DLMC_RESULT_REGISTER) {
-		if (result) {
-			/* shouldn't happen */
-			log_error("process_dlmcontrol register %d", result);
-			client_reply_join_full(mg, result);
-			mg->mount_client_notified = 1;
-		} else {
-			log_group(mg, "process_dlmcontrol register %d", result);
-			mg->dlm_registered = 1;
-		}
-	} else {
-		log_group(mg, "process_dlmcontrol unknown type %d", type);
-	}
-
-	poll_dlm = 0;
-
-	apply_changes_recovery(mg);
-}
-
-static int check_dlm_notify_done(struct mountgroup *mg)
-{
-	struct node *node;
-	int rv;
-
-	/* we're waiting for a notify result from the dlm (could we fire off
-	   all dlmc_fs_notified() calls at once instead of serially?) */
-
-	if (mg->dlm_notify_nodeid)
-		return 0;
-
-	list_for_each_entry(node, &mg->node_history, list) {
-
-		/* check_dlm is set when we see a node fail, and is cleared
-		   below when we find that the dlm has also seen it fail */
-
-		if (!node->check_dlm)
-			continue;
-
-		/* we're in sync with the dlm for this nodeid, i.e. we've
-		   both seen this node fail */
-
-		if (node->dlm_notify_callback && !node->dlm_notify_result) {
-			node->dlm_notify_callback = 0;
-			node->check_dlm = 0;
-			continue;
-		}
-
-		/* we're not in sync with the dlm for this nodeid, i.e.
-		   the dlm hasn't seen this node fail yet; try calling
-		   dlmc_fs_notified() again in a bit */
-
-		if (node->dlm_notify_callback && node->dlm_notify_result) {
-			log_group(mg, "check_dlm_notify result %d will retry nodeid %d",
-				  node->dlm_notify_result, node->nodeid);
-			node->dlm_notify_callback = 0;
-			poll_dlm = 1;
-			return 0;
-		}
-
-		/* check if the dlm has seen this nodeid fail, we get the
-		   answer asynchronously in process_dlmcontrol */
-
-		log_group(mg, "check_dlm_notify nodeid %d begin", node->nodeid);
-
-		rv = dlmc_fs_notified(dlmcontrol_fd, mg->name, node->nodeid);
-		if (rv) {
-			log_error("dlmc_fs_notified error %d", rv);
-			return 0;
-		}
-
-		mg->dlm_notify_nodeid = node->nodeid;
-		return 0;
-	}
-
-	log_group(mg, "check_dlm_notify done");
-	return 1;
-}
-
-static int wait_conditions_done(struct mountgroup *mg)
-{
-	if (!mg->dlm_registered) {
-		log_group(mg, "wait_conditions need dlm register");
-		return 0;
-	}
-
-	if (mg->first_recovery_needed) {
-		log_group(mg, "wait_conditions skip for first_recovery_needed");
-		return 1;
-	}
-
-	if (!mg->started_count) {
-		log_group(mg, "wait_conditions skip for zero started_count");
-		return 1;
-	}
-
-	if (!journals_need_recovery(mg)) {
-		log_group(mg, "wait_conditions skip for zero "
-			 "journals_need_recovery");
-		return 1;
-	}
-
-	if (!mg->mount_client_notified) {
-		log_group(mg, "wait_conditions skip mount client not notified");
-		return 1;
-	}
-
-	if (mg->kernel_mount_done && mg->kernel_mount_error) {
-		log_group(mg, "wait_conditions skip for kernel_mount_error");
-		return 1;
-	}
-
-	if (!mg->kernel_mount_done) {
-		log_group(mg, "wait_conditions need mount_done");
-		return 0;
-	}
-
-	stop_kernel(mg);
-
-	if (!check_dlm_notify_done(mg))
-		return 0;
-
-	return 1;
-}
-
-static int wait_messages_done(struct mountgroup *mg)
-{
-	struct change *cg = list_first_entry(&mg->changes, struct change, list);
-	struct member *memb;
-	int need = 0, total = 0;
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (!memb->start)
-			need++;
-		total++;
-	}
-
-	if (need) {
-		log_group(mg, "wait_messages cg %u need %d of %d",
-			  cg->seq, need, total);
-		return 0;
-	}
-
-	log_group(mg, "wait_messages cg %u got all %d", cg->seq, total);
-	return 1;
-}
-
-static void cleanup_changes(struct mountgroup *mg)
-{
-	struct change *cg = list_first_entry(&mg->changes, struct change, list);
-	struct change *safe;
-
-	list_del(&cg->list);
-	if (mg->started_change)
-		free_cg(mg->started_change);
-	mg->started_change = cg;
-
-	/* zero started_count means "never started" */
-
-	mg->started_count++;
-	if (!mg->started_count)
-		mg->started_count++;
-
-	cg->combined_seq = cg->seq; /* for queries */
-
-	list_for_each_entry_safe(cg, safe, &mg->changes, list) {
-		mg->started_change->combined_seq = cg->seq; /* for queries */
-		list_del(&cg->list);
-		free_cg(cg);
-	}
-}
-
-/* do the change details in the message match the details of the given change */
-
-static int match_change(struct mountgroup *mg, struct change *cg,
-			struct gfs_header *hd, struct mg_info *mi,
-			struct id_info *ids)
-{
-	struct id_info *id;
-	struct member *memb;
-	struct node *node;
-	uint32_t seq = hd->msgdata;
-	int i, members_mismatch;
-
-	/* We can ignore messages if we're not in the list of members.
-	   The one known time this will happen is after we've joined
-	   the cpg, we can get messages for changes prior to the change
-	   in which we're added. */
-
-	id = get_id_struct(ids, mi->id_info_count, mi->id_info_size,our_nodeid);
-
-	if (!id || !(id->flags & IDI_NODEID_IS_MEMBER)) {
-		log_group(mg, "match_change %d:%u skip cg %u we are not in members",
-			  hd->nodeid, seq, cg->seq);
-		return 0;
-	}
-
-	memb = find_memb(cg, hd->nodeid);
-	if (!memb) {
-		log_group(mg, "match_change %d:%u skip cg %u sender not member",
-			  hd->nodeid, seq, cg->seq);
-		return 0;
-	}
-
-	if (memb->start && hd->type == GFS_MSG_START) {
-		log_group(mg, "match_change %d:%u skip %u already start",
-			  hd->nodeid, seq, cg->seq);
-		return 0;
-	}
-
-	/* a node's start can't match a change if the node joined the cluster
-	   more recently than the change was created */
-
-	node = get_node_history(mg, hd->nodeid);
-	if (!node) {
-		log_group(mg, "match_change %d:%u skip cg %u no node history",
-			  hd->nodeid, seq, cg->seq);
-		return 0;
-	}
-
-	if (node->cluster_add_time > cg->create_time) {
-		log_group(mg, "match_change %d:%u skip cg %u created %llu "
-			  "cluster add %llu", hd->nodeid, seq, cg->seq,
-			  (unsigned long long)cg->create_time,
-			  (unsigned long long)node->cluster_add_time);
-		return 0;
-	}
-
-	/* verify this is the right change by matching the counts
-	   and the nodeids of the current members */
-
-	if (mi->member_count != cg->member_count ||
-	    mi->joined_count != cg->joined_count ||
-	    mi->remove_count != cg->remove_count ||
-	    mi->failed_count != cg->failed_count) {
-		log_group(mg, "match_change %d:%u skip cg %u expect counts "
-			  "%d %d %d %d", hd->nodeid, seq, cg->seq,
-			  cg->member_count, cg->joined_count,
-			  cg->remove_count, cg->failed_count);
-		return 0;
-	}
-
-	members_mismatch = 0;
-	id = ids;
-
-	for (i = 0; i < mi->id_info_count; i++) {
-		if (id->flags & IDI_NODEID_IS_MEMBER) {
-			memb = find_memb(cg, id->nodeid);
-			if (!memb) {
-				log_group(mg, "match_change %d:%u skip cg %u "
-					  "no memb %d", hd->nodeid, seq,
-					  cg->seq, id->nodeid);
-				members_mismatch = 1;
-				break;
-			}
-		}
-		id = (struct id_info *)((char *)id + mi->id_info_size);
-	}
-
-	if (members_mismatch)
-		return 0;
-
-	log_group(mg, "match_change %d:%u matches cg %u", hd->nodeid, seq,
-		  cg->seq);
-	return 1;
-}
-
-/* Unfortunately, there's no really simple way to match a message with the
-   specific change that it was sent for.  We hope that by passing all the
-   details of the change in the message, we will be able to uniquely match
-   it to the correct change. */
-
-/* A start message will usually be for the first (current) change on our list.
-   In some cases it will be for a non-current change, and we can ignore it:
-
-   1. A,B,C get confchg1 adding C
-   2. C sends start for confchg1
-   3. A,B,C get confchg2 adding D
-   4. A,B,C,D recv start from C for confchg1 - ignored
-   5. C,D send start for confchg2
-   6. A,B send start for confchg2
-   7. A,B,C,D recv all start messages for confchg2; start barrier/cycle done
- 
-   In step 4, how do the nodes know whether the start message from C is
-   for confchg1 or confchg2?  Hopefully by comparing the counts and members. */
-
-static struct change *find_change(struct mountgroup *mg, struct gfs_header *hd,
-				  struct mg_info *mi, struct id_info *ids)
-{
-	struct change *cg;
-
-	list_for_each_entry_reverse(cg, &mg->changes, list) {
-		if (!match_change(mg, cg, hd, mi, ids))
-			continue;
-		return cg;
-	}
-
-	log_group(mg, "find_change %d:%u no match", hd->nodeid, hd->msgdata);
-	return NULL;
-}
-
-static void mg_info_in(struct mg_info *mi)
-{
-	mi->mg_info_size  = le32_to_cpu(mi->mg_info_size);
-	mi->id_info_size  = le32_to_cpu(mi->id_info_size);
-	mi->id_info_count = le32_to_cpu(mi->id_info_count);
-	mi->started_count = le32_to_cpu(mi->started_count);
-	mi->member_count  = le32_to_cpu(mi->member_count);
-	mi->joined_count  = le32_to_cpu(mi->joined_count);
-	mi->remove_count  = le32_to_cpu(mi->remove_count);
-	mi->failed_count  = le32_to_cpu(mi->failed_count);
-	mi->first_recovery_needed = le32_to_cpu(mi->first_recovery_needed);
-	mi->first_recovery_master = le32_to_cpu(mi->first_recovery_master);
-}
-
-static void id_info_in(struct id_info *id)
-{
-	id->nodeid = le32_to_cpu(id->nodeid);
-	id->jid    = le32_to_cpu(id->jid);
-	id->flags  = le32_to_cpu(id->flags);
-}
-
-static void ids_in(struct mg_info *mi, struct id_info *ids)
-{
-	struct id_info *id;
-	int i;
-
-	id = ids;
-	for (i = 0; i < mi->id_info_count; i++) {
-		id_info_in(id);
-		id = (struct id_info *)((char *)id + mi->id_info_size);
-	}
-}
-
-static void receive_start(struct mountgroup *mg, struct gfs_header *hd, int len)
-{
-	struct change *cg;
-	struct member *memb;
-	struct mg_info *mi;
-	struct id_info *ids;
-	uint32_t seq = hd->msgdata;
-	int added;
-
-	log_group(mg, "receive_start %d:%u len %d", hd->nodeid, seq, len);
-
-	mi = (struct mg_info *)((char *)hd + sizeof(struct gfs_header));
-	ids = (struct id_info *)((char *)mi + sizeof(struct mg_info));
-
-	mg_info_in(mi);
-	ids_in(mi, ids);
-
-	cg = find_change(mg, hd, mi, ids);
-	if (!cg)
-		return;
-
-	memb = find_memb(cg, hd->nodeid);
-	if (!memb) {
-		/* this should never happen since match_change checks it */
-		log_error("receive_start no member %d", hd->nodeid);
-		return;
-	}
-
-	added = is_added(mg, hd->nodeid);
-
-	if (added && mi->started_count && mg->started_count) {
-		log_error("receive_start %d:%u add node with started_count %u",
-			  hd->nodeid, seq, mi->started_count);
-
-		/* see comment in fence/fenced/cpg.c */
-		memb->disallowed = 1;
-		return;
-	}
-
-	node_history_start(mg, hd->nodeid);
-	memb->start = 1;
-
-	if (memb->start_msg) {
-		/* shouldn't happen */
-		log_error("receive_start %d:%u dup start msg", hd->nodeid, seq);
-		return;
-	}
-
-	/* save a copy of each start message */
-	memb->start_msg = malloc(len);
-	if (!memb->start_msg) {
-		log_error("receive_start len %d no mem", len);
-		return;
-	}
-	memcpy(memb->start_msg, hd, len);
-
-	/* a shortcut to the saved mg_info */
-	memb->mg_info = (struct mg_info *)(memb->start_msg +
-					   sizeof(struct gfs_header));
-}
-
-/* start messages are associated with a specific change and use the
-   find_change/match_change routines to make sure all start messages
-   are matched with the same change on all nodes.  The current set of
-   changes are cleared after a completed start cycle.  Other messages
-   happen outside the context of changes.  An "incomplete" start cycle
-   is when a confchg arrives (adding a new change struct) before all
-   start messages have been received for the current change.  In this
-   case, all members send a new start message for the latest change,
-   and any start messages received for the previous change(s) are ignored.
-
-   To sync state with start messages, we need to include:
-   - the state before applying any of the current set of queued changes
-     (new nodes will initialize with this)
-   - the essential info from changes in the set that's being started,
-     so nodes added by one of the queued changes can apply the same changes
-     to the init state that the existing nodes do. */ 
-
-/* recovery_result and mount_done messages may arrive between the time
-   that an old node sends start and the time a new node receives it.
-   two old nodes may also send start before/after a recovery_result or
-   mount_done message, creating inconsistent data in their start messages.
-
-   Soln: a new node saves recovery_result/mount_done messages between
-   last confchg and final start.  the new node knows that a start message
-   from an old node may or may not include the effects from rr/md messages
-   since the last confchg, but *will* include all effects from prior to
-   the last confchg.  The saved rr/md messages can be applied on top of
-   the state from an old node's start message; applying them a second time
-   should not change anything, producing the same result. */
-
-static int count_ids(struct mountgroup *mg)
-{
-	struct change *cg;
-	struct member *memb;
-	struct journal *j;
-	int count = 0;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(memb, &cg->members, list)
-		count++;
-
-	list_for_each_entry(j, &mg->journals, list)
-		if (j->needs_recovery)
-			count++;
-
-	list_for_each_entry(cg, &mg->changes, list) {
-		list_for_each_entry(memb, &cg->removed, list) {
-			if (!memb->failed && !is_withdraw(mg, memb->nodeid))
-				continue;
-			j = find_journal_by_nodeid(mg, memb->nodeid);
-			if (j)
-				count++;
-		}
-	}
-
-	return count;
-}
-
-/* old member: current member that has completed a start cycle
-   new member: current member that has not yet completed a start cycle */
-
-static void send_start(struct mountgroup *mg)
-{
-	struct change *cg, *c;
-	struct gfs_header *hd;
-	struct mg_info *mi;
-	struct id_info *id;
-	struct member *memb;
-	struct node *node;
-	struct journal *j;
-	char *buf;
-	uint32_t flags;
-	int len, id_count, jid;
-	int old_memb = 0, new_memb = 0, old_journal = 0, new_journal = 0;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	id_count = count_ids(mg);
-
-	/* sanity check */
-
-	if (!mg->started_count && id_count != cg->member_count) {
-		log_error("send_start bad counts id_count %d member_count %d",
-			  cg->member_count, id_count);
-		return;
-	}
-
-	len = sizeof(struct gfs_header) + sizeof(struct mg_info) +
-	      id_count * sizeof(struct id_info);
-
-	buf = malloc(len);
-	if (!buf) {
-		log_error("send_start len %d no mem", len);
-		return;
-	}
-	memset(buf, 0, len);
-
-	hd = (struct gfs_header *)buf;
-	mi = (struct mg_info *)(buf + sizeof(*hd));
-	id = (struct id_info *)(buf + sizeof(*hd) + sizeof(*mi));
-
-	/* fill in header (gfs_send_message handles part of header) */
-
-	hd->type = GFS_MSG_START;
-	hd->msgdata = cg->seq;
-	hd->flags |= mg->joining ? GFS_MFLG_JOINING : 0;
-
-	/* fill in mg_info */
-
-	mi->mg_info_size  = cpu_to_le32(sizeof(struct mg_info));
-	mi->id_info_size  = cpu_to_le32(sizeof(struct id_info));
-	mi->id_info_count = cpu_to_le32(id_count);
-	mi->started_count = cpu_to_le32(mg->started_count);
-	mi->member_count  = cpu_to_le32(cg->member_count);
-	mi->joined_count  = cpu_to_le32(cg->joined_count);
-	mi->remove_count  = cpu_to_le32(cg->remove_count);
-	mi->failed_count  = cpu_to_le32(cg->failed_count);
-	mi->first_recovery_needed = cpu_to_le32(mg->first_recovery_needed);
-	mi->first_recovery_master = cpu_to_le32(mg->first_recovery_master);
-
-	/* fill in id_info entries */
-
-	/* New members send info about themselves, and empty id_info slots for
-	   all other members.  Old members send full info about all old
-	   members, and empty id_info slots about new members.  The union of
-	   start messages from a single old node and all new nodes give a
-	   complete picture of state for all members.  In sync_state, all nodes
-	   (old and new) make this union, and then assign jid's to new nodes. */
-
-	list_for_each_entry(memb, &cg->members, list) {
-
-		if (!mg->started_count || is_added(mg, memb->nodeid)) {
-			/* send empty slot for new member */
-			jid = JID_NONE;
-			flags = IDI_NODEID_IS_MEMBER;
-
-			/* include our own info which no one knows yet */
-			if (!mg->started_count && memb->nodeid == our_nodeid) {
-				flags |= mg->ro ? IDI_MOUNT_RO : 0;
-				flags |= mg->spectator ? IDI_MOUNT_SPECTATOR : 0;
-			}
-			new_memb++;
-
-		} else {
-			/* send full info for old member */
-			node = get_node_history(mg, memb->nodeid);
-			if (!node) {
-				log_error("send_start no nodeid %d", memb->nodeid);
-				continue;
-			}
-
-			jid = node->jid;
-			flags = IDI_NODEID_IS_MEMBER;
-			flags |= node->ro ? IDI_MOUNT_RO : 0;
-			flags |= node->spectator ? IDI_MOUNT_SPECTATOR : 0;
-			flags |= node->kernel_mount_done ? IDI_MOUNT_DONE : 0;
-			flags |= node->kernel_mount_error ? IDI_MOUNT_ERROR : 0;
-			old_memb++;
-		}
-
-		id->nodeid = cpu_to_le32(memb->nodeid);
-		id->jid    = cpu_to_le32(jid);
-		id->flags  = cpu_to_le32(flags);
-		id++;
-	}
-
-	/* journals needing recovery from previous start cycles */
-
-	list_for_each_entry(j, &mg->journals, list) {
-		if (j->needs_recovery) {
-			flags = IDI_JID_NEEDS_RECOVERY;
-			id->jid = cpu_to_le32(j->jid);
-			id->flags = cpu_to_le32(flags);
-			id++;
-			old_journal++;
-		}
-	}
-
-	/* journals needing recovery from the current start cycle */
-
-	list_for_each_entry(c, &mg->changes, list) {
-		list_for_each_entry(memb, &c->removed, list) {
-			if (!memb->failed && !is_withdraw(mg, memb->nodeid))
-				continue;
-			j = find_journal_by_nodeid(mg, memb->nodeid);
-			if (j) {
-				flags = IDI_JID_NEEDS_RECOVERY;
-				id->jid = cpu_to_le32(j->jid);
-				id->flags = cpu_to_le32(flags);
-				id++;
-				new_journal++;
-			}
-		}
-	}
-
-	/* sanity check */
-
-	if (!mg->started_count && (old_memb || old_journal || new_journal)) {
-		log_error("send_start cg %u bad counts om %d nm %d oj %d nj %d",
-			  cg->seq, old_memb, new_memb, old_journal, new_journal);
-		return;
-	}
-
-	log_group(mg, "send_start cg %u id_count %d om %d nm %d oj %d nj %d",
-		  cg->seq, id_count, old_memb, new_memb, old_journal,
-		  new_journal);
-
-	gfs_send_message(mg, buf, len);
-
-	free(buf);
-}
-
-static void send_mount_done(struct mountgroup *mg, int result)
-{
-	struct gfs_header h;
-
-	memset(&h, 0, sizeof(h));
-
-	h.type = GFS_MSG_MOUNT_DONE;
-	h.msgdata = result;
-
-	gfs_send_message(mg, (char *)&h, sizeof(h));
-}
-
-static void send_first_recovery_done(struct mountgroup *mg)
-{
-	struct gfs_header h;
-
-	memset(&h, 0, sizeof(h));
-
-	h.type = GFS_MSG_FIRST_RECOVERY_DONE;
-
-	gfs_send_message(mg, (char *)&h, sizeof(h));
-}
-
-static void send_recovery_result(struct mountgroup *mg, int jid, int result)
-{
-	struct gfs_header *hd;
-	char *buf;
-	int len, *p;
-
-	len = sizeof(struct gfs_header) + 2 * sizeof(int);
-
-	buf = malloc(len);
-	if (!buf) {
-		log_error("send_recovery_result no mem %d", len);
-		return;
-	}
-	memset(buf, 0, len);
-
-	hd = (struct gfs_header *)buf;
-	hd->type = GFS_MSG_RECOVERY_RESULT;
-
-	p = (int *)(buf + sizeof(struct gfs_header));
-
-	p[0] = cpu_to_le32(jid);
-	p[1] = cpu_to_le32(result);
-
-	gfs_send_message(mg, buf, len);
-
-	free(buf);
-}
-
-void send_remount(struct mountgroup *mg, int ro)
-{
-	struct gfs_header h;
-
-	memset(&h, 0, sizeof(h));
-
-	h.type = GFS_MSG_REMOUNT;
-	h.msgdata = ro;
-
-	gfs_send_message(mg, (char *)&h, sizeof(h));
-}
-
-void send_withdraw(struct mountgroup *mg)
-{
-	struct gfs_header h;
-
-	memset(&h, 0, sizeof(h));
-
-	h.type = GFS_MSG_WITHDRAW;
-
-	gfs_send_message(mg, (char *)&h, sizeof(h));
-}
-
-static void save_message(struct mountgroup *mg, struct gfs_header *hd, int len)
-{
-	struct change *cg;
-	struct save_msg *sm;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	sm = malloc(sizeof(struct save_msg) + len);
-	if (!sm) {
-		log_error("save_message len %d no mem", len);
-		return;
-	}
-
-	sm->len = len;
-	memcpy(sm->buf, hd, len);
-
-	list_add_tail(&sm->list, &cg->saved_messages);
-}
-
-void gfs_mount_done(struct mountgroup *mg)
-{
-	send_mount_done(mg, mg->kernel_mount_error);
-}
-
-static void receive_mount_done(struct mountgroup *mg, struct gfs_header *hd,
-			       int len)
-{
-	struct node *node;
-
-	log_group(mg, "receive_mount_done from %d result %d",
-		  hd->nodeid, hd->msgdata);
-
-	node = get_node_history(mg, hd->nodeid);
-	if (!node) {
-		log_error("receive_mount_done no nodeid %d", hd->nodeid);
-		return;
-	}
-
-	node->kernel_mount_done = 1;
-	node->kernel_mount_error = hd->msgdata;
-}
-
-static void receive_recovery_result(struct mountgroup *mg,
-				    struct gfs_header *hd, int len)
-{
-	struct journal *j;
-	int jid, result, *p;
-
-	p = (int *)((char *)hd + sizeof(struct gfs_header));
-	jid = le32_to_cpu(p[0]);
-	result = le32_to_cpu(p[1]);
-
-	log_group(mg, "receive_recovery_result from %d jid %d result %d",
-		  hd->nodeid, jid, result);
-
-	j = find_journal(mg, jid);
-	if (!j) {
-		log_error("receive_recovery_result from %d no jid %d",
-			  hd->nodeid, jid);
-		return;
-	}
-
-	if (!j->needs_recovery)
-		return;
-
-	if (result == LM_RD_SUCCESS)
-		j->needs_recovery = 0;
-	else {
-		j->failed_recovery_count++;
-		log_group(mg, "jid %d failed_recovery_count %d", jid,
-			  j->failed_recovery_count);
-	}
-}
-
-static void receive_first_recovery_done(struct mountgroup *mg,
-					struct gfs_header *hd, int len)
-{
-	int master = mg->first_recovery_master;
-
-	log_group(mg, "receive_first_recovery_done from %d master %d "
-		  "mount_client_notified %d",
-		  hd->nodeid, master, mg->mount_client_notified);
-
-	if (list_empty(&mg->changes)) {
-		/* everything is idle, no changes in progress */
-
-		mg->first_recovery_needed = 0;
-		mg->first_recovery_master = 0;
-		mg->first_recovery_msg = 1;
-
-		if (master != our_nodeid)
-			start_kernel(mg);
-	} else {
-		/* Everyone will receive this message in the same sequence
-		   wrt other start messages and confchgs:
-
-		   - If a new confchg arrives after this message (and before
-		     the final start message in the current start cycle),
-		     a new start cycle will begin.  All nodes before the
-		     confchg will have frn=0 due to receiving this message,
-		     and nodes added by the confchg will see frn=0 in all
-		     start messages (in any_nodes_first_recovery() which
-		     returns 0).
-
-		   - If the final start message arrives after this message,
-		     the start cycle will complete, running sync_state(), on
-		     all current nodes with all having seen this message.
-		     Old and new nodes in the current start cycle will see
-		     this msg and use it (first_recovery_msg) instead of the
-		     first_recovery_needed/master data in the start messages
-		     (which may be inconsistent due to members sending their
-		     start messages either before or after receiving this
-		     message). */
-
-		/* exclude new nodes from this sanity check since they've
-		   never set a master value to compare against */
-		if (mg->started_count && (master != hd->nodeid))
-			log_error("receive_first_recovery_done from %d "
-				  "master %d", hd->nodeid, master);
-
-		mg->first_recovery_needed = 0;
-		mg->first_recovery_master = 0;
-		mg->first_recovery_msg = 1;
-	}
-}
-
-static void receive_remount(struct mountgroup *mg, struct gfs_header *hd,
-			    int len)
-{
-	struct node *node;
-
-	log_group(mg, "receive_remount from %d ro %d", hd->nodeid, hd->msgdata);
-
-	node = get_node_history(mg, hd->nodeid);
-	if (!node) {
-		log_error("receive_remount no nodeid %d", hd->nodeid);
-		return;
-	}
-
-	node->ro = hd->msgdata;
-
-	if (hd->nodeid == our_nodeid)
-		mg->ro = node->ro;
-}
-
-/* The node with the withdraw wants to leave the mountgroup, but have
-   the other nodes do recovery for it when it leaves.  They wouldn't usually
-   do recovery for a node that leaves "normally", i.e. without failing at the
-   cluster membership level.  So, we send a withdraw message to tell the
-   others that our succeeding leave-removal should be followed by recovery
-   like a failure-removal would be.
-
-   The withdrawing node can't release dlm locks for the fs before other
-   nodes have stopped the fs.  The same reason as for any gfs journal
-   recovery; the locks on the failed/withdrawn fs "protect" the parts of
-   the fs that need to be recovered, and until the fs on all mounters has
-   been stopped/blocked, our existing dlm locks need to remain to prevent
-   other nodes from touching these parts of the fs.
-
-   So, the node doing withdraw needs to know that other nodes in the mountgroup
-   have blocked the fs before it sets /sys/fs/gfs/foo/withdraw to 1, which
-   tells gfs-kernel to continue and release dlm locks.
-
-   Until the node doing withdraw has released the dlm locks on the withdrawn
-   fs, the other nodes' attempts to recover the given journal will fail (they
-   fail to acquire the journal lock.) So, these nodes need to either wait until
-   the dlm locks have been released before attempting to recover the journal,
-   or retry failed attempts at recovering the journal.
-
-   How it works
-   . nodes A,B,C in mountgroup for fs foo
-   . foo is withrawn on node C
-   . C sends withdraw to all
-   . all set C->withraw = 1
-   . C leaves mountgroup
-   . A,B,C get confchg removing C
-   . A,B stop kernel foo
-   . A,B send out-of-band message to C indicating foo is stopped
-   . C gets OOB message and set /sys/fs/gfs/foo/withdraw to 1
-   . dlm locks for foo are released on C
-   . A,B will now be able to acquire C's journal lock for foo
-   . A,B will complete recovery of foo
-
-   An "in-band" message would be through cpg foo, but since C has left cpg
-   foo, we can't use that cpg, and have to go through an external channel.
-*/
-
-static void receive_withdraw(struct mountgroup *mg, struct gfs_header *hd,
-			     int len)
-{
-	struct node *node;
-
-	log_group(mg, "receive_withdraw from %d", hd->nodeid);
-
-	node = get_node_history(mg, hd->nodeid);
-	if (!node) {
-		log_error("receive_withdraw no nodeid %d", hd->nodeid);
-		return;
-	}
-	node->withdraw = 1;
-
-	if (hd->nodeid == our_nodeid)
-		gfs_leave_mountgroup(mg, 0);
-}
-
-/* start message from all nodes shows zero started_count */
-
-static int all_nodes_new(struct mountgroup *mg)
-{
-	struct change *cg;
-	struct member *memb;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (memb->mg_info->started_count)
-			return 0;
-	}
-	return 1;
-}
-
-/* does start message from any node with non-zero started_count have
-   first_recovery_needed set?  (verify that all started nodes agree on
-   first_recovery_needed) */
-
-static int any_nodes_first_recovery(struct mountgroup *mg)
-{
-	struct change *cg;
-	struct member *memb;
-	int yes = 0, no = 0, master = 0;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (!memb->mg_info->started_count)
-			continue;
-		if (memb->mg_info->first_recovery_needed)
-			yes++;
-		else
-			no++;
-	}
-
-	if (no && yes) {
-		/* disagreement on first_recovery_needed, shouldn't happen */
-		log_error("any_nodes_first_recovery no %d yes %d", no, yes);
-		return 1;
-	}
-
-	if (no)
-		return 0;
-
-	/* sanity check: verify agreement on the master */
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (!memb->mg_info->started_count)
-			continue;
-		if (!master) {
-			master = memb->mg_info->first_recovery_master;
-			continue;
-		}
-		if (master == memb->mg_info->first_recovery_master)
-			continue;
-
-		/* disagreement on master, shouldn't happen */
-		log_error("any_nodes_first_recovery master %d vs %d",
-			  master, memb->mg_info->first_recovery_master);
-	}
-
-	return 1;
-}
-
-/* If all nodes new, there's no previous master, pick low nodeid;
-   if not all nodes new, there will be a previous master, use that one unless
-   it's no longer a member; if master is no longer a member pick low nodeid.
-   The current master will already be set in mg->first_recovery_master for old
-   nodes, but new nodes will need to look in the start messages to find it. */
-
-static int pick_first_recovery_master(struct mountgroup *mg, int all_new)
-{
-	struct change *cg;
-	struct member *memb;
-	int old = 0, low = 0;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (memb->mg_info->started_count)
-			old = memb->mg_info->first_recovery_master;
-
-		if (!low)
-			low = memb->nodeid;
-		else if (memb->nodeid < low)
-			low = memb->nodeid;
-	}
-
-	memb = find_memb(cg, old);
-
-	if (!memb || all_new) {
-		log_group(mg, "pick_first_recovery_master low %d old %d",
-			  low, old);
-		return low;
-	}
-
-	log_group(mg, "pick_first_recovery_master old %d", old);
-	return old;
-}
-
-/* use a start message from an old node to create node info for each old node */
-
-static void create_old_nodes(struct mountgroup *mg)
-{
-	struct change *cg;
-	struct member *memb;
-	struct node *node;
-	struct journal *j;
-	struct id_info *ids, *id;
-	int id_count, id_size, rv;
-
-	/* get ids from a start message of an old node */
-
-	rv = get_id_list(mg, &ids, &id_count, &id_size);
-	if (rv) {
-		/* all new nodes, no old nodes */
-		log_group(mg, "create_old_nodes all new");
-		return;
-	}
-
-	/* use id list to set info for all old nodes */
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (!memb->mg_info->started_count)
-			continue;
-
-		node = get_node_history(mg, memb->nodeid);
-		id = get_id_struct(ids, id_count, id_size, memb->nodeid);
-
-		if (!node || !id) {
-			/* shouldn't happen */
-			log_error("create_old_nodes %d node %d id %d",
-				  memb->nodeid, !!node, !!id);
-			return;
-		}
-
-		if (!(id->flags & IDI_NODEID_IS_MEMBER) ||
-		     (id->flags & IDI_JID_NEEDS_RECOVERY)) {
-			/* shouldn't happen */
-			log_error("create_old_nodes %d bad flags %x",
-				  memb->nodeid, id->flags);
-			return;
-		}
-
-		node->jid		 = id->jid;
-		node->kernel_mount_done  = !!(id->flags & IDI_MOUNT_DONE);
-		node->kernel_mount_error = !!(id->flags & IDI_MOUNT_ERROR);
-		node->ro		 = !!(id->flags & IDI_MOUNT_RO);
-		node->spectator		 = !!(id->flags & IDI_MOUNT_SPECTATOR);
-
-		j = malloc(sizeof(struct journal));
-		if (!j) {
-			log_error("create_old_nodes no mem");
-			return;
-		}
-		memset(j, 0, sizeof(struct journal));
-
-		j->nodeid = node->nodeid;
-		j->jid = node->jid;
-		list_add(&j->list, &mg->journals);
-
-		log_group(mg, "create_old_nodes %d jid %d ro %d spect %d "
-			  "kernel_mount_done %d error %d",
-			  node->nodeid, node->jid, node->ro, node->spectator,
-			  node->kernel_mount_done, node->kernel_mount_error);
-	}
-}
-
-/* use start messages from new nodes to create node info for each new node */
-
-static void create_new_nodes(struct mountgroup *mg)
-{
-	struct change *cg;
-	struct member *memb;
-	struct id_info *ids, *id;
-	struct node *node;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (memb->mg_info->started_count)
-			continue;
-
-		node = get_node_history(mg, memb->nodeid);
-		if (!node) {
-			/* shouldn't happen */
-			log_error("create_new_nodes %d no node", memb->nodeid);
-			return;
-		}
-
-		ids = (struct id_info *)(memb->start_msg +
-					 sizeof(struct gfs_header) +
-					 memb->mg_info->mg_info_size);
-
-		id = get_id_struct(ids, memb->mg_info->id_info_count,
-				   memb->mg_info->id_info_size, memb->nodeid);
-
-		if (!(id->flags & IDI_NODEID_IS_MEMBER) ||
-		     (id->flags & IDI_JID_NEEDS_RECOVERY)) {
-			/* shouldn't happen */
-			log_error("create_new_nodes %d bad flags %x",
-				  memb->nodeid, id->flags);
-			return;
-		}
-
-		node->jid       = JID_NONE;
-		node->ro	= !!(id->flags & IDI_MOUNT_RO);
-		node->spectator = !!(id->flags & IDI_MOUNT_SPECTATOR);
-
-		log_group(mg, "create_new_nodes %d ro %d spect %d",
-			  node->nodeid, node->ro, node->spectator);
-	}
-}
-
-static void print_id_list(struct mountgroup *mg, struct id_info *ids,
-			  int id_count, int id_size)
-{
-	struct id_info *id = ids;
-	int i;
-
-	for (i = 0; i < id_count; i++) {
-		log_group(mg, "id nodeid %d jid %d flags %08x",
-			  id->nodeid, id->jid, id->flags);
-		id = (struct id_info *)((char *)id + id_size);
-	}
-}
-
-static void create_failed_journals(struct mountgroup *mg)
-{
-	struct journal *j;
-	struct id_info *ids, *id;
-	int id_count, id_size;
-	int rv, i;
-
-	rv = get_id_list(mg, &ids, &id_count, &id_size);
-	if (rv) {
-		/* all new nodes, no old nodes */
-		log_group(mg, "create_failed_journals all new");
-		return;
-	}
-	print_id_list(mg, ids, id_count, id_size);
-
-	id = ids;
-
-	for (i = 0; i < id_count; i++) {
-		if (!(id->flags & IDI_JID_NEEDS_RECOVERY))
-			goto next;
-
-		j = malloc(sizeof(struct journal));
-		if (!j) {
-			log_error("create_failed_journals no mem");
-			return;
-		}
-		memset(j, 0, sizeof(struct journal));
-
-		j->jid = id->jid;
-		j->needs_recovery = 1;
-		list_add(&j->list, &mg->journals);
-		log_group(mg, "create_failed_journals jid %d", j->jid);
- next:
-		id = (struct id_info *)((char *)id + id_size);
-	}
-}
-
-/* This pattern (for each failed memb in removed list of each change) is
-   repeated and needs to match in four places: here, count_ids(),
-   send_start(), and journals_need_recovery(). */
-
-static void set_failed_journals(struct mountgroup *mg)
-{
-	struct change *cg;
-	struct member *memb;
-	struct journal *j;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(cg, &mg->changes, list) {
-		list_for_each_entry(memb, &cg->removed, list) {
-			if (!memb->failed && !is_withdraw(mg, memb->nodeid))
-				continue;
-			j = find_journal_by_nodeid(mg, memb->nodeid);
-			if (j) {
-				j->needs_recovery = 1;
-				j->failed_nodeid = j->nodeid;
-				j->nodeid = 0;
-				log_group(mg, "set_failed_journals jid %d "
-					  "nodeid %d", j->jid, memb->nodeid);
-			} else {
-				log_group(mg, "set_failed_journals no journal "
-					  "for nodeid %d ", memb->nodeid);
-			}
-		}
-	}
-}
-
-/* returns nodeid of new member with the next highest nodeid */
-
-static int next_new_nodeid(struct mountgroup *mg, int prev)
-{
-	struct change *cg;
-	struct member *memb;
-	int low = 0;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry(memb, &cg->members, list) {
-		if (memb->mg_info->started_count)
-			continue;
-		if (memb->nodeid <= prev)
-			continue;
-		if (!low)
-			low = memb->nodeid;
-		else if (memb->nodeid < low)
-			low = memb->nodeid;
-	}
-
-	return low;
-}
-
-/* returns lowest unused jid */
-
-static int next_free_jid(struct mountgroup *mg)
-{
-	int i;
-
-	for (i = 0; i < MAX_JOURNALS; i++) {
-		if (!find_journal(mg, i))
-			return i;
-	}
-	return -1;
-}
-
-static void create_new_journals(struct mountgroup *mg)
-{
-	struct journal *j, *safe;
-	struct change *cg;
-	struct node *node;
-	int nodeid = 0;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	/* first get rid of journal structs that are no longer used
-	   or dirty, i.e. from nodes that have unmounted/left, or
-	   journals that have been recovered */
-
-	list_for_each_entry_safe(j, safe, &mg->journals, list) {
-		if (j->needs_recovery)
-			continue;
-
-		if (find_memb(cg, j->nodeid))
-			continue;
-
-		list_del(&j->list);
-		free(j);
-	}
-
-	while (1) {
-		nodeid = next_new_nodeid(mg, nodeid);
-		if (!nodeid)
-			break;
-
-		node = get_node_history(mg, nodeid);
-		if (!node) {
-			/* shouldn't happen */
-			log_error("create_new_journals no nodeid %d", nodeid);
-			continue;
-		}
-
-		if (node->spectator)
-			node->jid = JID_NONE;
-		else
-			node->jid = next_free_jid(mg);
-
-		if (node->nodeid == our_nodeid)
-			mg->our_jid = node->jid;
-
-		log_group(mg, "create_new_journals %d gets jid %d",
-			  node->nodeid, node->jid);
-
-		if (node->jid == JID_NONE)
-			continue;
-
-		j = malloc(sizeof(struct journal));
-		if (!j) {
-			log_error("create_new_journals no mem");
-			continue;
-		}
-		memset(j, 0, sizeof(struct journal));
-
-		j->nodeid = nodeid;
-		j->jid = node->jid;
-		list_add(&j->list, &mg->journals);
-	}
-}
-
-/* recovery_result and mount_done messages are saved by new members until
-   they've completed the start cycle and have member state to apply them to.
-   The start messages from old nodes may not reflect the rr/md updates. */
-
-static void apply_saved_messages(struct mountgroup *mg)
-{
-	struct change *cg;
-	struct save_msg *sm, *safe;
-	struct gfs_header *hd;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	list_for_each_entry_safe(sm, safe, &cg->saved_messages, list) {
-		hd = (struct gfs_header *)sm->buf;
-
-		switch (hd->type) {
-		case GFS_MSG_MOUNT_DONE:
-			receive_mount_done(mg, hd, sm->len);
-			break;
-		case GFS_MSG_RECOVERY_RESULT:
-			receive_recovery_result(mg, hd, sm->len);
-			break;
-		}
-
-		list_del(&sm->list);
-		free(sm);
-	}
-}
-
-/* this is run immediately after receiving the final start message in a start
-   cycle, so all nodes will run this in the same sequence wrt other messages
-   and confchgs */
-
-static void sync_state(struct mountgroup *mg)
-{
-	/* This is needed for the case where the first_recovery_done message
-	   arrives while a change/start cycle is in progress.  The
-	   first_recovery data in the start messages (used by new nodes in this
-	   cycle to determine the first_recovery state) may be inconsistent in
-	   different start messages (because nodes sent their start messages at
-	   different times wrt the first_recovery_done message.)  But, in the
-	   case where the new nodes received the first_recovery_done message,
-	   they can just use that and don't need the (possibly inconsistent)
-	   first recovery data in the start messages. */
-
-	if (mg->first_recovery_msg) {
-		if (mg->first_recovery_needed || mg->first_recovery_master) {
-			/* shouldn't happen */
-			log_error("sync_state first_recovery_msg needed %d "
-				  "master %d", mg->first_recovery_needed,
-				  mg->first_recovery_master);
-		}
-
-		log_group(mg, "sync_state first_recovery_msg");
-		goto out;
-	}
-
-	/* This is the path the initial start cycle for the group always
-	   follows.  It's the case where one or more nodes are all starting up
-	   for the first time.  No one has completed a start cycle yet because
-	   everyone is joining, and one node needs to do first recovery. */
-
-	if (all_nodes_new(mg)) {
-		if (mg->first_recovery_needed || mg->first_recovery_master) {
-			/* shouldn't happen */
-			log_error("sync_state all_nodes_new first_recovery "
-				  "needed %d master %d",
-				  mg->first_recovery_needed,
-				  mg->first_recovery_master);
-		}
-		mg->first_recovery_needed = 1;
-		mg->first_recovery_master = pick_first_recovery_master(mg, 1);
-
-		log_group(mg, "sync_state all_nodes_new first_recovery_needed "
-			  "master %d", mg->first_recovery_master);
-		goto out;
-	}
-
-	/* This is for the case where new nodes are added to existing members
-	   that have first_recovery_needed set. */
-
-	if (any_nodes_first_recovery(mg)) {
-		mg->first_recovery_needed = 1;
-		mg->first_recovery_master = pick_first_recovery_master(mg, 0);
-
-		log_group(mg, "sync_state first_recovery_needed master %d",
-			  mg->first_recovery_master);
-		goto out;
-	}
-
-	/* Normal case where nodes join an established group that completed
-	   first recovery sometime in the past.  Existing nodes that weren't
-	   around during first recovery come through here, and new nodes
-	   being added in this cycle come through here. */
-
-	if (mg->first_recovery_needed) {
-		/* shouldn't happen */
-		log_error("sync_state frn should not be set");
-		goto out;
-	}
-
-	log_group(mg, "sync_state");
- out:
-	send_withdraw_acks(mg);
-
-	if (!mg->started_count) {
-		create_old_nodes(mg);
-		create_new_nodes(mg);
-		create_failed_journals(mg);
-		apply_saved_messages(mg);
-		create_new_journals(mg);
-	} else {
-		create_new_nodes(mg);
-		set_failed_journals(mg);
-		create_new_journals(mg);
-	}
-}
-
-static void apply_changes(struct mountgroup *mg)
-{
-	struct change *cg;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	switch (cg->state) {
-
-	case CGST_WAIT_CONDITIONS:
-		if (wait_conditions_done(mg)) {
-			send_start(mg);
-			cg->state = CGST_WAIT_MESSAGES;
-		}
-		break;
-
-	case CGST_WAIT_MESSAGES:
-		if (wait_messages_done(mg)) {
-			sync_state(mg);
-			cleanup_changes(mg);
-		}
-		break;
-
-	default:
-		log_error("apply_changes invalid state %d", cg->state);
-	}
-}
-
-void process_first_mount(struct mountgroup *mg)
-{
-	/*
-	 * Assumption here is that only the first mounter will get
-	 * uevents when first_recovery_needed is set.
-	 */
-
-	/* make a local record of jid and recover_status; we may want
-	   to check below that we've seen uevents for all jids
-	   during first recovery before sending first_recovery_done. */
-
-	log_group(mg, "recovery_uevent mg %s first recovery done", mg->name);
-
-	/* ignore extraneous uevent from others_may_mount */
-	if (mg->first_done_uevent)
-		return;
-
-	log_group(mg, "recovery_uevent first_done");
-	mg->first_done_uevent = 1;
-	send_first_recovery_done(mg);
-
-	apply_changes_recovery(mg);
-}
-
-/* We send messages with the info from kernel uevents or mount.gfs ipc,
-   and then process the uevent/ipc upon receiving the message for it, so
-   that it can be processed in the same order by all nodes. */
-
-void process_recovery_uevent(struct mountgroup *mg, int jid, int recover_status)
-{
-	struct journal *j;
-
-	if (mg->first_recovery_needed)
-		return;
-
-	if (!mg->local_recovery_busy) {
-		/* This will happen in two known situations:
-		   - we get a recovery_done uevent for our own journal
-		     when we mount  (jid == mg->our_jid)
-		   - the first mounter can read first_done and clear
-		     first_recovery_needed before seeing the change
-		     uevent from others_may_mount */
-		log_group(mg, "recovery_uevent jid %d ignore", jid);
-		return;
-	}
-
-	mg->local_recovery_busy = 0;
-
-	if (mg->local_recovery_jid != jid) {
-		log_error("recovery_uevent jid %d expected %d", jid,
-			  mg->local_recovery_jid);
-		return;
-	}
-
-	j = find_journal(mg, jid);
-	if (!j) {
-		log_error("recovery_uevent no journal %d", jid);
-		return;
-	}
-
-	log_group(mg, "recovery_uevent jid %d status %d "
-		  "local_recovery_done %d needs_recovery %d",
-		  jid, recover_status, j->local_recovery_done,
-		  j->needs_recovery);
-
-	j->local_recovery_done = 1;
-	j->local_recovery_result = recover_status;
-
-	/* j->needs_recovery will be cleared when we receive this
-	   recovery_result message.  if it's already set, then
-	   someone else has completed the recovery and there's
-	   no need to send our result */
-
-	if (j->needs_recovery)
-		send_recovery_result(mg, jid, recover_status);
-
-	apply_changes_recovery(mg);
-}
-
-static void start_journal_recovery(struct mountgroup *mg, int jid)
-{
-	int rv;
-
-	log_group(mg, "start_journal_recovery jid %d", jid);
-
-	rv = set_sysfs(mg, "recover", jid);
-	if (rv < 0) {
-		log_error("start_journal_recovery %d error %d", jid, rv);
-		return;
-	}
-
-	mg->local_recovery_busy = 1;
-	mg->local_recovery_jid = jid;
-}
-
-static int wait_recoveries_done(struct mountgroup *mg)
-{
-	struct journal *j;
-	int wait_count = 0;
-
-	list_for_each_entry(j, &mg->journals, list) {
-		if (j->needs_recovery) {
-			log_group(mg, "wait_recoveries jid %d nodeid %d "
-				  "unrecovered", j->jid, j->failed_nodeid);
-			wait_count++;
-		}
-	}
-
-	if (wait_count)
-		return 0;
-
-	log_group(mg, "wait_recoveries done");
-	return 1;
-}
-
-/* pick a jid that has not been successfully recovered by someone else
-   (received recovery_result success message) and hasn't been recovered
-   by us (local record); if nothing to recover, return 0 */
-
-static int pick_journal_to_recover(struct mountgroup *mg, int *jid)
-{
-	struct journal *j;
-
-	list_for_each_entry(j, &mg->journals, list) {
-		if (j->needs_recovery && !j->local_recovery_done) {
-			*jid = j->jid;
-			return 1;
-		}
-	}
-
-#if 0
-	/* FIXME: do something so this doesn't happen so regularly; maybe
-	 * retry only after all nodes have failed. This code doesn't work
-	 * but shows an idea of roughly how to fix the issue.
-	 */
-
-	/* Retry recoveries that failed the first time.  This is necessary
-	   at times for withrawn journals when all nodes fail the recovery
-	   (fail to get journal lock) before the withdrawing node has had a
-	   chance to clear its dlm locks for the withdrawn journal.
-	   32 max retries is random, and includes attempts by all nodes. */
-
-	list_for_each_entry(j, &mg->journals, list) {
-		if (j->needs_recovery && j->local_recovery_done &&
-		    (j->local_recovery_result == LM_RD_GAVEUP) &&
-		    (j->failed_recovery_count > 1) &&
-		    (j->failed_recovery_count < 32)) {
-			log_group(mg, "retrying jid %d recovery", j->jid);
-			*jid = j->jid;
-			sleep(1); /* might this cause problems? */
-			return 1;
-		}
-	}
-#endif
-
-	return 0;
-}
-
-/* processing that happens after all changes have been dealt with */
-
-static void apply_recovery(struct mountgroup *mg)
-{
-	int jid;
-
-	if (mg->first_recovery_needed) {
-		if (mg->first_recovery_master == our_nodeid &&
-		    !mg->mount_client_notified) {
-			log_group(mg, "apply_recovery first start_kernel");
-			mg->first_mounter = 1; /* adds first=1 to hostdata */
-			start_kernel(mg);      /* includes reply to mount.gfs */
-		}
-		return;
-	}
-
-	/* The normal non-first-recovery mode.  When a recovery_done message
-	   is received, check whether any more journals need recovery.  If
-	   so, start recovery on the next one, if not, start the kernel. */
-
-	if (!wait_recoveries_done(mg)) {
-		if (!mg->kernel_mount_done || mg->kernel_mount_error)
-			return;
-		if (mg->spectator)
-			return;
-		if (mg->local_recovery_busy)
-			return;
-		if (pick_journal_to_recover(mg, &jid))
-			start_journal_recovery(mg, jid);
-	} else {
-		if (!mg->kernel_stopped)
-			return;
-		log_group(mg, "apply_recovery start_kernel");
-		start_kernel(mg);
-	}
-}
-
-static void apply_changes_recovery(struct mountgroup *mg)
-{
-	if (!list_empty(&mg->changes))
-		apply_changes(mg);
-	
-	if (mg->started_change && list_empty(&mg->changes))
-		apply_recovery(mg);
-}
-
-void process_mountgroups(void)
-{
-	struct mountgroup *mg, *safe;
-
-	list_for_each_entry_safe(mg, safe, &mountgroups, list)
-		apply_changes_recovery(mg);
-}
-
-static int add_change(struct mountgroup *mg,
-		      const struct cpg_address *member_list,
-		      size_t member_list_entries,
-		      const struct cpg_address *left_list,
-		      size_t left_list_entries,
-		      const struct cpg_address *joined_list,
-		      size_t joined_list_entries,
-		      struct change **cg_out)
-{
-	struct change *cg;
-	struct member *memb;
-	int i, error;
-
-	cg = malloc(sizeof(struct change));
-	if (!cg)
-		goto fail_nomem;
-	memset(cg, 0, sizeof(struct change));
-	INIT_LIST_HEAD(&cg->members);
-	INIT_LIST_HEAD(&cg->removed);
-	INIT_LIST_HEAD(&cg->saved_messages);
-	cg->state = CGST_WAIT_CONDITIONS;
-	cg->create_time = time(NULL);
-	cg->seq = ++mg->change_seq;
-	if (!cg->seq)
-		cg->seq = ++mg->change_seq;
-
-	cg->member_count = member_list_entries;
-	cg->joined_count = joined_list_entries;
-	cg->remove_count = left_list_entries;
-
-	for (i = 0; i < member_list_entries; i++) {
-		memb = malloc(sizeof(struct member));
-		if (!memb)
-			goto fail_nomem;
-		memset(memb, 0, sizeof(struct member));
-		memb->nodeid = member_list[i].nodeid;
-		list_add_tail(&memb->list, &cg->members);
-	}
-
-	for (i = 0; i < left_list_entries; i++) {
-		memb = malloc(sizeof(struct member));
-		if (!memb)
-			goto fail_nomem;
-		memset(memb, 0, sizeof(struct member));
-		memb->nodeid = left_list[i].nodeid;
-		if (left_list[i].reason == CPG_REASON_NODEDOWN ||
-		    left_list[i].reason == CPG_REASON_PROCDOWN) {
-			memb->failed = 1;
-			cg->failed_count++;
-		}
-		list_add_tail(&memb->list, &cg->removed);
-
-		if (memb->failed)
-			node_history_fail(mg, memb->nodeid, cg,
-					  left_list[i].reason);
-		else
-			node_history_left(mg, memb->nodeid, cg);
-
-		log_group(mg, "add_change cg %u remove nodeid %d reason %d",
-			  cg->seq, memb->nodeid, left_list[i].reason);
-
-		if (left_list[i].reason == CPG_REASON_PROCDOWN)
-			kick_node_from_cluster(memb->nodeid);
-	}
-
-	for (i = 0; i < joined_list_entries; i++) {
-		memb = find_memb(cg, joined_list[i].nodeid);
-		if (!memb) {
-			log_error("no member %d", joined_list[i].nodeid);
-			error = -ENOENT;
-			goto fail;
-		}
-		memb->added = 1;
-
-		if (memb->nodeid == our_nodeid)
-			cg->we_joined = 1;
-		else
-			node_history_init(mg, memb->nodeid, cg);
-
-		log_group(mg, "add_change cg %u joined nodeid %d", cg->seq,
-			  memb->nodeid);
-	}
-
-	if (cg->we_joined) {
-		log_group(mg, "add_change cg %u we joined", cg->seq);
-		list_for_each_entry(memb, &cg->members, list)
-			node_history_init(mg, memb->nodeid, cg);
-	}
-
-	log_group(mg, "add_change cg %u counts member %d joined %d remove %d "
-		  "failed %d", cg->seq, cg->member_count, cg->joined_count,
-		  cg->remove_count, cg->failed_count);
-
-	list_add(&cg->list, &mg->changes);
-	*cg_out = cg;
-	return 0;
-
- fail_nomem:
-	log_error("no memory");
-	error = -ENOMEM;
- fail:
-	free_cg(cg);
-	return error;
-}
-
-static int we_left(const struct cpg_address *left_list,
-		   size_t left_list_entries)
-{
-	int i;
-
-	for (i = 0; i < left_list_entries; i++) {
-		if (left_list[i].nodeid == our_nodeid)
-			return 1;
-	}
-	return 0;
-}
-
-static void confchg_cb(cpg_handle_t handle,
-		       const struct cpg_name *group_name,
-		       const struct cpg_address *member_list,
-		       size_t member_list_entries,
-		       const struct cpg_address *left_list,
-		       size_t left_list_entries,
-		       const struct cpg_address *joined_list,
-		       size_t joined_list_entries)
-{
-	struct mountgroup *mg;
-	struct change *cg;
-	int rv;
-
-	log_config(group_name, member_list, member_list_entries,
-		   left_list, left_list_entries,
-		   joined_list, joined_list_entries);
-
-	mg = find_mg_handle(handle);
-	if (!mg) {
-		log_error("confchg_cb no mountgroup for cpg %s",
-			  group_name->value);
-		return;
-	}
-
-	if (mg->leaving && we_left(left_list, left_list_entries)) {
-		/* we called cpg_leave(), and this should be the final
-		   cpg callback we receive */
-		log_group(mg, "confchg for our leave");
-		dlmc_fs_unregister(dlmcontrol_fd, mg->name);
-		cpg_finalize(mg->cpg_handle);
-		client_dead(mg->cpg_client);
-		list_del(&mg->list);
-		if (!mg->withdraw_uevent) {
-			free_mg(mg);
-		} else {
-			if (!member_list_entries) {
-				/* no one remaining to send us an ack */
-				set_sysfs(mg, "withdraw", 1);
-				free_mg(mg);
-			} else {
-				/* set the sysfs withdraw file and free the mg
-				   when the ack arrives */
-				list_add(&mg->list, &withdrawn_mounts);
-			}
-		}
-		return;
-	}
-
-	rv = add_change(mg, member_list, member_list_entries,
-			left_list, left_list_entries,
-			joined_list, joined_list_entries, &cg);
-	if (rv)
-		return;
-
-	apply_changes_recovery(mg);
-}
-
-static void gfs_header_in(struct gfs_header *hd)
-{
-	hd->version[0]  = le16_to_cpu(hd->version[0]);
-	hd->version[1]  = le16_to_cpu(hd->version[1]);
-	hd->version[2]  = le16_to_cpu(hd->version[2]);
-	hd->type        = le16_to_cpu(hd->type);
-	hd->nodeid      = le32_to_cpu(hd->nodeid);
-	hd->to_nodeid   = le32_to_cpu(hd->to_nodeid);
-	hd->global_id   = le32_to_cpu(hd->global_id);
-	hd->flags       = le32_to_cpu(hd->flags);
-	hd->msgdata     = le32_to_cpu(hd->msgdata);
-}
-
-static int gfs_header_check(struct gfs_header *hd, int nodeid)
-{
-	if (hd->version[0] != our_protocol.daemon_run[0] ||
-	    hd->version[1] != our_protocol.daemon_run[1]) {
-		log_error("reject message from %d version %u.%u.%u vs %u.%u.%u",
-			  nodeid, hd->version[0], hd->version[1],
-			  hd->version[2], our_protocol.daemon_run[0],
-			  our_protocol.daemon_run[1],
-			  our_protocol.daemon_run[2]);
-		return -1;
-	}
-
-	if (hd->nodeid != nodeid) {
-		log_error("bad message nodeid %d %d", hd->nodeid, nodeid);
-		return -1;
-	}
-
-	return 0;
-}
-
-static void deliver_cb(cpg_handle_t handle,
-		       const struct cpg_name *group_name,
-		       uint32_t nodeid, uint32_t pid,
-		       void *data, size_t len)
-{
-	struct mountgroup *mg;
-	struct gfs_header *hd;
-
-	mg = find_mg_handle(handle);
-	if (!mg) {
-		log_error("deliver_cb no mg for cpg %s", group_name->value);
-		return;
-	}
-
-	if (len < sizeof(*hd)) {
-		log_error("deliver_cb short message %zd", len);
-		return;
-	}
-
-	hd = (struct gfs_header *)data;
-	gfs_header_in(hd);
-
-	if (gfs_header_check(hd, nodeid) < 0)
-		return;
-
-	switch (hd->type) {
-	case GFS_MSG_START:
-		receive_start(mg, hd, len);
-		break;
-	case GFS_MSG_MOUNT_DONE:
-		if (!mg->started_count)
-			save_message(mg, hd, len);
-		else
-			receive_mount_done(mg, hd, len);
-		break;
-	case GFS_MSG_FIRST_RECOVERY_DONE:
-		receive_first_recovery_done(mg, hd, len);
-		break;
-	case GFS_MSG_RECOVERY_RESULT:
-		if (!mg->started_count)
-			save_message(mg, hd, len);
-		else
-			receive_recovery_result(mg, hd, len);
-		break;
-	case GFS_MSG_REMOUNT:
-		receive_remount(mg, hd, len);
-		break;
-	case GFS_MSG_WITHDRAW:
-		receive_withdraw(mg, hd, len);
-		break;
-	default:
-		log_error("unknown msg type %d", hd->type);
-	}
-
-	apply_changes_recovery(mg);
-}
-
-static cpg_callbacks_t cpg_callbacks = {
-	.cpg_deliver_fn = deliver_cb,
-	.cpg_confchg_fn = confchg_cb,
-};
-
-static void process_cpg_mountgroup(int ci)
-{
-	struct mountgroup *mg;
-	cpg_error_t error;
-
-	mg = find_mg_ci(ci);
-	if (!mg) {
-		log_error("process_mountgroup_cpg no mountgroup for ci %d", ci);
-		return;
-	}
-
-	error = cpg_dispatch(mg->cpg_handle, CPG_DISPATCH_ALL);
-	if (error != CPG_OK) {
-		log_error("cpg_dispatch error %d", error);
-		return;
-	}
-
-	update_flow_control_status();
-}
-
-int gfs_join_mountgroup(struct mountgroup *mg)
-{
-	cpg_error_t error;
-	cpg_handle_t h;
-	struct cpg_name name;
-	int i = 0, fd, ci, rv;
-
-	/* I think this registration with dlm_controld could be done
-	   just about anywhere before we do the mount(2). */
-	rv = dlmc_fs_register(dlmcontrol_fd, mg->name);
-	if (rv) {
-		log_error("dlmc_fs_register failed %d", rv);
-		return rv;
-	}
-
-	error = cpg_initialize(&h, &cpg_callbacks);
-	if (error != CPG_OK) {
-		log_error("cpg_initialize error %d", error);
-		goto fail;
-	}
-
-	cpg_fd_get(h, &fd);
-
-	ci = client_add(fd, process_cpg_mountgroup, NULL);
-
-	mg->cpg_handle = h;
-	mg->cpg_client = ci;
-	mg->cpg_fd = fd;
-	mg->kernel_stopped = 1;
-	mg->joining = 1;
-
-	memset(&name, 0, sizeof(name));
-	sprintf(name.value, "gfs:mount:%s", mg->name);
-	name.length = strlen(name.value) + 1;
-
-	/* TODO: allow global_id to be set in cluster.conf? */
-	mg->id = cpgname_to_crc(name.value, name.length);
-
- retry:
-	error = cpg_join(h, &name);
-	if (error == CPG_ERR_TRY_AGAIN) {
-		sleep(1);
-		if (!(++i % 10))
-			log_error("cpg_join error retrying");
-		goto retry;
-	}
-	if (error != CPG_OK) {
-		log_error("cpg_join error %d", error);
-		cpg_finalize(h);
-		goto fail_client;
-	}
-
-	return 0;
-
- fail_client:
-	client_dead(ci);
-	cpg_finalize(h);
- fail:
-	dlmc_fs_unregister(dlmcontrol_fd, mg->name);
-	return -ENOTCONN;
-}
-
-/* If mount(2) fails, we'll often get two leaves, one from seeing the remove
-   uevent, and the other from mount.gfs.  I suspect they could arrive in either
-   order.  We can just ignore the second.  The second would either not find
-   the mg here, or would see mg->leaving of 1 from the first. */
-
-void gfs_leave_mountgroup(struct mountgroup *mg, int mnterr)
-{
-	cpg_error_t error;
-	struct cpg_name name;
-	int i = 0;
-
-	if (mg->leaving) {
-		log_group(mg, "leave: already leaving");
-		return;
-	}
-	mg->leaving = 1;
-
-	memset(&name, 0, sizeof(name));
-	sprintf(name.value, "gfs:mount:%s", mg->name);
-	name.length = strlen(name.value) + 1;
-
- retry:
-	error = cpg_leave(mg->cpg_handle, &name);
-	if (error == CPG_ERR_TRY_AGAIN) {
-		sleep(1);
-		if (!(++i % 10))
-			log_error("cpg_leave error retrying");
-		goto retry;
-	}
-	if (error != CPG_OK)
-		log_error("cpg_leave error %d", error);
-}
-
-void do_leave(struct mountgroup *mg, int mnterr)
-{
-	log_debug("do_leave %s mnterr %d", mg->name, mnterr);
-
-	if (mg->withdraw_uevent) {
-		log_group(mg, "do_leave: ignored during withdraw");
-		return;
-	}
-
-	gfs_leave_mountgroup(mg, mnterr);
-}
-
-static void receive_withdraw_ack(struct gfs_header *hd, int len)
-{
-	struct mountgroup *mg;
-
-	if (hd->to_nodeid != our_nodeid)
-		return;
-
-	log_debug("receive_withdraw_ack from %d global_id %x",
-		  hd->nodeid, hd->global_id);
-
-	list_for_each_entry(mg, &withdrawn_mounts, list) {
-		if (mg->id != hd->global_id)
-			continue;
-		set_sysfs(mg, "withdraw", 1);
-		list_del(&mg->list);
-		free_mg(mg);
-		break;
-	}
-}
-
-static void send_withdraw_ack(struct mountgroup *mg, int nodeid)
-{
-	struct gfs_header h;
-
-	memset(&h, 0, sizeof(h));
-
-	h.version[0]	= cpu_to_le16(our_protocol.daemon_run[0]);
-	h.version[1]	= cpu_to_le16(our_protocol.daemon_run[1]);
-	h.version[2]	= cpu_to_le16(our_protocol.daemon_run[2]);
-	h.type		= cpu_to_le16(GFS_MSG_WITHDRAW_ACK);
-	h.nodeid	= cpu_to_le32(our_nodeid);
-	h.to_nodeid	= cpu_to_le32(nodeid);
-	h.global_id	= cpu_to_le32(mg->id);
-
-	_send_message(cpg_handle_daemon, (char *)&h, sizeof(h),
-		      GFS_MSG_WITHDRAW_ACK);
-}
-
-/* Everyone remaining in the group will send an ack for the withdrawn fs;
-   all but the first will be ignored. */
-
-static void send_withdraw_acks(struct mountgroup *mg)
-{
-	struct node *node;
-
-	list_for_each_entry(node, &mg->node_history, list) {
-		if (node->withdraw && !node->send_withdraw_ack) {
-			send_withdraw_ack(mg, node->nodeid);
-			node->send_withdraw_ack = 1;
-		}
-	}
-}
-
-static struct node *get_node_daemon(int nodeid)
-{
-	struct node *node;
-
-	list_for_each_entry(node, &daemon_nodes, list) {
-		if (node->nodeid == nodeid)
-			return node;
-	}
-	return NULL;
-}
-
-static void add_node_daemon(int nodeid)
-{
-	struct node *node;
-
-	if (get_node_daemon(nodeid))
-		return;
-
-	node = malloc(sizeof(struct node));
-	if (!node) {
-		log_error("add_node_daemon no mem");
-		return;
-	}
-	memset(node, 0, sizeof(struct node));
-	node->nodeid = nodeid;
-	list_add_tail(&node->list, &daemon_nodes);
-}
-
-static void pv_in(struct protocol_version *pv)
-{
-	pv->major = le16_to_cpu(pv->major);
-	pv->minor = le16_to_cpu(pv->minor);
-	pv->patch = le16_to_cpu(pv->patch);
-	pv->flags = le16_to_cpu(pv->flags);
-}
-
-static void pv_out(struct protocol_version *pv)
-{
-	pv->major = cpu_to_le16(pv->major);
-	pv->minor = cpu_to_le16(pv->minor);
-	pv->patch = cpu_to_le16(pv->patch);
-	pv->flags = cpu_to_le16(pv->flags);
-}
-
-static void protocol_in(struct protocol *proto)
-{
-	pv_in(&proto->dm_ver);
-	pv_in(&proto->km_ver);
-	pv_in(&proto->dr_ver);
-	pv_in(&proto->kr_ver);
-}
-
-static void protocol_out(struct protocol *proto)
-{
-	pv_out(&proto->dm_ver);
-	pv_out(&proto->km_ver);
-	pv_out(&proto->dr_ver);
-	pv_out(&proto->kr_ver);
-}
-
-/* go through member list saved in last confchg, see if we have received a
-   proto message from each */
-
-static int all_protocol_messages(void)
-{
-	struct node *node;
-	int i;
-
-	if (!daemon_member_count)
-		return 0;
-
-	for (i = 0; i < daemon_member_count; i++) {
-		node = get_node_daemon(daemon_member[i].nodeid);
-		if (!node) {
-			log_error("all_protocol_messages no node %d",
-				  daemon_member[i].nodeid);
-			return 0;
-		}
-
-		if (!node->proto.daemon_max[0])
-			return 0;
-	}
-	return 1;
-}
-
-static int pick_min_protocol(struct protocol *proto)
-{
-	uint16_t mind[4];
-	uint16_t mink[4];
-	struct node *node;
-	int i;
-
-	memset(&mind, 0, sizeof(mind));
-	memset(&mink, 0, sizeof(mink));
-
-	/* first choose the minimum major */
-
-	for (i = 0; i < daemon_member_count; i++) {
-		node = get_node_daemon(daemon_member[i].nodeid);
-		if (!node) {
-			log_error("pick_min_protocol no node %d",
-				  daemon_member[i].nodeid);
-			return -1;
-		}
-
-		if (!mind[0] || node->proto.daemon_max[0] < mind[0])
-			mind[0] = node->proto.daemon_max[0];
-
-		if (!mink[0] || node->proto.kernel_max[0] < mink[0])
-			mink[0] = node->proto.kernel_max[0];
-	}
-
-	if (!mind[0] || !mink[0]) {
-		log_error("pick_min_protocol zero major number");
-		return -1;
-	}
-
-	/* second pick the minimum minor with the chosen major */
-
-	for (i = 0; i < daemon_member_count; i++) {
-		node = get_node_daemon(daemon_member[i].nodeid);
-		if (!node)
-			continue;
-
-		if (mind[0] == node->proto.daemon_max[0]) {
-			if (!mind[1] || node->proto.daemon_max[1] < mind[1])
-				mind[1] = node->proto.daemon_max[1];
-		}
-
-		if (mink[0] == node->proto.kernel_max[0]) {
-			if (!mink[1] || node->proto.kernel_max[1] < mink[1])
-				mink[1] = node->proto.kernel_max[1];
-		}
-	}
-
-	if (!mind[1] || !mink[1]) {
-		log_error("pick_min_protocol zero minor number");
-		return -1;
-	}
-
-	/* third pick the minimum patch with the chosen major.minor */
-
-	for (i = 0; i < daemon_member_count; i++) {
-		node = get_node_daemon(daemon_member[i].nodeid);
-		if (!node)
-			continue;
-
-		if (mind[0] == node->proto.daemon_max[0] &&
-		    mind[1] == node->proto.daemon_max[1]) {
-			if (!mind[2] || node->proto.daemon_max[2] < mind[2])
-				mind[2] = node->proto.daemon_max[2];
-		}
-
-		if (mink[0] == node->proto.kernel_max[0] &&
-		    mink[1] == node->proto.kernel_max[1]) {
-			if (!mink[2] || node->proto.kernel_max[2] < mink[2])
-				mink[2] = node->proto.kernel_max[2];
-		}
-	}
-
-	if (!mind[2] || !mink[2]) {
-		log_error("pick_min_protocol zero patch number");
-		return -1;
-	}
-
-	memcpy(&proto->daemon_run, &mind, sizeof(mind));
-	memcpy(&proto->kernel_run, &mink, sizeof(mink));
-	return 0;
-}
-
-static void receive_protocol(struct gfs_header *hd, int len)
-{
-	struct protocol *p;
-	struct node *node;
-
-	p = (struct protocol *)((char *)hd + sizeof(struct gfs_header));
-	protocol_in(p);
-
-	if (len < sizeof(struct gfs_header) + sizeof(struct protocol)) {
-		log_error("receive_protocol invalid len %d from %d",
-			  len, hd->nodeid);
-		return;
-	}
-
-	/* zero is an invalid version value */
-
-	if (!p->daemon_max[0] || !p->daemon_max[1] || !p->daemon_max[2] ||
-	    !p->kernel_max[0] || !p->kernel_max[1] || !p->kernel_max[2]) {
-		log_error("receive_protocol invalid max value from %d "
-			  "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
-			  p->daemon_max[0], p->daemon_max[1], p->daemon_max[2],
-			  p->kernel_max[0], p->kernel_max[1], p->kernel_max[2]);
-		return;
-	}
-
-	/* the run values will be zero until a version is set, after
-	   which none of the run values can be zero */
-
-	if (p->daemon_run[0] && (!p->daemon_run[1] || !p->daemon_run[2] ||
-	    !p->kernel_run[0] || !p->kernel_run[1] || !p->kernel_run[2])) {
-		log_error("receive_protocol invalid run value from %d "
-			  "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
-			  p->daemon_run[0], p->daemon_run[1], p->daemon_run[2],
-			  p->kernel_run[0], p->kernel_run[1], p->kernel_run[2]);
-		return;
-	}
-
-	/* if we have zero run values, and this msg has non-zero run values,
-	   then adopt them as ours; otherwise save this proto message */
-
-	if (our_protocol.daemon_run[0])
-		return;
-
-	if (p->daemon_run[0]) {
-		memcpy(&our_protocol.daemon_run, &p->daemon_run,
-		       sizeof(struct protocol_version));
-		memcpy(&our_protocol.kernel_run, &p->kernel_run,
-		       sizeof(struct protocol_version));
-		log_debug("run protocol from nodeid %d", hd->nodeid);
-		return;
-	}
-
-	/* save this node's proto so we can tell when we've got all, and
-	   use it to select a minimum protocol from all */
-
-	node = get_node_daemon(hd->nodeid);
-	if (!node) {
-		log_error("receive_protocol no node %d", hd->nodeid);
-		return;
-	}
-	memcpy(&node->proto, p, sizeof(struct protocol));
-}
-
-static void send_protocol(struct protocol *proto)
-{
-	struct gfs_header *hd;
-	struct protocol *pr;
-	char *buf;
-	int len;
-
-	len = sizeof(struct gfs_header) + sizeof(struct protocol);
-	buf = malloc(len);
-	if (!buf) {
-		log_error("send_protocol no mem %d", len);
-		return;
-	}
-	memset(buf, 0, len);
-
-	hd = (struct gfs_header *)buf;
-	pr = (struct protocol *)(buf + sizeof(*hd));
-
-	hd->type = cpu_to_le16(GFS_MSG_PROTOCOL);
-	hd->nodeid = cpu_to_le32(our_nodeid);
-
-	memcpy(pr, proto, sizeof(struct protocol));
-	protocol_out(pr);
-
-	_send_message(cpg_handle_daemon, buf, len, GFS_MSG_PROTOCOL);
-}
-
-int set_protocol(void)
-{
-	struct protocol proto;
-	struct pollfd pollfd;
-	int sent_proposal = 0;
-	int rv;
-
-	memset(&pollfd, 0, sizeof(pollfd));
-	pollfd.fd = cpg_fd_daemon;
-	pollfd.events = POLLIN;
-
-	while (1) {
-		if (our_protocol.daemon_run[0])
-			break;
-
-		if (!sent_proposal && all_protocol_messages()) {
-			/* propose a protocol; look through info from all
-			   nodes and pick the min for both daemon and kernel,
-			   and propose that */
-
-			sent_proposal = 1;
-
-			/* copy our max values */
-			memcpy(&proto, &our_protocol, sizeof(struct protocol));
-
-			rv = pick_min_protocol(&proto);
-			if (rv < 0)
-				return rv;
-
-			log_debug("set_protocol member_count %d propose "
-				  "daemon %u.%u.%u kernel %u.%u.%u",
-				  daemon_member_count,
-				  proto.daemon_run[0], proto.daemon_run[1],
-				  proto.daemon_run[2], proto.kernel_run[0],
-				  proto.kernel_run[1], proto.kernel_run[2]);
-
-			send_protocol(&proto);
-		}
-
-		/* only process messages/events from daemon cpg until protocol
-		   is established */
-
-		rv = poll(&pollfd, 1, -1);
-		if (rv == -1 && errno == EINTR) {
-			if (daemon_quit)
-				return -1;
-			continue;
-		}
-		if (rv < 0) {
-			log_error("set_protocol poll errno %d", errno);
-			return -1;
-		}
-
-		if (pollfd.revents & POLLIN)
-			process_cpg_daemon(0);
-		if (pollfd.revents & (POLLERR | POLLHUP | POLLNVAL)) {
-			log_error("set_protocol poll revents %u",
-				  pollfd.revents);
-			return -1;
-		}
-	}
-
-	if (our_protocol.daemon_run[0] != our_protocol.daemon_max[0] ||
-	    our_protocol.daemon_run[1] > our_protocol.daemon_max[1]) {
-		log_error("incompatible daemon protocol run %u.%u.%u max %u.%u.%u",
-			our_protocol.daemon_run[0],
-			our_protocol.daemon_run[1],
-			our_protocol.daemon_run[2],
-			our_protocol.daemon_max[0],
-			our_protocol.daemon_max[1],
-			our_protocol.daemon_max[2]);
-		return -1;
-	}
-
-	if (our_protocol.kernel_run[0] != our_protocol.kernel_max[0] ||
-	    our_protocol.kernel_run[1] > our_protocol.kernel_max[1]) {
-		log_error("incompatible kernel protocol run %u.%u.%u max %u.%u.%u",
-			our_protocol.kernel_run[0],
-			our_protocol.kernel_run[1],
-			our_protocol.kernel_run[2],
-			our_protocol.kernel_max[0],
-			our_protocol.kernel_max[1],
-			our_protocol.kernel_max[2]);
-		return -1;
-	}
-
-	log_debug("daemon run %u.%u.%u max %u.%u.%u "
-		  "kernel run %u.%u.%u max %u.%u.%u",
-		  our_protocol.daemon_run[0],
-		  our_protocol.daemon_run[1],
-		  our_protocol.daemon_run[2],
-		  our_protocol.daemon_max[0],
-		  our_protocol.daemon_max[1],
-		  our_protocol.daemon_max[2],
-		  our_protocol.kernel_run[0],
-		  our_protocol.kernel_run[1],
-		  our_protocol.kernel_run[2],
-		  our_protocol.kernel_max[0],
-		  our_protocol.kernel_max[1],
-		  our_protocol.kernel_max[2]);
-
-	send_protocol(&our_protocol);
-	return 0;
-}
-
-static void deliver_cb_daemon(cpg_handle_t handle,
-			      const struct cpg_name *group_name,
-			      uint32_t nodeid, uint32_t pid,
-			      void *data, size_t len)
-{
-	struct gfs_header *hd;
-
-	if (len < sizeof(*hd)) {
-		log_error("deliver_cb short message %zd", len);
-		return;
-	}
-
-	hd = (struct gfs_header *)data;
-	gfs_header_in(hd);
-
-	switch (hd->type) {
-	case GFS_MSG_PROTOCOL:
-		receive_protocol(hd, len);
-		break;
-	case GFS_MSG_WITHDRAW_ACK:
-		if (gfs_header_check(hd, nodeid) < 0)
-			return;
-		receive_withdraw_ack(hd, len);
-		break;
-	default:
-		log_error("deliver_cb_daemon unknown msg type %d", hd->type);
-	}
-}
-
-static void confchg_cb_daemon(cpg_handle_t handle,
-			      const struct cpg_name *group_name,
-			      const struct cpg_address *member_list,
-			      size_t member_list_entries,
-			      const struct cpg_address *left_list,
-			      size_t left_list_entries,
-			      const struct cpg_address *joined_list,
-			      size_t joined_list_entries)
-{
-	int i;
-
-	log_config(group_name, member_list, member_list_entries,
-		   left_list, left_list_entries,
-		   joined_list, joined_list_entries);
-
-	if (joined_list_entries)
-		send_protocol(&our_protocol);
-
-	memset(&daemon_member, 0, sizeof(daemon_member));
-	daemon_member_count = member_list_entries;
-
-	for (i = 0; i < member_list_entries; i++) {
-		daemon_member[i] = member_list[i];
-		add_node_daemon(member_list[i].nodeid);
-	}
-}
-
-static cpg_callbacks_t cpg_callbacks_daemon = {
-	.cpg_deliver_fn = deliver_cb_daemon,
-	.cpg_confchg_fn = confchg_cb_daemon,
-};
-
-void process_cpg_daemon(int ci)
-{
-	cpg_error_t error;
-
-	error = cpg_dispatch(cpg_handle_daemon, CPG_DISPATCH_ALL);
-	if (error != CPG_OK)
-		log_error("daemon cpg_dispatch error %d", error);
-}
-
-int setup_cpg_daemon(void)
-{
-	cpg_error_t error;
-	cpg_handle_t h;
-	struct cpg_name name;
-	int i = 0;
-
-	INIT_LIST_HEAD(&daemon_nodes);
-
-	memset(&our_protocol, 0, sizeof(our_protocol));
-	our_protocol.daemon_max[0] = 1;
-	our_protocol.daemon_max[1] = 1;
-	our_protocol.daemon_max[2] = 1;
-	our_protocol.kernel_max[0] = 1;
-	our_protocol.kernel_max[1] = 1;
-	our_protocol.kernel_max[2] = 1;
-
-	error = cpg_initialize(&h, &cpg_callbacks_daemon);
-	if (error != CPG_OK) {
-		log_error("daemon cpg_initialize error %d", error);
-		return -1;
-	}
-
-	cpg_fd_get(h, &cpg_fd_daemon);
-
-	cpg_handle_daemon = h;
-
-	memset(&name, 0, sizeof(name));
-	sprintf(name.value, "gfs:controld");
-	name.length = strlen(name.value) + 1;
-
- retry:
-	error = cpg_join(h, &name);
-	if (error == CPG_ERR_TRY_AGAIN) {
-		sleep(1);
-		if (!(++i % 10))
-			log_error("daemon cpg_join error retrying");
-		goto retry;
-	}
-	if (error != CPG_OK) {
-		log_error("daemon cpg_join error %d", error);
-		goto fail;
-	}
-
-	log_debug("setup_cpg_daemon %d", cpg_fd_daemon);
-	return cpg_fd_daemon;
-
- fail:
-	cpg_finalize(h);
-	return -1;
-}
-
-void close_cpg_daemon(void)
-{
-	struct mountgroup *mg;
-	cpg_error_t error;
-	struct cpg_name name;
-	int i = 0;
-
-	if (!cpg_handle_daemon)
-		return;
-	if (cluster_down)
-		goto fin;
-
-	memset(&name, 0, sizeof(name));
-	sprintf(name.value, "gfs:controld");
-	name.length = strlen(name.value) + 1;
-
- retry:
-	error = cpg_leave(cpg_handle_daemon, &name);
-	if (error == CPG_ERR_TRY_AGAIN) {
-		sleep(1);
-		if (!(++i % 10))
-			log_error("daemon cpg_leave error retrying");
-		goto retry;
-	}
-	if (error != CPG_OK)
-		log_error("daemon cpg_leave error %d", error);
- fin:
-	list_for_each_entry(mg, &mountgroups, list) {
-		if (mg->cpg_handle)
-			cpg_finalize(mg->cpg_handle);
-	}
-	cpg_finalize(cpg_handle_daemon);
-}
-
-int setup_dlmcontrol(void)
-{
-	int fd;
-
-	fd = dlmc_fs_connect();
-	if (fd < 0)
-		log_error("cannot connect to dlm_controld %d", fd);
-	else
-		dlmcontrol_fd = fd;
-
-	return fd;
-}
-
-int set_mountgroup_info(struct mountgroup *mg, struct gfsc_mountgroup *out)
-{
-	struct change *cg, *last = NULL;
-
-	strncpy(out->name, mg->name, GFS_MOUNTGROUP_LEN);
-	out->global_id = mg->id;
-
-	if (mg->joining)
-		out->flags |= GFSC_MF_JOINING;
-	if (mg->leaving)
-		out->flags |= GFSC_MF_LEAVING;
-	if (mg->kernel_stopped)
-		out->flags |= GFSC_MF_KERNEL_STOPPED;
-	if (mg->kernel_mount_done)
-		out->flags |= GFSC_MF_KERNEL_MOUNT_DONE;
-	if (mg->kernel_mount_error)
-		out->flags |= GFSC_MF_KERNEL_MOUNT_ERROR;
-	if (mg->first_recovery_needed)
-		out->flags |= GFSC_MF_FIRST_RECOVERY_NEEDED;
-	if (mg->first_recovery_msg)
-		out->flags |= GFSC_MF_FIRST_RECOVERY_MSG;
-	if (mg->local_recovery_busy)
-		out->flags |= GFSC_MF_LOCAL_RECOVERY_BUSY;
-
-	if (!mg->started_change)
-		goto next;
-
-	cg = mg->started_change;
-
-	out->cg_prev.member_count = cg->member_count;
-	out->cg_prev.joined_count = cg->joined_count;
-	out->cg_prev.remove_count = cg->remove_count;
-	out->cg_prev.failed_count = cg->failed_count;
-	out->cg_prev.combined_seq = cg->combined_seq;
-	out->cg_prev.seq = cg->seq;
-
- next:
-	if (list_empty(&mg->changes))
-		goto out;
-
-	list_for_each_entry(cg, &mg->changes, list)
-		last = cg;
-
-	cg = list_first_entry(&mg->changes, struct change, list);
-
-	out->cg_next.member_count = cg->member_count;
-	out->cg_next.joined_count = cg->joined_count;
-	out->cg_next.remove_count = cg->remove_count;
-	out->cg_next.failed_count = cg->failed_count;
-	out->cg_next.combined_seq = last->seq;
-	out->cg_next.seq = cg->seq;
-
-	/* FIXME: use real definitions for these conditions
-	   (also in dlm_controld) */
-
-	if (cg->state == CGST_WAIT_CONDITIONS)
-		out->cg_next.wait_condition = 4;
-	if (!mg->kernel_mount_done)
-		out->cg_next.wait_condition = 1;
-	if (mg->dlm_notify_nodeid)
-		out->cg_next.wait_condition = 2;
-	if (poll_dlm)
-		out->cg_next.wait_condition = 3;
-
-	if (cg->state == CGST_WAIT_MESSAGES)
-		out->cg_next.wait_messages = 1;
- out:
-	return 0;
-}
-
-static int _set_node_info(struct mountgroup *mg, struct change *cg, int nodeid,
-			  struct gfsc_node *node)
-{
-	struct member *m = NULL;
-	struct node *n;
-
-	node->nodeid = nodeid;
-
-	if (cg)
-		m = find_memb(cg, nodeid);
-	if (!m)
-		goto history;
-
-	node->flags |= GFSC_NF_MEMBER;
-
-	if (m->start)
-		node->flags |= GFSC_NF_START;
-	if (m->disallowed)
-		node->flags |= GFSC_NF_DISALLOWED;
-
- history:
-	n = get_node_history(mg, nodeid);
-	if (!n)
-		goto out;
-
-	node->jid = n->jid;
-
-	if (n->kernel_mount_done)
-		node->flags |= GFSC_NF_KERNEL_MOUNT_DONE;
-	if (n->kernel_mount_error)
-		node->flags |= GFSC_NF_KERNEL_MOUNT_ERROR;
-	if (n->check_dlm)
-		node->flags |= GFSC_NF_CHECK_DLM;
-	if (n->ro)
-		node->flags |= GFSC_NF_READONLY;
-	if (n->spectator)
-		node->flags |= GFSC_NF_SPECTATOR;
-
-	node->added_seq = n->added_seq;
-	node->removed_seq = n->removed_seq;
-	node->failed_reason = n->failed_reason;
- out:
-	return 0;
-}
-
-int set_node_info(struct mountgroup *mg, int nodeid, struct gfsc_node *node)
-{
-	struct change *cg;
-
-	if (!list_empty(&mg->changes)) {
-		cg = list_first_entry(&mg->changes, struct change, list);
-		return _set_node_info(mg, cg, nodeid, node);
-	}
-
-	return _set_node_info(mg, mg->started_change, nodeid, node);
-}
-
-int set_mountgroups(int *count, struct gfsc_mountgroup **mgs_out)
-{
-	struct mountgroup *mg;
-	struct gfsc_mountgroup *mgs, *mgp;
-	int mg_count = 0;
-
-	list_for_each_entry(mg, &mountgroups, list)
-		mg_count++;
-
-	mgs = malloc(mg_count * sizeof(struct gfsc_mountgroup));
-	if (!mgs)
-		return -ENOMEM;
-	memset(mgs, 0, mg_count * sizeof(struct gfsc_mountgroup));
-
-	mgp = mgs;
-	list_for_each_entry(mg, &mountgroups, list) {
-		set_mountgroup_info(mg, mgp++);
-	}
-
-	*count = mg_count;
-	*mgs_out = mgs;
-	return 0;
-}
-
-int set_mountgroup_nodes(struct mountgroup *mg, int option, int *node_count,
-                        struct gfsc_node **nodes_out)
-{
-	struct change *cg;
-	struct node *n;
-	struct gfsc_node *nodes = NULL, *nodep;
-	struct member *memb;
-	int count = 0;
-
-	if (option == GFSC_NODES_ALL) {
-		if (!list_empty(&mg->changes))
-			cg = list_first_entry(&mg->changes, struct change,list);
-		else
-			cg = mg->started_change;
-
-		list_for_each_entry(n, &mg->node_history, list)
-			count++;
-
-	} else if (option == GFSC_NODES_MEMBERS) {
-		if (!mg->started_change)
-			goto out;
-		cg = mg->started_change;
-		count = cg->member_count;
-
-	} else if (option == GFSC_NODES_NEXT) {
-		if (list_empty(&mg->changes))
-			goto out;
-		cg = list_first_entry(&mg->changes, struct change, list);
-		count = cg->member_count;
-	} else
-		goto out;
-
-	nodes = malloc(count * sizeof(struct gfsc_node));
-	if (!nodes)
-		return -ENOMEM;
-	memset(nodes, 0, count * sizeof(struct gfsc_node));
-	nodep = nodes;
-
-	if (option == GFSC_NODES_ALL) {
-		list_for_each_entry(n, &mg->node_history, list)
-			_set_node_info(mg, cg, n->nodeid, nodep++);
-	} else {
-		list_for_each_entry(memb, &cg->members, list)
-			_set_node_info(mg, cg, memb->nodeid, nodep++);
-	}
- out:
-	*node_count = count;
-	*nodes_out = nodes;
-	return 0;
-}
-
diff --git a/group/gfs_controld/crc.c b/group/gfs_controld/crc.c
deleted file mode 100644
index 29bc096..0000000
--- a/group/gfs_controld/crc.c
+++ /dev/null
@@ -1,72 +0,0 @@
-#include "gfs_daemon.h"
-
-static const uint32_t crc_32_tab[] = {
-  0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
-  0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
-  0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
-  0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
-  0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
-  0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
-  0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
-  0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
-  0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
-  0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
-  0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
-  0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
-  0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
-  0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
-  0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
-  0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
-  0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
-  0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
-  0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
-  0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
-  0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
-  0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
-  0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
-  0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
-  0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
-  0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
-  0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
-  0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
-  0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
-  0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
-  0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
-  0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
-};
-
-/**
- *
- * Copied from:
- *
- * gfs2_disk_hash - hash an array of data
- * @data: the data to be hashed
- * @len: the length of data to be hashed
- *
- * This function must produce the same results as the one in the kernel:
- *   crc32_le(0xFFFFFFFF, data, len) ^ 0xFFFFFFFF
- *
- * Take some data and convert it to a 32-bit hash.
- *
- * The hash function is a 32-bit CRC of the data.  The algorithm uses
- * the crc_32_tab table above.
- *
- * This may not be the fastest hash function, but it does a fair bit better
- * at providing uniform results than the others I've looked at.  That's
- * really important for efficient directories.
- *
- * Returns: the hash
- */
-
-uint32_t cpgname_to_crc(const char *data, int len)
-{
-	uint32_t hash = 0xFFFFFFFF;
-
-	for (; len--; data++)
-		hash = crc_32_tab[(hash ^ *data) & 0xFF] ^ (hash >> 8);
-
-	hash = ~hash;
-
-	return hash;
-}
-
diff --git a/group/gfs_controld/gfs_controld.h b/group/gfs_controld/gfs_controld.h
deleted file mode 100644
index 571dffb..0000000
--- a/group/gfs_controld/gfs_controld.h
+++ /dev/null
@@ -1,37 +0,0 @@
-#ifndef __GFS_CONTROLD_DOT_H__
-#define __GFS_CONTROLD_DOT_H__
-
-/* This defines the interface between gfs_controld and libgfscontrol, and
-   should only be used by libgfscontrol. */
-
-#define GFSC_SOCK_PATH                  "gfsc_sock"
-#define GFSC_QUERY_SOCK_PATH            "gfsc_query_sock"
-
-#define GFSC_MAGIC                      0x6F5C6F5C
-#define GFSC_VERSION                    0x00010001
-
-#define GFSC_CMD_DUMP_DEBUG             1
-#define GFSC_CMD_DUMP_PLOCKS            2
-#define GFSC_CMD_MOUNTGROUP_INFO        3
-#define GFSC_CMD_NODE_INFO              4
-#define GFSC_CMD_MOUNTGROUPS            5
-#define GFSC_CMD_MOUNTGROUP_NODES       6
-#define GFSC_CMD_FS_JOIN		7
-#define GFSC_CMD_FS_REMOUNT		8
-#define GFSC_CMD_FS_MOUNT_DONE		9
-#define GFSC_CMD_FS_LEAVE		10
-
-struct gfsc_header {
-	unsigned int magic;
-	unsigned int version;
-	unsigned int command;
-	unsigned int option;
-	unsigned int len;
-	int data;       /* embedded command-specific data, for convenience */
-	int unused1;
-	int unsued2;
-	char name[GFS_MOUNTGROUP_LEN]; /* no terminating null space */
-};
-
-#endif
-
diff --git a/group/gfs_controld/gfs_daemon.h b/group/gfs_controld/gfs_daemon.h
deleted file mode 100644
index d5ea341..0000000
--- a/group/gfs_controld/gfs_daemon.h
+++ /dev/null
@@ -1,241 +0,0 @@
-#ifndef __GFS_DAEMON_DOT_H__
-#define __GFS_DAEMON_DOT_H__
-
-#include "clusterautoconfig.h"
-
-#include <sys/types.h>
-#include <asm/types.h>
-#include <sys/uio.h>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/poll.h>
-#include <sys/wait.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <net/if.h>
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <limits.h>
-#include <unistd.h>
-#include <time.h>
-#include <syslog.h>
-#include <sched.h>
-#include <signal.h>
-#include <sys/time.h>
-#include <dirent.h>
-#include <openais/saAis.h>
-#include <openais/saCkpt.h>
-#include <corosync/cpg.h>
-#include <liblogthread.h>
-
-#include <linux/dlmconstants.h>
-#include "libgfscontrol.h"
-#include "gfs_controld.h"
-#include "list.h"
-#include "linux_endian.h"
-
-/* TODO: warn if
-   DLM_LOCKSPACE_LEN (from dlmconstants.h) !=
-   GFS_MOUNTGROUP_LEN (from libgfscontrol.h)
-*/
-
-/* Maximum members of a mountgroup, should match CPG_MEMBERS_MAX in
-   corosync/cpg.h.  There are no max defines in gfs-kernel for
-   mountgroup members. (FIXME verify gfs-kernel/lock_dlm) */
-
-#define MAX_NODES       128
-
-/* Max string length printed on a line, for debugging/dump output. */
-
-#define MAXLINE         256
-
-extern int daemon_debug_opt;
-extern int daemon_quit;
-extern int cluster_down;
-extern int poll_dlm;
-extern struct list_head mountgroups;
-extern int our_nodeid;
-extern char clustername[1024]; /* actual limit is sure to be smaller */
-extern char daemon_debug_buf[256];
-extern char dump_buf[GFSC_DUMP_SIZE];
-extern int dump_point;
-extern int dump_wrap;
-extern int dmsetup_wait;
-extern cpg_handle_t cpg_handle_daemon;
-extern int libcpg_flow_control_on;
-extern struct list_head withdrawn_mounts;
-
-void daemon_dump_save(void);
-
-#define log_level(lvl, fmt, args...) \
-do { \
-	snprintf(daemon_debug_buf, 255, "%ld " fmt "\n", time(NULL), ##args); \
-	daemon_dump_save(); \
-	logt_print(lvl, fmt "\n", ##args); \
-	if (daemon_debug_opt) \
-		fprintf(stderr, "%s", daemon_debug_buf); \
-} while (0)
-
-#define log_debug(fmt, args...) log_level(LOG_DEBUG, fmt, ##args)
-#define log_error(fmt, args...) log_level(LOG_ERR, fmt, ##args)
-
-#define log_group(g, fmt, args...) \
-do { \
-	snprintf(daemon_debug_buf, 255, "%ld %s " fmt "\n", time(NULL), \
-		 (g)->name, ##args); \
-	daemon_dump_save(); \
-	logt_print(LOG_DEBUG, "%s " fmt "\n", (g)->name, ##args); \
-	if (daemon_debug_opt) \
-		fprintf(stderr, "%s", daemon_debug_buf); \
-} while (0)
-
-#define log_plock(g, fmt, args...) \
-do { \
-	snprintf(daemon_debug_buf, 255, "%ld %s " fmt "\n", time(NULL), \
-		 (g)->name, ##args); \
-	if (daemon_debug_opt && cfgd_plock_debug) \
-		fprintf(stderr, "%s", daemon_debug_buf); \
-} while (0)
-
-struct mountgroup {
-	struct list_head	list;
-	uint32_t		id;
-	struct gfsc_mount_args	mount_args;
-	char			name[GFS_MOUNTGROUP_LEN+1];
-
-	int			mount_client;
-	int			mount_client_result;
-	int			mount_client_notified;
-	int			mount_client_delay;
-	int			remount_client;
-
-	int			withdraw_uevent;
-	int			withdraw_suspend;
-	int			dmsetup_wait;
-	pid_t			dmsetup_pid;
-	int			our_jid;
-	int			spectator;
-	int			ro;
-	int                     joining;
-	int                     leaving;
-	int			kernel_mount_error;
-	int			kernel_mount_done;
-	int			first_mounter;
-	int			no_mount_helper;
-
-	/* cpg-new stuff */
-
-	cpg_handle_t            cpg_handle;
-	int                     cpg_client;
-	int                     cpg_fd;
-	int                     kernel_stopped;
-	uint32_t                change_seq;
-	uint32_t                started_count;
-	struct change           *started_change;
-	struct list_head        changes;
-	struct list_head        node_history;
-	struct list_head	journals;
-	int			dlm_registered;
-	int			dlm_notify_nodeid;
-	int			first_done_uevent;
-	int			first_recovery_needed;
-	int			first_recovery_master;
-	int			first_recovery_msg;
-	int			local_recovery_jid;
-	int			local_recovery_busy;
-};
-
-/* these need to match the kernel defines of the same name in lm_interface.h */
-
-#define LM_RD_GAVEUP 308
-#define LM_RD_SUCCESS 309
-
-/* config.c */
-int setup_ccs(void);
-void close_ccs(void);
-void read_ccs_name(const char *path, char *name);
-void read_ccs_yesno(const char *path, int *yes, int *no);
-int read_ccs_int(const char *path, int *config_val);
-void read_ccs_nodir(struct mountgroup *mg, char *buf);
-
-/* cpg-new.c */
-int setup_cpg_daemon(void);
-void close_cpg_daemon(void);
-void process_cpg_daemon(int ci);
-int setup_dlmcontrol(void);
-void process_dlmcontrol(int ci);
-int set_protocol(void);
-void process_recovery_uevent(struct mountgroup *mg, int jid, int status);
-void process_first_mount(struct mountgroup *mg);
-void process_mountgroups(void);
-int gfs_join_mountgroup(struct mountgroup *mg);
-void do_leave(struct mountgroup *mg, int mnterr);
-void gfs_mount_done(struct mountgroup *mg);
-void send_remount(struct mountgroup *mg, int ro);
-void send_withdraw(struct mountgroup *mg);
-int set_mountgroup_info(struct mountgroup *mg, struct gfsc_mountgroup *out);
-int set_node_info(struct mountgroup *mg, int nodeid, struct gfsc_node *node);
-int set_mountgroups(int *count, struct gfsc_mountgroup **mgs_out);
-int set_mountgroup_nodes(struct mountgroup *mg, int option, int *node_count,
-	struct gfsc_node **nodes_out);
-void free_mg(struct mountgroup *mg);
-void node_history_cluster_add(int nodeid);
-void node_history_cluster_remove(int nodeid);
-void gfs_leave_mountgroup(struct mountgroup *mg, int mnterr);
-
-/* main.c */
-int do_read(int fd, void *buf, size_t count);
-int do_write(int fd, void *buf, size_t count);
-void client_dead(int ci);
-int client_add(int fd, void (*workfn)(int ci), void (*deadfn)(int ci));
-int client_fd(int ci);
-void client_ignore(int ci, int fd);
-void client_back(int ci, int fd);
-struct mountgroup *create_mg(const char *name);
-struct mountgroup *find_mg(char *name);
-struct mountgroup *find_mg_id(uint32_t id);
-void client_reply_remount(struct mountgroup *mg, int ci, int result);
-void client_reply_join_full(struct mountgroup *mg, int result);
-void query_lock(void);
-void query_unlock(void);
-void process_connection(int ci);
-void cluster_dead(int ci);
-
-/* member_cman.c */
-int setup_cluster(void);
-void close_cluster(void);
-void process_cluster(int ci);
-void update_cluster(void);
-int setup_cluster_cfg(void);
-void close_cluster_cfg(void);
-void process_cluster_cfg(int ci);
-void kick_node_from_cluster(int nodeid);
-
-/* util.c */
-int we_are_in_fence_domain(void);
-int set_sysfs(struct mountgroup *mg, const char *field, int val);
-int run_dmsetup_suspend(struct mountgroup *mg, char *dev);
-void update_dmsetup_wait(void);
-void update_flow_control_status(void);
-int check_uncontrolled_filesystems(void);
-
-/* logging.c */
-
-void init_logging(void);
-void setup_logging(void);
-void close_logging(void);
-
-/* crc.c */
-uint32_t cpgname_to_crc(const char *data, int len);
-
-#endif
diff --git a/group/gfs_controld/logging.c b/group/gfs_controld/logging.c
deleted file mode 100644
index 6c6d024..0000000
--- a/group/gfs_controld/logging.c
+++ /dev/null
@@ -1,65 +0,0 @@
-#include <syslog.h>
-#include "gfs_daemon.h"
-#include "config.h"
-#include "ccs.h"
-
-extern int ccs_handle;
-
-#define DAEMON_NAME "gfs_controld"
-#define DEFAULT_LOG_MODE LOG_MODE_OUTPUT_FILE|LOG_MODE_OUTPUT_SYSLOG
-#define DEFAULT_SYSLOG_FACILITY		LOG_LOCAL4 /* Is this really right? */
-#define DEFAULT_SYSLOG_PRIORITY		LOG_INFO
-#define DEFAULT_LOGFILE_PRIORITY	LOG_INFO /* ? */
-#ifndef LOGDIR
-#define LOGDIR "/var/log/cluster"
-#endif /* LOGDIR */
-#define DEFAULT_LOGFILE			LOGDIR "/" DAEMON_NAME ".log"
-
-static int log_mode;
-static int syslog_facility;
-static int syslog_priority;
-static int logfile_priority;
-static char logfile[PATH_MAX];
-
-void init_logging(void)
-{
-	log_mode = DEFAULT_LOG_MODE;
-	syslog_facility = DEFAULT_SYSLOG_FACILITY;
-	syslog_priority = DEFAULT_SYSLOG_PRIORITY;
-	logfile_priority = DEFAULT_LOGFILE_PRIORITY;
-	strcpy(logfile, DEFAULT_LOGFILE);
-
-	/* logfile_priority is the only one of these options that
-	   can be controlled from command line or environment variable */
-
-	if (cfgd_debug_logfile)
-		logfile_priority = LOG_DEBUG;
-
-	log_debug("logging mode %d syslog f %d p %d logfile p %d %s",
-		  log_mode, syslog_facility, syslog_priority,
-		  logfile_priority, logfile);
-
-	logt_init(DAEMON_NAME, log_mode, syslog_facility, syslog_priority,
-		  logfile_priority, logfile);
-}
-
-void setup_logging(void)
-{
-	ccs_read_logging(ccs_handle, DAEMON_NAME,
-			 &cfgd_debug_logfile, &log_mode,
-			 &syslog_facility, &syslog_priority,
-			 &logfile_priority, logfile);
-
-	log_debug("logging mode %d syslog f %d p %d logfile p %d %s",
-		  log_mode, syslog_facility, syslog_priority,
-		  logfile_priority, logfile);
-
-	logt_conf(DAEMON_NAME, log_mode, syslog_facility, syslog_priority,
-		  logfile_priority, logfile);
-}
-
-void close_logging(void)
-{
-	logt_exit();
-}
-
diff --git a/group/gfs_controld/main.c b/group/gfs_controld/main.c
deleted file mode 100644
index aa38839..0000000
--- a/group/gfs_controld/main.c
+++ /dev/null
@@ -1,1496 +0,0 @@
-#include "gfs_daemon.h"
-#include "config.h"
-#include <pthread.h>
-#include "copyright.cf"
-
-#include <linux/netlink.h>
-
-#ifndef CLUSTERVARRUN
-#define CLUSTERVARRUN "/var/run/cluster"
-#endif /* CLUSTERVARRUN */
-#define LOCKFILE_NAME CLUSTERVARRUN "/gfs_controld.pid"
-#define CLIENT_NALLOC   32
-#define UEVENT_BUF_SIZE 4096
-
-static int client_maxi;
-static int client_size;
-static struct client *client;
-static struct pollfd *pollfd;
-static pthread_t query_thread;
-static pthread_mutex_t query_mutex;
-
-struct client {
-	int fd;
-	void *workfn;
-	void *deadfn;
-	struct mountgroup *mg;
-};
-
-static void do_withdraw(struct mountgroup *mg);
-
-int do_read(int fd, void *buf, size_t count)
-{
-	int rv, off = 0;
-
-	while (off < count) {
-		rv = read(fd, (char *)buf + off, count - off);
-		if (rv == 0)
-			return -1;
-		if (rv == -1 && errno == EINTR)
-			continue;
-		if (rv == -1)
-			return -1;
-		off += rv;
-	}
-	return 0;
-}
-
-int do_write(int fd, void *buf, size_t count)
-{
-	int rv, off = 0;
-
- retry:
-	rv = write(fd, (char *)buf + off, count);
-	if (rv == -1 && errno == EINTR)
-		goto retry;
-	if (rv < 0) {
-		log_error("write errno %d", errno);
-		return rv;
-	}
-
-	if (rv != count) {
-		count -= rv;
-		off += rv;
-		goto retry;
-	}
-	return 0;
-}
-
-static void client_alloc(void)
-{
-	int i;
-
-	if (!client) {
-		client = malloc(CLIENT_NALLOC * sizeof(struct client));
-		pollfd = malloc(CLIENT_NALLOC * sizeof(struct pollfd));
-	} else {
-		client = realloc(client, (client_size + CLIENT_NALLOC) *
-					 sizeof(struct client));
-		pollfd = realloc(pollfd, (client_size + CLIENT_NALLOC) *
-					 sizeof(struct pollfd));
-		if (!pollfd)
-			log_error("can't alloc for pollfd");
-	}
-	if (!client || !pollfd)
-		log_error("can't alloc for client array");
-
-	for (i = client_size; i < client_size + CLIENT_NALLOC; i++) {
-		client[i].workfn = NULL;
-		client[i].deadfn = NULL;
-		client[i].fd = -1;
-		pollfd[i].fd = -1;
-		pollfd[i].revents = 0;
-	}
-	client_size += CLIENT_NALLOC;
-}
-
-void client_dead(int ci)
-{
-	close(client[ci].fd);
-	client[ci].workfn = NULL;
-	client[ci].fd = -1;
-	pollfd[ci].fd = -1;
-}
-
-int client_add(int fd, void (*workfn)(int ci), void (*deadfn)(int ci))
-{
-	int i;
-
-	if (!client)
-		client_alloc();
- again:
-	for (i = 0; i < client_size; i++) {
-		if (client[i].fd == -1) {
-			client[i].workfn = workfn;
-			if (deadfn)
-				client[i].deadfn = deadfn;
-			else
-				client[i].deadfn = client_dead;
-			client[i].fd = fd;
-			pollfd[i].fd = fd;
-			pollfd[i].events = POLLIN;
-			if (i > client_maxi)
-				client_maxi = i;
-			return i;
-		}
-	}
-
-	client_alloc();
-	goto again;
-}
-
-int client_fd(int ci)
-{
-	return client[ci].fd;
-}
-
-void client_ignore(int ci, int fd)
-{
-	pollfd[ci].fd = -1;
-	pollfd[ci].events = 0;
-}
-
-void client_back(int ci, int fd)
-{
-	pollfd[ci].fd = fd;
-	pollfd[ci].events = POLLIN;
-}
-
-static void sigterm_handler(int sig)
-{
-	daemon_quit = 1;
-}
-
-struct mountgroup *create_mg(const char *name)
-{
-	struct mountgroup *mg;
-
-	if (strlen(name) > GFS_MOUNTGROUP_LEN) {
-		log_error("create_mg: name %s too long", name);
-		return NULL;
-	}
-
-	mg = malloc(sizeof(struct mountgroup));
-	if (!mg)
-		return NULL;
-	memset(mg, 0, sizeof(struct mountgroup));
-
-	INIT_LIST_HEAD(&mg->changes);
-	INIT_LIST_HEAD(&mg->journals);
-	INIT_LIST_HEAD(&mg->node_history);
-
-	strncpy(mg->name, name, GFS_MOUNTGROUP_LEN);
-
-	return mg;
-}
-
-struct mountgroup *find_mg(char *name)
-{
-	struct mountgroup *mg;
-
-	if (strlen(name) > GFS_MOUNTGROUP_LEN) {
-		log_error("find_mg: name %s too long", name);
-		return NULL;
-	}
-
-	list_for_each_entry(mg, &mountgroups, list) {
-		if ((strlen(mg->name) == strlen(name)) &&
-		    !strncmp(mg->name, name, strlen(name)))
-			return mg;
-	}
-	return NULL;
-}
-
-struct mountgroup *find_mg_id(uint32_t id)
-{
-	struct mountgroup *mg;
-
-	list_for_each_entry(mg, &mountgroups, list) {
-		if (mg->id == id)
-			return mg;
-	}
-	return NULL;
-}
-
-enum {
-	Env_ACTION = 0,
-	Env_SUBSYSTEM,
-	Env_LOCKPROTO,
-	Env_LOCKTABLE,
-	Env_DEVPATH,
-	Env_RECOVERY,
-	Env_FIRSTMOUNT,
-	Env_JID,
-	Env_SPECTATOR,
-	Env_RDONLY,
-	Env_Last, /* Flag for end of vars */
-};
-
-static const char *uevent_vars[] = {
-	[Env_ACTION]		= "ACTION=",
-	[Env_SUBSYSTEM]		= "SUBSYSTEM=",
-	[Env_LOCKPROTO]		= "LOCKPROTO=",
-	[Env_LOCKTABLE]		= "LOCKTABLE=",
-	[Env_DEVPATH]		= "DEVPATH=",
-	[Env_RECOVERY]		= "RECOVERY=",
-	[Env_FIRSTMOUNT]	= "FIRSTMOUNT=",
-	[Env_JID]		= "JID=",
-	[Env_SPECTATOR]		= "SPECTATOR=",
-	[Env_RDONLY]		= "RDONLY=",
-};
-
-/*
- * Parses a uevent message for the interesting bits. It requires a list
- * of variables to look for, and an equally long list of pointers into
- * which to write the results.
- */
-static void decode_uevent(const char *buf, unsigned len, const char *vars[],
-			  unsigned nvars, const char *vals[])
-{
-	const char *ptr;
-	unsigned int i;
-	int slen, vlen;
-
-	memset(vals, 0, sizeof(const char *) * nvars);
-
-	while (len > 0) {
-		ptr = buf;
-		slen = strlen(ptr);
-		buf += slen;
-		len -= slen;
-		buf++;
-		len--;
-
-		for (i = 0; i < nvars; i++) {
-			vlen = strlen(vars[i]);
-			if (vlen > slen)
-				continue;
-			if (memcmp(vars[i], ptr, vlen) != 0)
-				continue;
-			vals[i] = ptr + vlen;
-			break;
-		}
-	}
-}
-
-static char *uevent_fsname(const char *vals[])
-{
-	char *name = NULL;
-
-	if (vals[Env_LOCKTABLE]) {
-		name = strchr(vals[Env_LOCKTABLE], ':');
-		if (name && *name)
-			name++;
-	}
-	return name;
-}
-
-/*
- * This is called only if mount.gfs2 has not already set up the
- * mount group. In that case we know that the mount helper doesn't
- * exist and thus the no_mount_helper flag is set, to indicate that
- * this mount will be administrated entirely via the uevent/sysfs
- * interface.
- */
-
-static void do_new_mount(const char *name, struct mountgroup *mg,
-			 const char *uevent_vals[])
-{
-	int rv;
-
-	if (!uevent_vars[Env_LOCKPROTO] ||
-	    !uevent_vars[Env_LOCKTABLE])
-		return;
-
-	/* We only care about lock_dlm mounts */
-	if (strcmp(uevent_vals[Env_LOCKPROTO], "lock_dlm") != 0)
-		return;
-
-	if (mg) {
-		/* Might have already been set up by mount.gfs2 */
-		if (mg->no_mount_helper == 0)
-			return;
-		log_error("do_new_mount: duplicate mount %s",
-			  uevent_vals[Env_LOCKTABLE]);
-		return;
-	}
-
-	mg = create_mg(name);
-	if (mg == NULL)
-		return;
-
-	mg->no_mount_helper = 1;
-
-	strncpy(mg->mount_args.type, uevent_vals[Env_SUBSYSTEM], PATH_MAX);
-	strncpy(mg->mount_args.proto, uevent_vals[Env_LOCKPROTO], PATH_MAX);
-	strncpy(mg->mount_args.table, uevent_vals[Env_LOCKTABLE], PATH_MAX);
-
-	if (uevent_vals[Env_SPECTATOR] &&
-	    strcmp(uevent_vals[Env_SPECTATOR], "1") == 0)
-		mg->spectator = 1;
-
-	if (uevent_vals[Env_RDONLY] &&
-	    strcmp(uevent_vals[Env_RDONLY], "1") == 0)
-		mg->ro = 1;
-
-	list_add(&mg->list, &mountgroups);
-	rv = gfs_join_mountgroup(mg);
-	if (rv) {
-		log_error("join: group join error %d", rv);
-		goto fail;
-	}
-	log_group(mg, "do_new_mount ci %d result %d first=%d:jid=%d",
-                  mg->mount_client, rv, mg->first_mounter, mg->our_jid);
-	return;
-
-fail:
-	list_del(&mg->list);
-	free(mg);
-	return;
-}
-
-/*
- * This is called upon successful mount and also upon a successful
- * remount operation. Unless the no_mount_helper flag is set on the
- * mount group, this is a no-op.
- */
-static void do_online(struct mountgroup *mg, const char *uevent_vals[])
-{
-	int ro = 0;
-
-	/* If using mount helper, ignore the message here */
-	if (mg->no_mount_helper == 0)
-		return;
-
-	/* Catch successful original mount */
-	if (!mg->kernel_mount_done) {
-		mg->mount_client = 0;
-		mg->kernel_mount_done = 1;
-		mg->kernel_mount_error = 0;
-		gfs_mount_done(mg);
-		return;
-	}
-
-	/* From here on, its remounts only */
-
-	if (uevent_vals[Env_RDONLY] &&
-	    strcmp(uevent_vals[Env_RDONLY], "1") == 0)
-		ro = 1;
-
-	send_remount(mg, ro);
-}
-
-static void process_uevent(int ci)
-{
-	struct mountgroup *mg;
-	char buf[UEVENT_BUF_SIZE];
-	const char *uevent_vals[Env_Last];
-	char *fsname;
-	int rv;
-
- retry_recv:
-	rv = recv(client[ci].fd, &buf, sizeof(buf), 0);
-	if (rv < 0) {
-		if (errno == EINTR)
-			goto retry_recv;
-		if (errno != EAGAIN)
-			log_error("uevent recv error %d errno %d", rv, errno);
-		return;
-	}
-	buf[rv] = 0;
-
-	decode_uevent(buf, rv, uevent_vars, Env_Last, uevent_vals);
-
-	if (!uevent_vals[Env_DEVPATH] ||
-	    !uevent_vals[Env_ACTION] ||
-	    !uevent_vals[Env_SUBSYSTEM])
-		return;
-
-	if (strncmp(uevent_vals[Env_DEVPATH], "/fs/gfs", 7) != 0)
-		return;
-
-	log_debug("uevent %s %s %s",
-		  uevent_vals[Env_ACTION],
-		  uevent_vals[Env_SUBSYSTEM],
-		  uevent_vals[Env_DEVPATH]);
-
-	fsname = uevent_fsname(uevent_vals);
-	if (!fsname) {
-		log_error("no fsname uevent %s %s %s",
-		  	  uevent_vals[Env_ACTION],
-		  	  uevent_vals[Env_SUBSYSTEM],
-		  	  uevent_vals[Env_DEVPATH]);
-		return;
-	}
-
-	mg = find_mg(fsname);
-
-	if (!strcmp(uevent_vals[Env_ACTION], "add")) {
-		do_new_mount(fsname, mg, uevent_vals);
-		return;
-	}
-
-	if (!mg) {
-		log_error("mount group %s not found", fsname);
-		return;
-	}
-
-	if (!strcmp(uevent_vals[Env_ACTION], "remove")) {
-		/* We want to trigger the leave at the very end of the kernel's
-		   unmount process, i.e. at the end of put_super(), so we do the
-		   leave when the second uevent (from the gfs kobj) arrives. */
-
-		if (strcmp(uevent_vals[Env_SUBSYSTEM], "lock_dlm") == 0)
-			return;
-
-		/* Catch original mount failure */
-		if (mg->no_mount_helper && !mg->kernel_mount_done) {
-			mg->mount_client = 0;
-			mg->kernel_mount_done = 1;
-			mg->kernel_mount_error = -1;
-			gfs_mount_done(mg);
-			return;
-		}
-
-		do_leave(mg, 0);
-		return;
-	}
-
-	if (!strcmp(uevent_vals[Env_ACTION], "change")) {
-		int jid, status = -1;
-
-
-		if (uevent_vals[Env_RECOVERY]) {
-			if (!uevent_vals[Env_JID] ||
-			    (sscanf(uevent_vals[Env_JID], "%d", &jid) != 1))
-				return;
-			if (strcmp(uevent_vals[Env_RECOVERY], "Done") == 0)
-				status = LM_RD_SUCCESS;
-			if (strcmp(uevent_vals[Env_RECOVERY], "Failed") == 0)
-				status = LM_RD_GAVEUP;
-			if (status < 0)
-				return;
-			process_recovery_uevent(mg, jid, status);
-			return;
-		}
-
-		if (uevent_vals[Env_FIRSTMOUNT] &&
-		    (strcmp(uevent_vals[Env_FIRSTMOUNT], "Done") == 0)) {
-			process_first_mount(mg);
-		}
-		return;
-	}
-
-	if (!strcmp(uevent_vals[Env_ACTION], "online"))
-		do_online(mg, uevent_vals);
-
-	if (!strcmp(uevent_vals[Env_ACTION], "offline"))
-		do_withdraw(mg);
-}
-
-static int setup_uevent(void)
-{
-	struct sockaddr_nl snl;
-	int s, rv;
-
-	s = socket(AF_NETLINK, SOCK_DGRAM, NETLINK_KOBJECT_UEVENT);
-	if (s < 0) {
-		log_error("uevent netlink socket");
-		return s;
-	}
-
-	memset(&snl, 0, sizeof(snl));
-	snl.nl_family = AF_NETLINK;
-	snl.nl_pid = getpid();
-	snl.nl_groups = 1;
-
-	rv = bind(s, (struct sockaddr *) &snl, sizeof(snl));
-	if (rv < 0) {
-		log_error("uevent bind error %d errno %d", rv, errno);
-		close(s);
-		return rv;
-	}
-
-	return s;
-}
-
-static void init_header(struct gfsc_header *h, int cmd, char *name, int result,
-			int extra_len)
-{
-	memset(h, 0, sizeof(struct gfsc_header));
-
-	h->magic = GFSC_MAGIC;
-	h->version = GFSC_VERSION;
-	h->len = sizeof(struct gfsc_header) + extra_len;
-	h->command = cmd;
-	h->data = result;
-
-	if (name)
-		strncpy(h->name, name, GFS_MOUNTGROUP_LEN);
-}
-
-static void query_dump_debug(int fd)
-{
-	struct gfsc_header h;
-	int extra_len;
-	int len;
-
-	/* in the case of dump_wrap, extra_len will go in two writes,
-	   first the log tail, then the log head */
-	if (dump_wrap)
-		extra_len = GFSC_DUMP_SIZE;
-	else
-		extra_len = dump_point;
-
-	init_header(&h, GFSC_CMD_DUMP_DEBUG, NULL, 0, extra_len);
-	do_write(fd, &h, sizeof(h));
-
-	if (dump_wrap) {
-		len = GFSC_DUMP_SIZE - dump_point;
-		do_write(fd, dump_buf + dump_point, len);
-		len = dump_point;
-	} else
-		len = dump_point;
-
-	/* NUL terminate the debug string */
-	dump_buf[dump_point] = '\0';
-
-	do_write(fd, dump_buf, len);
-}
-
-/* combines a header and the data and sends it back to the client in
-   a single do_write() call */
-
-static void do_reply(int fd, int cmd, char *name, int result, void *buf,
-		     int buflen)
-{
-	char *reply;
-	int reply_len;
-
-	reply_len = sizeof(struct gfsc_header) + buflen;
-	reply = malloc(reply_len);
-	if (!reply)
-		return;
-	memset(reply, 0, reply_len);
-
-	init_header((struct gfsc_header *)reply, cmd, name, result, buflen);
-
-	if (buf && buflen)
-		memcpy(reply + sizeof(struct gfsc_header), buf, buflen);
-
-	do_write(fd, reply, reply_len);
-
-	free(reply);
-}
-
-static void query_mountgroup_info(int fd, char *name)
-{
-	struct mountgroup *mg;
-	struct gfsc_mountgroup mountgroup;
-	int rv;
-
-	mg = find_mg(name);
-	if (!mg) {
-		rv = -ENOENT;
-		goto out;
-	}
-
-	memset(&mountgroup, 0, sizeof(mountgroup));
-
-	rv = set_mountgroup_info(mg, &mountgroup);
- out:
-	do_reply(fd, GFSC_CMD_MOUNTGROUP_INFO, name, rv,
-		 (char *)&mountgroup, sizeof(mountgroup));
-}
-
-static void query_node_info(int fd, char *name, int nodeid)
-{
-	struct mountgroup *mg;
-	struct gfsc_node node;
-	int rv;
-
-	mg = find_mg(name);
-	if (!mg) {
-		rv = -ENOENT;
-		goto out;
-	}
-
-	rv = set_node_info(mg, nodeid, &node);
- out:
-	do_reply(fd, GFSC_CMD_NODE_INFO, name, rv,
-		 (char *)&node, sizeof(node));
-}
-
-static void query_mountgroups(int fd, int max)
-{
-	int mg_count = 0;
-	struct gfsc_mountgroup *mgs = NULL;
-	int rv, result;
-
-	rv = set_mountgroups(&mg_count, &mgs);
-	if (rv < 0) {
-		result = rv;
-		mg_count = 0;
-		goto out;
-	}
-
-	if (mg_count > max) {
-		result = -E2BIG;
-		mg_count = max;
-	} else {
-		result = mg_count;
-	}
- out:
-	do_reply(fd, GFSC_CMD_MOUNTGROUPS, NULL, result,
-		 (char *)mgs, mg_count * sizeof(struct gfsc_mountgroup));
-
-	if (mgs)
-		free(mgs);
-}
-
-static void query_mountgroup_nodes(int fd, char *name, int option, int max)
-{
-	struct mountgroup *mg;
-	int node_count = 0;
-	struct gfsc_node *nodes = NULL;
-	int rv, result;
-
-	mg = find_mg(name);
-	if (!mg) {
-		result = -ENOENT;
-		node_count = 0;
-		goto out;
-	}
-
-	rv = set_mountgroup_nodes(mg, option, &node_count, &nodes);
-	if (rv < 0) {
-		result = rv;
-		node_count = 0;
-		goto out;
-	}
-
-	/* node_count is the number of structs copied/returned; the caller's
-	   max may be less than that, in which case we copy as many as they
-	   asked for and return -E2BIG */
-
-	if (node_count > max) {
-		result = -E2BIG;
-		node_count = max;
-	} else {
-		result = node_count;
-	}
- out:
-	do_reply(fd, GFSC_CMD_MOUNTGROUP_NODES, name, result,
-		 (char *)nodes, node_count * sizeof(struct gfsc_node));
-
-	if (nodes)
-		free(nodes);
-}
-
-static void client_reply_join(int ci, struct gfsc_mount_args *ma, int result)
-{
-	char *name = strstr(ma->table, ":") + 1;
-
-	log_debug("client_reply_join %s ci %d result %d", name, ci, result);
-
-	do_reply(client[ci].fd, GFSC_CMD_FS_JOIN,
-		 name, result, ma, sizeof(struct gfsc_mount_args));
-}
-
-static void client_sysfs_join(struct mountgroup *mg, int result)
-{
-	int rv;
-
-	if (result) {
-		rv = set_sysfs(mg, "jid", result);
-		if (rv) {
-			log_error("join: error %d returning result %d", rv, result);
-		}
-		return;
-	}
-
-	if (mg->spectator) {
-		rv = set_sysfs(mg, "jid", 0);
-		if (rv) {
-			log_error("join: error setting jid %d", rv);
-		}
-		return;
-	}
-
-	rv = set_sysfs(mg, "first", mg->first_mounter);
-	if (rv) {
-		log_error("join: error setting first %d", rv);
-	}
-	rv = set_sysfs(mg, "jid", mg->our_jid);
-	if (rv) {
-		log_error("join: error setting jid %d", rv);
-	}
-}
-
-void client_reply_join_full(struct mountgroup *mg, int result)
-{
-	char nodir_str[32];
-
-	if (result)
-		goto out;
-
-	if (mg->our_jid < 0) {
-		snprintf(mg->mount_args.hostdata, PATH_MAX,
-			 "hostdata=id=%u:first=%d",
-			 mg->id, mg->first_mounter);
-	} else {
-		snprintf(mg->mount_args.hostdata, PATH_MAX,
-			 "hostdata=jid=%d:id=%u:first=%d",
-			 mg->our_jid, mg->id, mg->first_mounter);
-	}
-
-	memset(nodir_str, 0, sizeof(nodir_str));
-
-	read_ccs_nodir(mg, nodir_str);
-	if (nodir_str[0])
-		strcat(mg->mount_args.hostdata, nodir_str);
- out:
-	log_group(mg, "client_reply_join_full ci %d result %d %s",
-		  mg->mount_client, result, mg->mount_args.hostdata);
-
-	if (mg->no_mount_helper)
-		client_sysfs_join(mg, result);
-	else
-		client_reply_join(mg->mount_client, &mg->mount_args, result);
-}
-
-static void do_join(int ci, struct gfsc_mount_args *ma)
-{
-	struct mountgroup *mg = NULL;
-	char table2[PATH_MAX];
-	char *cluster = NULL, *name = NULL;
-	int rv;
-
-	log_debug("join: %s %s %s %s %s %s", ma->dir, ma->type, ma->proto,
-		  ma->table, ma->options, ma->dev);
-
-	if (strcmp(ma->proto, "lock_dlm")) {
-		log_error("join: lockproto %s not supported", ma->proto);
-		rv = -EPROTONOSUPPORT;
-		goto fail;
-	}
-
-	if (strstr(ma->options, "jid=") ||
-	    strstr(ma->options, "first=") ||
-	    strstr(ma->options, "id=")) {
-		log_error("join: jid, first and id are reserved options");
-		rv = -EOPNOTSUPP;
-		goto fail;
-	}
-
-	/* table is <cluster>:<name> */
-
-	memset(table2, 0, sizeof(table2));
-	strncpy(table2, ma->table, sizeof(table2));
-
-	name = strstr(table2, ":");
-	if (!name) {
-		rv = -EBADFD;
-		goto fail;
-	}
-
-	*name = '\0';
-	name++;
-	cluster = table2;
-
-	mg = find_mg(name);
-	if (mg) {
-		if (strcmp(mg->mount_args.dev, ma->dev)) {
-			log_error("different fs dev %s with same name",
-				  mg->mount_args.dev);
-			rv = -EADDRINUSE;
-		} else if (mg->leaving) {
-			/* we're leaving the group */
-			log_error("join: reject mount due to unmount");
-			rv = -ESTALE;
-		} else if (mg->mount_client || !mg->kernel_mount_done) {
-			log_error("join: other mount in progress %d %d",
-				  mg->mount_client, mg->kernel_mount_done);
-			rv = -EBUSY;
-		} else {
-			log_group(mg, "join: already mounted");
-			rv = -EALREADY;
-		}
-		goto fail;
-	}
-
-	mg = create_mg(name);
-	if (!mg) {
-		rv = -ENOMEM;
-		goto fail;
-	}
-	mg->mount_client = ci;
-	memcpy(&mg->mount_args, ma, sizeof(struct gfsc_mount_args));
-
-	if (strlen(cluster) != strlen(clustername) ||
-	    strlen(cluster) == 0 || strcmp(cluster, clustername)) {
-		log_error("join: fs requires cluster=\"%s\" current=\"%s\"",
-			  cluster, clustername);
-		rv = -EBADR;
-		goto fail_free;
-	}
-	log_group(mg, "join: cluster name matches: %s", clustername);
-
-	if (strstr(ma->options, "spectator")) {
-		log_group(mg, "join: spectator mount");
-		mg->spectator = 1;
-	} else {
-		if (!we_are_in_fence_domain()) {
-			log_error("join: not in default fence domain");
-			rv = -ENOANO;
-			goto fail_free;
-		}
-	}
-
-	if (strstr(ma->options, "ro")) {
-		if (mg->spectator) {
-			log_error("join: readonly invalid with spectator");
-			rv = -EROFS;
-			goto fail_free;
-		}
-		mg->ro = 1;
-	}
-
-	list_add(&mg->list, &mountgroups);
-
-	rv = gfs_join_mountgroup(mg);
-	if (rv) {
-		log_error("join: group join error %d", rv);
-		list_del(&mg->list);
-		goto fail_free;
-	}
-	return;
-
- fail_free:
-	free(mg);
- fail:
-	client_reply_join(ci, ma, rv);
-}
-
-/* The basic rule of withdraw is that we don't want to tell the kernel to drop
-   all locks until we know gfs has been stopped/blocked on all nodes.
-   A withdrawing node is very much like a readonly node, differences are
-   that others recover its journal when they remove it from the group,
-   and when it's been removed from the group, it tells the locally withdrawing
-   gfs to clear out locks. */
-
-static void do_withdraw(struct mountgroup *mg)
-{
-	int rv;
-
-	log_debug("withdraw: %s", mg->name);
-
-	if (!cfgd_enable_withdraw) {
-		log_error("withdraw feature not enabled");
-		return;
-	}
-
-	mg->withdraw_uevent = 1;
-
-	rv = run_dmsetup_suspend(mg, mg->mount_args.dev);
-	if (rv) {
-		log_error("do_withdraw %s: dmsetup %s error %d", mg->name,
-			  mg->mount_args.dev, rv);
-		return;
-	}
-
-	dmsetup_wait = 1;
-}
-
-static void do_mount_done(char *table, int result)
-{
-	struct mountgroup *mg;
-	char *name = strstr(table, ":") + 1;
-
-	log_debug("mount_done: %s result %d", name, result);
-
-	mg = find_mg(name);
-	if (!mg) {
-		log_error("mount_done: %s not found", name);
-		return;
-	}
-
-	mg->mount_client = 0;
-	mg->kernel_mount_done = 1;
-	mg->kernel_mount_error = result;
-
-	gfs_mount_done(mg);
-}
-
-void client_reply_remount(struct mountgroup *mg, int ci, int result)
-{
-	do_reply(client[ci].fd, GFSC_CMD_FS_REMOUNT, mg->name, result,
-		 &mg->mount_args, sizeof(struct gfsc_mount_args));
-}
-
-/* mount.gfs creates a special ma->options string with only "ro" or "rw" */
-
-static void do_remount(int ci, struct gfsc_mount_args *ma)
-{
-	struct mountgroup *mg;
-	char *name = strstr(ma->table, ":") + 1;
-	int ro = 0, result = 0;
-
-	log_debug("remount: %s ci %d options %s", name, ci, ma->options);
-
-	mg = find_mg(name);
-	if (!mg) {
-		log_error("remount: %s not found", name);
-		result = -1;
-		goto out;
-	}
-
-	/* FIXME: Should allow remounts */
-	if (mg->spectator) {
-		log_error("remount of spectator not allowed");
-		result = -1;
-		goto out;
-	}
-
-	if (!strcmp(ma->options, "ro"))
-		ro = 1;
-
-	send_remount(mg, ro);
-out:
-	client_reply_remount(mg, ci, result);
-}
-
-void process_connection(int ci)
-{
-	struct gfsc_header h;
-	struct gfsc_mount_args empty;
-	struct gfsc_mount_args *ma;
-	struct mountgroup *mg;
-	char *extra = NULL;
-	int rv, extra_len;
-
-	rv = do_read(client[ci].fd, &h, sizeof(h));
-	if (rv < 0) {
-		log_debug("connection %d read error %d", ci, rv);
-		goto out;
-	}
-
-	if (h.magic != GFSC_MAGIC) {
-		log_debug("connection %d magic error %x", ci, h.magic);
-		goto out;
-	}
-
-	if ((h.version & 0xFFFF0000) != (GFSC_VERSION & 0xFFFF0000)) {
-		log_debug("connection %d version error %x", ci, h.version);
-		goto out;
-	}
-
-	if (h.len > sizeof(h)) {
-		extra_len = h.len - sizeof(h);
-		extra = malloc(extra_len);
-		if (!extra) {
-			log_error("process_connection no mem %d", extra_len);
-			goto out;
-		}
-		memset(extra, 0, extra_len);
-
-		rv = do_read(client[ci].fd, extra, extra_len);
-		if (rv < 0) {
-			log_debug("connection %d extra read error %d", ci, rv);
-			goto out;
-		}
-	}
-
-	ma = (struct gfsc_mount_args *)extra;
-
-	if (!ma) {
-		memset(&empty, 0, sizeof(empty));
-
-		if (h.command == GFSC_CMD_FS_JOIN ||
-		    h.command == GFSC_CMD_FS_REMOUNT) {
-			do_reply(client[ci].fd, h.command, h.name, -EINVAL,
-				 &empty, sizeof(empty));
-		}
-		log_debug("connection %d cmd %d no data", ci, h.command);
-		goto out;
-	}
-
-	switch (h.command) {
-
-	case GFSC_CMD_FS_JOIN:
-		do_join(ci, ma);
-		break;
-
-	case GFSC_CMD_FS_LEAVE:
-		mg = find_mg(ma->table);
-		if (!mg) {
-			log_error("do_leave: %s not found", ma->table);
-			break;
-		}
-		do_leave(mg, h.data);
-		break;
-
-	case GFSC_CMD_FS_MOUNT_DONE:
-		do_mount_done(ma->table, h.data);
-		break;
-
-	case GFSC_CMD_FS_REMOUNT:
-		do_remount(ci, ma);
-		break;
-
-	default:
-		log_error("process_connection %d unknown command %d",
-			  ci, h.command);
-	}
- out:
-	if (extra)
-		free(extra);
-
-	/* no client_dead(ci) here, since the connection for
-	   join/remount is reused */
-}
-
-static void process_listener(int ci)
-{
-	int fd, i;
-
-	fd = accept(client[ci].fd, NULL, NULL);
-	if (fd < 0) {
-		log_error("process_listener: accept error %d %d", fd, errno);
-		return;
-	}
-
-	i = client_add(fd, process_connection, NULL);
-
-	log_debug("client connection %d fd %d", i, fd);
-}
-
-static int setup_listener(const char *sock_path)
-{
-	struct sockaddr_un addr;
-	socklen_t addrlen;
-	int rv, s;
-
-	/* we listen for new client connections on socket s */
-
-	s = socket(AF_LOCAL, SOCK_STREAM, 0);
-	if (s < 0) {
-		log_error("socket error %d %d", s, errno);
-		return s;
-	}
-
-	memset(&addr, 0, sizeof(addr));
-	addr.sun_family = AF_LOCAL;
-	strcpy(&addr.sun_path[1], sock_path);
-	addrlen = sizeof(sa_family_t) + strlen(addr.sun_path+1) + 1;
-
-	rv = bind(s, (struct sockaddr *) &addr, addrlen);
-	if (rv < 0) {
-		log_error("bind error %d %d", rv, errno);
-		close(s);
-		return rv;
-	}
-
-	rv = listen(s, 5);
-	if (rv < 0) {
-		log_error("listen error %d %d", rv, errno);
-		close(s);
-		return rv;
-	}
-	return s;
-}
-
-void query_lock(void)
-{
-	pthread_mutex_lock(&query_mutex);
-}
-
-void query_unlock(void)
-{
-	pthread_mutex_unlock(&query_mutex);
-}
-
-/* This is a thread, so we have to be careful, don't call log_ functions.
-   We need a thread to process queries because the main thread may block
-   for long periods. */
-
-static void *process_queries(void *arg)
-{
-	struct gfsc_header h;
-	int f, rv, s;
-
-	rv = setup_listener(GFSC_QUERY_SOCK_PATH);
-	if (rv < 0)
-		return NULL;
-
-	s = rv;
-
-	for (;;) {
-		f = accept(s, NULL, NULL);
-		if (f < 0)
-			return NULL;
-
-		rv = do_read(f, &h, sizeof(h));
-		if (rv < 0) {
-			goto out;
-		}
-
-		if (h.magic != GFSC_MAGIC) {
-			goto out;
-		}
-
-		if ((h.version & 0xFFFF0000) != (GFSC_VERSION & 0xFFFF0000)) {
-			goto out;
-		}
-
-		query_lock();
-
-		switch (h.command) {
-		case GFSC_CMD_DUMP_DEBUG:
-			query_dump_debug(f);
-			break;
-		case GFSC_CMD_MOUNTGROUP_INFO:
-			query_mountgroup_info(f, h.name);
-			break;
-		case GFSC_CMD_NODE_INFO:
-			query_node_info(f, h.name, h.data);
-			break;
-		case GFSC_CMD_MOUNTGROUPS:
-			query_mountgroups(f, h.data);
-			break;
-		case GFSC_CMD_MOUNTGROUP_NODES:
-			query_mountgroup_nodes(f, h.name, h.option, h.data);
-			break;
-		default:
-			break;
-		}
-		query_unlock();
-
- out:
-		close(f);
-	}
-}
-
-static int setup_queries(void)
-{
-	int rv;
-
-	pthread_mutex_init(&query_mutex, NULL);
-
-	rv = pthread_create(&query_thread, NULL, process_queries, NULL);
-	if (rv < 0) {
-		log_error("can't create query thread");
-		return rv;
-	}
-	return 0;
-}
-
-void cluster_dead(int ci)
-{
-	if (!cluster_down)
-		log_error("cluster is down, exiting");
-	daemon_quit = 1;
-	cluster_down = 1;
-}
-
-static void loop(void)
-{
-	int poll_timeout = -1;
-	int rv, i;
-	void (*workfn) (int ci);
-	void (*deadfn) (int ci);
-
-	rv = setup_queries();
-	if (rv < 0)
-		goto out;
-
-	rv = setup_listener(GFSC_SOCK_PATH);
-	if (rv < 0)
-		goto out;
-	client_add(rv, process_listener, NULL);
-
-	rv = setup_cluster_cfg();
-	if (rv < 0)
-		goto out;
-	client_add(rv, process_cluster_cfg, cluster_dead);
-
-	rv = setup_cluster();
-	if (rv < 0)
-		goto out;
-	client_add(rv, process_cluster, cluster_dead);
-
-	update_cluster();
-
-	rv = setup_ccs();
-	if (rv < 0)
-		goto out;
-
-	setup_logging();
-
-	rv = check_uncontrolled_filesystems();
-	if (rv < 0)
-		goto out;
-
-	rv = setup_uevent();
-	if (rv < 0)
-		goto out;
-	client_add(rv, process_uevent, NULL);
-
-	rv = setup_cpg_daemon();
-	if (rv < 0)
-		goto out;
-	client_add(rv, process_cpg_daemon, cluster_dead);
-
-	rv = set_protocol();
-	if (rv < 0)
-		goto out;
-
-	rv = setup_dlmcontrol();
-	if (rv < 0)
-		goto out;
-	client_add(rv, process_dlmcontrol, cluster_dead);
-
-	for (;;) {
-		rv = poll(pollfd, client_maxi + 1, poll_timeout);
-		if (rv == -1 && errno == EINTR) {
-			if (daemon_quit && list_empty(&mountgroups))
-				goto out;
-			daemon_quit = 0;
-			continue;
-		}
-		if (rv < 0) {
-			log_error("poll errno %d", errno);
-			goto out;
-		}
-
-		query_lock();
-
-		for (i = 0; i <= client_maxi; i++) {
-			if (client[i].fd < 0)
-				continue;
-			if (pollfd[i].revents & POLLIN) {
-				workfn = client[i].workfn;
-				workfn(i);
-			}
-			if (pollfd[i].revents & (POLLERR | POLLHUP | POLLNVAL)) {
-				deadfn = client[i].deadfn;
-				deadfn(i);
-			}
-		}
-		query_unlock();
-
-		if (daemon_quit)
-			break;
-
-		query_lock();
-
-		poll_timeout = -1;
-
-		if (poll_dlm) {
-			process_mountgroups();
-			poll_timeout = 500;
-		}
-
-		if (dmsetup_wait) {
-			update_dmsetup_wait();
-			if (dmsetup_wait) {
-				if (poll_timeout == -1)
-					poll_timeout = 1000;
-			}
-		}
-		query_unlock();
-	}
- out:
-	close_cpg_daemon();
-	close_logging();
-	close_ccs();
-	close_cluster();
-	close_cluster_cfg();
-
-	if (!list_empty(&mountgroups))
-		log_error("mountgroups abandoned");
-}
-
-static void lockfile(void)
-{
-	int fd, error;
-	struct flock lock;
-	char buf[33];
-
-	memset(buf, 0, 33);
-
-	fd = open(LOCKFILE_NAME, O_CREAT|O_WRONLY,
-		  S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
-	if (fd < 0) {
-		fprintf(stderr, "cannot open/create lock file %s\n",
-			LOCKFILE_NAME);
-		exit(EXIT_FAILURE);
-	}
-
-	lock.l_type = F_WRLCK;
-	lock.l_start = 0;
-	lock.l_whence = SEEK_SET;
-	lock.l_len = 0;
-
-	error = fcntl(fd, F_SETLK, &lock);
-	if (error) {
-		fprintf(stderr, "gfs_controld is already running\n");
-		exit(EXIT_FAILURE);
-	}
-
-	error = ftruncate(fd, 0);
-	if (error) {
-		fprintf(stderr, "cannot clear lock file %s\n", LOCKFILE_NAME);
-		exit(EXIT_FAILURE);
-	}
-
-	sprintf(buf, "%d\n", getpid());
-
-	error = write(fd, buf, strlen(buf));
-	if (error <= 0) {
-		fprintf(stderr, "cannot write lock file %s\n", LOCKFILE_NAME);
-		exit(EXIT_FAILURE);
-	}
-}
-
-static void print_usage(void)
-{
-	printf("Usage:\n");
-	printf("\n");
-	printf("gfs_controld [options]\n");
-	printf("\n");
-	printf("Options:\n");
-	printf("\n");
-	printf("  -D           Enable debugging to stderr and don't fork\n");
-	printf("  -L           Enable debugging to log file\n");
-	printf("  -w <num>     Enable (1) or disable (0) withdraw\n");
-	printf("               Default is %d\n", DEFAULT_ENABLE_WITHDRAW);
-	printf("  -h           Print this help, then exit\n");
-	printf("  -V           Print program version information, then exit\n");
-}
-
-#define OPTION_STRING "LDw:hV"
-
-static void read_arguments(int argc, char **argv)
-{
-	int cont = 1;
-	int optchar;
-
-	while (cont) {
-		optchar = getopt(argc, argv, OPTION_STRING);
-
-		switch (optchar) {
-
-		case 'D':
-			daemon_debug_opt = 1;
-			break;
-
-		case 'L':
-			optd_debug_logfile = 1;
-			cfgd_debug_logfile = 1;
-			break;
-
-		case 'w':
-			optd_enable_withdraw = 1;
-			cfgd_enable_withdraw = atoi(optarg);
-			break;
-
-		case 'h':
-			print_usage();
-			exit(EXIT_SUCCESS);
-			break;
-
-		case 'V':
-			printf("gfs_controld %s (built %s %s)\n",
-				VERSION, __DATE__, __TIME__);
-			printf("%s\n", REDHAT_COPYRIGHT);
-			exit(EXIT_SUCCESS);
-			break;
-
-		case ':':
-		case '?':
-			fprintf(stderr, "Please use '-h' for usage.\n");
-			exit(EXIT_FAILURE);
-			break;
-
-		case EOF:
-			cont = 0;
-			break;
-
-		default:
-			fprintf(stderr, "unknown option: %c\n", optchar);
-			exit(EXIT_FAILURE);
-			break;
-		};
-	}
-
-	if (getenv("GFS_CONTROLD_DEBUG")) {
-		optd_debug_logfile = 1;
-		cfgd_debug_logfile = 1;
-	}
-}
-
-static void set_scheduler(void)
-{
-	struct sched_param sched_param;
-	int rv;
-
-	rv = sched_get_priority_max(SCHED_RR);
-	if (rv != -1) {
-		sched_param.sched_priority = rv;
-		rv = sched_setscheduler(0, SCHED_RR, &sched_param);
-		if (rv == -1)
-			log_error("could not set SCHED_RR priority %d err %d",
-				   sched_param.sched_priority, errno);
-	} else {
-		log_error("could not get maximum scheduler priority err %d",
-			  errno);
-	}
-}
-
-int main(int argc, char **argv)
-{
-	INIT_LIST_HEAD(&mountgroups);
-	INIT_LIST_HEAD(&withdrawn_mounts);
-
-	read_arguments(argc, argv);
-
-	if (!daemon_debug_opt) {
-		if (daemon(0, 0) < 0) {
-			perror("daemon error");
-			exit(EXIT_FAILURE);
-		}
-	}
-	lockfile();
-	init_logging();
-	log_level(LOG_INFO, "gfs_controld %s started", VERSION);
-	signal(SIGTERM, sigterm_handler);
-	set_scheduler();
-
-	loop();
-
-	return 0;
-}
-
-void daemon_dump_save(void)
-{
-	int len, i;
-
-	len = strlen(daemon_debug_buf);
-
-	for (i = 0; i < len; i++) {
-		dump_buf[dump_point++] = daemon_debug_buf[i];
-
-		if (dump_point == GFSC_DUMP_SIZE) {
-			dump_point = 0;
-			dump_wrap = 1;
-		}
-	}
-}
-
-int daemon_debug_opt;
-int daemon_quit;
-int cluster_down;
-int poll_dlm;
-struct list_head mountgroups;
-int our_nodeid;
-char clustername[1024];
-char daemon_debug_buf[256];
-char dump_buf[GFSC_DUMP_SIZE];
-int dump_point;
-int dump_wrap;
-int dmsetup_wait;
-cpg_handle_t cpg_handle_daemon;
-int libcpg_flow_control_on;
-struct list_head withdrawn_mounts;
-
diff --git a/group/gfs_controld/member_cman.c b/group/gfs_controld/member_cman.c
deleted file mode 100644
index 07fb982..0000000
--- a/group/gfs_controld/member_cman.c
+++ /dev/null
@@ -1,207 +0,0 @@
-#include "gfs_daemon.h"
-#include "config.h"
-#include <corosync/corotypes.h>
-#include <corosync/cfg.h>
-#include <corosync/quorum.h>
-
-static corosync_cfg_handle_t	ch;
-static quorum_handle_t		qh;
-static uint32_t			old_nodes[MAX_NODES];
-static int			old_node_count;
-static uint32_t			quorum_nodes[MAX_NODES];
-static int			quorum_node_count;
-
-static int is_member(uint32_t *node_list, int count, uint32_t nodeid)
-{
-	int i;
-
-	for (i = 0; i < count; i++) {
-		if (node_list[i] == nodeid)
-			return 1;
-	}
-	return 0;
-}
-
-static int is_old_member(uint32_t nodeid)
-{
-	return is_member(old_nodes, old_node_count, nodeid);
-}
-
-static int is_cluster_member(uint32_t nodeid)
-{
-	return is_member(quorum_nodes, quorum_node_count, nodeid);
-}
-
-static void quorum_callback(quorum_handle_t h, uint32_t quorate,
-			    uint64_t ring_seq, uint32_t node_list_entries,
-			    uint32_t *node_list)
-{
-	int i;
-
-	old_node_count = quorum_node_count;
-	memcpy(&old_nodes, &quorum_nodes, sizeof(old_nodes));
-
-	quorum_node_count = 0;
-	memset(&quorum_nodes, 0, sizeof(quorum_nodes));
-
-	for (i = 0; i < node_list_entries; i++)
-		quorum_nodes[quorum_node_count++] = node_list[i];
-
-	for (i = 0; i < old_node_count; i++) {
-		if (!is_cluster_member(old_nodes[i])) {
-			log_debug("cluster node %u removed", old_nodes[i]);
-			node_history_cluster_remove(old_nodes[i]);
-		}
-	}
-
-	for (i = 0; i < quorum_node_count; i++) {
-		if (!is_old_member(quorum_nodes[i])) {
-			log_debug("cluster node %u added", quorum_nodes[i]);
-			node_history_cluster_add(quorum_nodes[i]);
-		}
-	}
-}
-
-static quorum_callbacks_t quorum_callbacks =
-{
-	.quorum_notify_fn = quorum_callback,
-};
-
-void process_cluster(int ci)
-{
-	cs_error_t err;
-
-	err = quorum_dispatch(qh, CS_DISPATCH_ALL);
-	if (err != CS_OK)
-		cluster_dead(0);
-}
-
-/* Force re-read of quorum nodes */
-void update_cluster(void)
-{
-	cs_error_t err;
-
-	err = quorum_dispatch(qh, CS_DISPATCH_ONE);
-	if (err != CS_OK)
-		cluster_dead(0);
-}
-
-int setup_cluster(void)
-{
-	cs_error_t err;
-	int fd;
-
-	err = quorum_initialize(&qh, &quorum_callbacks);
-	if (err != CS_OK) {
-		log_error("quorum init error %d", err);
-		return -1;
-	}
-
-	err = quorum_fd_get(qh, &fd);
-	if (err != CS_OK) {
-		log_error("quorum fd_get error %d", err);
-		goto fail;
-	}
-
-	err = quorum_trackstart(qh, CS_TRACK_CHANGES);
-	if (err != CS_OK) {
-		log_error("quorum trackstart error %d", err);
-		goto fail;
-	}
-
-	old_node_count = 0;
-	memset(&old_nodes, 0, sizeof(old_nodes));
-	quorum_node_count = 0;
-	memset(&quorum_nodes, 0, sizeof(quorum_nodes));
-
-	return fd;
- fail:
-	quorum_finalize(qh);
-	return -1;
-}
-
-void close_cluster(void)
-{
-	quorum_trackstop(qh);
-	quorum_finalize(qh);
-}
-
-void kick_node_from_cluster(int nodeid)
-{
-	if (!nodeid) {
-		log_error("telling corosync to shut down cluster locally");
-		corosync_cfg_try_shutdown(ch,
-				COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
-	} else {
-		log_error("telling corosync to remove nodeid %d from cluster",
-			  nodeid);
-		corosync_cfg_kill_node(ch, nodeid, "gfs_controld");
-	}
-}
-
-static void shutdown_callback(corosync_cfg_handle_t h,
-			      corosync_cfg_shutdown_flags_t flags)
-{
-	if (flags == COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) {
-		if (list_empty(&mountgroups))
-			corosync_cfg_replyto_shutdown(ch,
-					COROSYNC_CFG_SHUTDOWN_FLAG_YES);
-		else {
-			log_debug("no to corosync shutdown");
-			corosync_cfg_replyto_shutdown(ch,
-					COROSYNC_CFG_SHUTDOWN_FLAG_NO);
-		}
-	}
-}
-
-static corosync_cfg_callbacks_t cfg_callbacks =
-{
-	.corosync_cfg_shutdown_callback = shutdown_callback,
-	.corosync_cfg_state_track_callback = NULL,
-};
-
-void process_cluster_cfg(int ci)
-{
-	cs_error_t err;
-
-	err = corosync_cfg_dispatch(ch, CS_DISPATCH_ALL);
-	if (err != CS_OK)
-		cluster_dead(0);
-}
-
-int setup_cluster_cfg(void)
-{
-	cs_error_t err;
-	unsigned int nodeid;
-	int fd;
-
-	err = corosync_cfg_initialize(&ch, &cfg_callbacks);
-	if (err != CS_OK) {
-		log_error("corosync cfg init error %d", err);
-		return -1;
-	}
-
-	err = corosync_cfg_fd_get(ch, &fd);
-	if (err != CS_OK) {
-		log_error("corosync cfg fd_get error %d", err);
-		corosync_cfg_finalize(ch);
-		return -1;
-	}
-
-	err = corosync_cfg_local_get(ch, &nodeid);
-	if (err != CS_OK) {
-		log_error("corosync cfg local_get error %d", err);
-		corosync_cfg_finalize(ch);
-		return -1;
-	}
-	our_nodeid = nodeid;
-	log_debug("our_nodeid %d", our_nodeid);
-
-	return fd;
-}
-
-void close_cluster_cfg(void)
-{
-	corosync_cfg_finalize(ch);
-}
-
diff --git a/group/gfs_controld/target.mk b/group/gfs_controld/target.mk
deleted file mode 100644
index 6d04792..0000000
--- a/group/gfs_controld/target.mk
+++ /dev/null
@@ -1,3 +0,0 @@
-
-$(eval $(call make-binary,/usr/sbin/gfs_controld,group/libgfscontrol/libgfscontrol.a -llogthread -lcpg -lpthread -lccs -lfenced -lcfg -ldlmcontrol -lcman -lquorum))
-
diff --git a/group/gfs_controld/util.c b/group/gfs_controld/util.c
deleted file mode 100644
index f77f85b..0000000
--- a/group/gfs_controld/util.c
+++ /dev/null
@@ -1,266 +0,0 @@
-#include "gfs_daemon.h"
-#include "config.h"
-#include "libfenced.h"
-
-void update_flow_control_status(void)
-{
-	cpg_flow_control_state_t flow_control_state;
-	cpg_error_t error;
-
-	error = cpg_flow_control_state_get(cpg_handle_daemon, &flow_control_state);
-	if (error != CPG_OK) {
-		log_error("cpg_flow_control_state_get %d", error);
-		return;
-	}
-
-	if (flow_control_state == CPG_FLOW_CONTROL_ENABLED) {
-		if (libcpg_flow_control_on == 0) {
-			log_debug("flow control on");
-		}
-		libcpg_flow_control_on = 1;
-	} else {
-		if (libcpg_flow_control_on) {
-			log_debug("flow control off");
-		}
-		libcpg_flow_control_on = 0;
-	}
-}
-
-int we_are_in_fence_domain(void)
-{
-	struct fenced_node nodeinfo;
-	int rv;
-
-	memset(&nodeinfo, 0, sizeof(nodeinfo));
-
-	rv = fenced_node_info(our_nodeid, &nodeinfo);
-	if (rv < 0) {
-		log_debug("fenced_node_info error %d", rv);
-		return 0;
-	}
-
-	if (nodeinfo.member)
-		return 1;
-	return 0;
-}
-
-#define SYSFS_DIR       "/sys/fs"
-#define SYSFS_BUFLEN    64
-
-int set_sysfs(struct mountgroup *mg, const char *field, int val)
-{
-	char fname[PATH_MAX];
-	char out[SYSFS_BUFLEN];
-	int rv, fd;
-
-	snprintf(fname, PATH_MAX, "%s/%s/%s/lock_module/%s",
-		 SYSFS_DIR, mg->mount_args.type, mg->mount_args.table, field);
-
-	log_group(mg, "set %s to %d", fname, val);
-
-	fd = open(fname, O_RDWR);
-	if (fd < 0) {
-		log_group(mg, "set open %s error %d %d", fname, fd, errno);
-		return -1;
-	}
-
-	memset(out, 0, sizeof(out));
-	sprintf(out, "%d", val);
-
-	rv = write(fd, out, strlen(out));
-	if (rv < 0)
-		log_group(mg, "set write %s error %d", fname, errno);
-	close(fd);
-
-	return 0;
-}
-
-int run_dmsetup_suspend(struct mountgroup *mg, char *dev)
-{
-	struct sched_param sched_param;
-	char fname[PATH_MAX];
-	char smajor[16];
-	char sminor[16];
-	pid_t pid;
-	int i;
-	int major, minor;
-	FILE *fp;
-
-	snprintf(fname, PATH_MAX, "%s/%s/%s/device/dev",
-		 SYSFS_DIR, mg->mount_args.type, mg->mount_args.table);
-
-	fp = fopen(fname, "r");
-	if (fp == NULL) {
-		log_group(mg, "set open %s error %d", fname, errno);
-		return -1;
-	}
-
-	if (fscanf(fp, "%d:%d", &major, &minor) != 2) {
-		log_group(mg, "cannot read device numbers %d", errno);
-		return -1;
-	}
-
-	fclose(fp);
-
-	log_group(mg, "run_dmsetup_suspend %d:%d", major, minor);
-
-	snprintf(smajor, 16, "%d", major);
-	snprintf(sminor, 16, "%d", minor);
-
-	pid = fork();
-	if (pid < 0)
-		return -1;
-
-	if (pid) {
-		mg->dmsetup_wait = 1;
-		mg->dmsetup_pid = pid;
-		return 0;
-	} else {
-		sched_param.sched_priority = 0;
-		sched_setscheduler(0, SCHED_OTHER, &sched_param);
-
-		for (i = 0; i < 50; i++)
-			close(i);
-
-		execlp("dmsetup", "dmsetup", "suspend",  "--nolockfs",
-		       "--noflush", "-j", smajor, "-m", sminor, NULL);
-		exit(EXIT_FAILURE);
-	}
-	return -1;
-}
-
-static void dmsetup_suspend_done(struct mountgroup *mg, int rv)
-{
-	log_group(mg, "dmsetup_suspend_done result %d", rv);
-	mg->dmsetup_wait = 0;
-	mg->dmsetup_pid = 0;
-
-	if (!rv) {
-		mg->withdraw_suspend = 1;
-		send_withdraw(mg);
-	}
-}
-
-void update_dmsetup_wait(void)
-{
-	struct mountgroup *mg;
-	int status;
-	int waiting = 0;
-	pid_t pid;
-
-	list_for_each_entry(mg, &mountgroups, list) {
-		if (mg->dmsetup_wait) {
-			pid = waitpid(mg->dmsetup_pid, &status, WNOHANG);
-
-			/* process not exited yet */
-			if (!pid) {
-				waiting++;
-				continue;
-			}
-
-			if (pid < 0) {
-				log_error("update_dmsetup_wait %s: waitpid %d "
-					  "error %d", mg->name,
-					  mg->dmsetup_pid, errno);
-				dmsetup_suspend_done(mg, -2);
-				continue;
-			}
-
-			/* process exited */
-
-			if (!WIFEXITED(status) || WEXITSTATUS(status))
-				dmsetup_suspend_done(mg, -1);
-			else
-				dmsetup_suspend_done(mg, 0);
-		}
-	}
-
-	if (!waiting) {
-		dmsetup_wait = 0;
-		log_debug("dmsetup_wait off");
-	}
-}
-
-static int ignore_nolock(const char *sysfs_dir, char *table)
-{
-	char path[PATH_MAX];
-	char buf[32];
-	int fd, rv;
-
-	memset(path, 0, PATH_MAX);
-
-	snprintf(path, PATH_MAX, "%s/%s/lock_module/proto_name",
-		 sysfs_dir, table);
-
-	/* lock_nolock doesn't create the "lock_module" dir at all,
-	   so we'll fail to open this */
-
-	fd = open(path, O_RDONLY);
-	if (fd < 0)
-		return 1;
-
-	memset(buf, 0, sizeof(buf));
-
-	rv = read(fd, buf, sizeof(buf));
-	close(fd);
-	if (rv < 0)
-		return 1;
-
-	if (!strncmp(buf, "lock_nolock", 11))
-		return 1;
-
-	return 0;
-}
-
-/* This is for the case where gfs_controld exits/fails, abandoning gfs
-   filesystems in the kernel, and then gfs_controld is restarted.  When
-   gfs_controld exits and abandons lockspaces, that node needs to be
-   rebooted to clear the uncontrolled filesystems from the kernel. */
-
-int check_uncontrolled_filesystems(void)
-{
-	DIR *d;
-	struct dirent *de;
-	int count = 0;
-
-	d = opendir("/sys/fs/gfs/");
-	if (!d)
-		goto gfs2;
-
-	while ((de = readdir(d))) {
-		if (de->d_name[0] == '.')
-			continue;
-
-		if (ignore_nolock("/sys/fs/gfs/", de->d_name))
-			continue;
-
-		log_error("found uncontrolled gfs fs %s", de->d_name);
-		count++;
-	}
-	closedir(d);
-
- gfs2:
-	d = opendir("/sys/fs/gfs2/");
-	if (!d)
-		goto out;
-
-	while ((de = readdir(d))) {
-		if (de->d_name[0] == '.')
-			continue;
-
-		if (ignore_nolock("/sys/fs/gfs2/", de->d_name))
-			continue;
-
-		log_error("found uncontrolled gfs2 fs %s", de->d_name);
-		count++;
-	}
-	closedir(d);
-
- out:
-	if (count) {
-		kick_node_from_cluster(our_nodeid);
-		return -1;
-	}
-	return 0;
-}
-
diff --git a/group/include/Makefile.am b/group/include/Makefile.am
deleted file mode 100644
index b2c3ce5..0000000
--- a/group/include/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-MAINTAINERCLEANFILES	= Makefile.in
-
-noinst_HEADERS		= linux_endian.h list.h
diff --git a/group/include/linux_endian.h b/group/include/linux_endian.h
deleted file mode 100644
index 43089d2..0000000
--- a/group/include/linux_endian.h
+++ /dev/null
@@ -1,68 +0,0 @@
-#ifndef __LINUX_ENDIAN_DOT_H__
-#define __LINUX_ENDIAN_DOT_H__
-
-
-#include <endian.h>
-#include <byteswap.h>
-
-
-/*  I'm not sure which versions of alpha glibc/gcc are broken,
-    so fix all of them.  */
-#ifdef __alpha__
-#undef bswap_64
-static __inline__ unsigned long bswap_64(unsigned long x)
-{
-  unsigned int h = x >> 32;
-  unsigned int l = x;
-
-  h = bswap_32(h);
-  l = bswap_32(l);
-
-  return ((unsigned long)l << 32) | h;
-}
-#endif  /*  __alpha__  */
-
-
-#if __BYTE_ORDER == __BIG_ENDIAN
-
-#define be16_to_cpu(x) (x)
-#define be32_to_cpu(x) (x)
-#define be64_to_cpu(x) (x)
-
-#define cpu_to_be16(x) (x)
-#define cpu_to_be32(x) (x)
-#define cpu_to_be64(x) (x)
-
-#define le16_to_cpu(x) (bswap_16((x)))
-#define le32_to_cpu(x) (bswap_32((x)))
-#define le64_to_cpu(x) (bswap_64((x)))
-
-#define cpu_to_le16(x) (bswap_16((x)))
-#define cpu_to_le32(x) (bswap_32((x)))
-#define cpu_to_le64(x) (bswap_64((x)))
-
-#endif  /*  __BYTE_ORDER == __BIG_ENDIAN  */
-
-
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-
-#define be16_to_cpu(x) (bswap_16((x)))
-#define be32_to_cpu(x) (bswap_32((x)))
-#define be64_to_cpu(x) (bswap_64((x)))
-
-#define cpu_to_be16(x) (bswap_16((x)))
-#define cpu_to_be32(x) (bswap_32((x)))
-#define cpu_to_be64(x) (bswap_64((x))) 
-
-#define le16_to_cpu(x) (x)
-#define le32_to_cpu(x) (x)
-#define le64_to_cpu(x) (x)
-
-#define cpu_to_le16(x) (x)
-#define cpu_to_le32(x) (x)
-#define cpu_to_le64(x) (x)
-
-#endif  /*  __BYTE_ORDER == __LITTLE_ENDIAN  */
-
-
-#endif  /*  __LINUX_ENDIAN_DOT_H__  */
diff --git a/group/include/list.h b/group/include/list.h
deleted file mode 100644
index 8100cbc..0000000
--- a/group/include/list.h
+++ /dev/null
@@ -1,336 +0,0 @@
-/* Copied from include/linux/list.h */
-
-#ifndef _LINUX_LIST_H
-#define _LINUX_LIST_H
-
-/**
- * container_of - cast a member of a structure out to the containing structure
- *
- * @ptr:        the pointer to the member.
- * @type:       the type of the container struct this is embedded in.
- * @member:     the name of the member within the struct.
- *
- */
-#define container_of(ptr, type, member) ({                      \
-	const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
-	(type *)( (char *)__mptr - offsetof(type,member) );})
-
-
-/*
- * These are non-NULL pointers that will result in page faults
- * under normal circumstances, used to verify that nobody uses
- * non-initialized list entries.
- */
-#define LIST_POISON1  ((void *) 0x00100100)
-#define LIST_POISON2  ((void *) 0x00200200)
-
-/*
- * Simple doubly linked list implementation.
- *
- * Some of the internal functions ("__xxx") are useful when
- * manipulating whole lists rather than single entries, as
- * sometimes we already know the next/prev entries and we can
- * generate better code by using them directly rather than
- * using the generic single-entry routines.
- */
-
-struct list_head {
-	struct list_head *next, *prev;
-};
-
-#define LIST_HEAD_INIT(name) { &(name), &(name) }
-
-#define LIST_HEAD(name) \
-	struct list_head name = LIST_HEAD_INIT(name)
-
-#define INIT_LIST_HEAD(ptr) do { \
-	(ptr)->next = (ptr); (ptr)->prev = (ptr); \
-} while (0)
-
-/*
- * Insert a new entry between two known consecutive entries. 
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_add(struct list_head *new,
-			      struct list_head *prev,
-			      struct list_head *next)
-{
-	next->prev = new;
-	new->next = next;
-	new->prev = prev;
-	prev->next = new;
-}
-
-/**
- * list_add - add a new entry
- * @new: new entry to be added
- * @head: list head to add it after
- *
- * Insert a new entry after the specified head.
- * This is good for implementing stacks.
- */
-static inline void list_add(struct list_head *new, struct list_head *head)
-{
-	__list_add(new, head, head->next);
-}
-
-/**
- * list_add_tail - add a new entry
- * @new: new entry to be added
- * @head: list head to add it before
- *
- * Insert a new entry before the specified head.
- * This is useful for implementing queues.
- */
-static inline void list_add_tail(struct list_head *new, struct list_head *head)
-{
-	__list_add(new, head->prev, head);
-}
-
-/*
- * Delete a list entry by making the prev/next entries
- * point to each other.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_del(struct list_head * prev, struct list_head * next)
-{
-	next->prev = prev;
-	prev->next = next;
-}
-
-/**
- * list_del - deletes entry from list.
- * @entry: the element to delete from the list.
- * Note: list_empty on entry does not return true after this, the entry is
- * in an undefined state.
- */
-static inline void list_del(struct list_head *entry)
-{
-	__list_del(entry->prev, entry->next);
-	entry->next = LIST_POISON1;
-	entry->prev = LIST_POISON2;
-}
-
-/**
- * list_del_init - deletes entry from list and reinitialize it.
- * @entry: the element to delete from the list.
- */
-static inline void list_del_init(struct list_head *entry)
-{
-	__list_del(entry->prev, entry->next);
-	INIT_LIST_HEAD(entry); 
-}
-
-/**
- * list_move - delete from one list and add as another's head
- * @list: the entry to move
- * @head: the head that will precede our entry
- */
-static inline void list_move(struct list_head *list, struct list_head *head)
-{
-        __list_del(list->prev, list->next);
-        list_add(list, head);
-}
-
-/**
- * list_move_tail - delete from one list and add as another's tail
- * @list: the entry to move
- * @head: the head that will follow our entry
- */
-static inline void list_move_tail(struct list_head *list,
-				  struct list_head *head)
-{
-        __list_del(list->prev, list->next);
-        list_add_tail(list, head);
-}
-
-/**
- * list_empty - tests whether a list is empty
- * @head: the list to test.
- */
-static inline int list_empty(const struct list_head *head)
-{
-	return head->next == head;
-}
-
-/**
- * list_empty_careful - tests whether a list is
- * empty _and_ checks that no other CPU might be
- * in the process of still modifying either member
- *
- * NOTE: using list_empty_careful() without synchronization
- * can only be safe if the only activity that can happen
- * to the list entry is list_del_init(). Eg. it cannot be used
- * if another CPU could re-list_add() it.
- *
- * @head: the list to test.
- */
-static inline int list_empty_careful(const struct list_head *head)
-{
-	struct list_head *next = head->next;
-	return (next == head) && (next == head->prev);
-}
-
-static inline void __list_splice(struct list_head *list,
-				 struct list_head *head)
-{
-	struct list_head *first = list->next;
-	struct list_head *last = list->prev;
-	struct list_head *at = head->next;
-
-	first->prev = head;
-	head->next = first;
-
-	last->next = at;
-	at->prev = last;
-}
-
-/**
- * list_splice - join two lists
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice(struct list_head *list, struct list_head *head)
-{
-	if (!list_empty(list))
-		__list_splice(list, head);
-}
-
-/**
- * list_splice_init - join two lists and reinitialise the emptied list.
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- *
- * The list at @list is reinitialised
- */
-static inline void list_splice_init(struct list_head *list,
-				    struct list_head *head)
-{
-	if (!list_empty(list)) {
-		__list_splice(list, head);
-		INIT_LIST_HEAD(list);
-	}
-}
-
-/**
- * list_entry - get the struct for this entry
- * @ptr:	the &struct list_head pointer.
- * @type:	the type of the struct this is embedded in.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_entry(ptr, type, member) \
-	container_of(ptr, type, member)
-
-/**
- * list_first_entry - get the first element from a list
- * @ptr:        the list head to take the element from.
- * @type:       the type of the struct this is embedded in.
- * @member:     the name of the list_struct within the struct.
- *
- * Note, that list is expected to be not empty.
- */
-#define list_first_entry(ptr, type, member) \
-	list_entry((ptr)->next, type, member)
-
-/**
- * list_for_each	-	iterate over a list
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- */
-#define list_for_each(pos, head) \
-	for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/**
- * __list_for_each	-	iterate over a list
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- *
- * This variant differs from list_for_each() in that it's the
- * simplest possible list iteration code, no prefetching is done.
- * Use this for code that knows the list to be very short (empty
- * or 1 entry) most of the time.
- */
-#define __list_for_each(pos, head) \
-	for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/**
- * list_for_each_prev	-	iterate over a list backwards
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- */
-#define list_for_each_prev(pos, head) \
-	for (pos = (head)->prev; pos != (head); pos = pos->prev)
-        	
-/**
- * list_for_each_safe	-	iterate over a list safe against removal of list entry
- * @pos:	the &struct list_head to use as a loop counter.
- * @n:		another &struct list_head to use as temporary storage
- * @head:	the head for your list.
- */
-#define list_for_each_safe(pos, n, head) \
-	for (pos = (head)->next, n = pos->next; pos != (head); \
-		pos = n, n = pos->next)
-
-/**
- * list_for_each_entry	-	iterate over list of given type
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry(pos, head, member)				\
-	for (pos = list_entry((head)->next, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_reverse - iterate backwards over list of given type.
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_reverse(pos, head, member)			\
-	for (pos = list_entry((head)->prev, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = list_entry(pos->member.prev, typeof(*pos), member))
-
-/**
- * list_prepare_entry - prepare a pos entry for use as a start point in
- *			list_for_each_entry_continue
- * @pos:	the type * to use as a start point
- * @head:	the head of the list
- * @member:	the name of the list_struct within the struct.
- */
-#define list_prepare_entry(pos, head, member) \
-	((pos) ? : list_entry(head, typeof(*pos), member))
-
-/**
- * list_for_each_entry_continue -	iterate over list of given type
- *			continuing after existing point
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_continue(pos, head, member) 		\
-	for (pos = list_entry(pos->member.next, typeof(*pos), member);  \
-	     &pos->member != (head);					\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe(pos, n, head, member)			\
-	for (pos = list_entry((head)->next, typeof(*pos), member),	\
-		n = list_entry(pos->member.next, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-
-#endif
diff --git a/group/libgfscontrol/Makefile.am b/group/libgfscontrol/Makefile.am
deleted file mode 100644
index 9e6eb0d..0000000
--- a/group/libgfscontrol/Makefile.am
+++ /dev/null
@@ -1,9 +0,0 @@
-MAINTAINERCLEANFILES	= Makefile.in
-
-noinst_HEADERS		= libgfscontrol.h
-
-noinst_LTLIBRARIES	= libgfscontrol.la
-
-libgfscontrol_la_SOURCES  = main.c
-
-libgfscontrol_la_CPPFLAGS = -I$(top_srcdir)/group/gfs_controld
diff --git a/group/libgfscontrol/libgfscontrol.h b/group/libgfscontrol/libgfscontrol.h
deleted file mode 100644
index e5bb969..0000000
--- a/group/libgfscontrol/libgfscontrol.h
+++ /dev/null
@@ -1,122 +0,0 @@
-#ifndef _LIBGFSCONTROL_H_
-#define _LIBGFSCONTROL_H_
-
-/* Maximum mountgroup name length, should match DLM_LOCKSPACE_LEN from
-   linux/dlmconstants.h.  The libcpg limit is larger at
-   CPG_MAX_NAME_LENGTH 128.  Our cpg name includes a "gfs:" prefix before
-   the mountgroup name. */
-
-#define GFS_MOUNTGROUP_LEN	64
-
-#define GFSC_DUMP_SIZE		(1024 * 1024)
-
-#define GFSC_NF_MEMBER			0x00000001 /* node is member in cg */
-#define GFSC_NF_START			0x00000002 /* start message recvd */
-#define GFSC_NF_DISALLOWED		0x00000004 /* node disallowed in cg */
-#define GFSC_NF_KERNEL_MOUNT_DONE	0x00000008
-#define GFSC_NF_KERNEL_MOUNT_ERROR	0x00000010
-#define GFSC_NF_READONLY		0x00000020
-#define GFSC_NF_SPECTATOR		0x00000040
-#define GFSC_NF_CHECK_DLM		0x00000080
-
-struct gfsc_node {
-	int nodeid;
-	int jid;
-	uint32_t flags;
-	uint32_t added_seq;
-	uint32_t removed_seq;
-	int failed_reason;
-};
-
-struct gfsc_change {
-	int member_count;
-	int joined_count;
-	int remove_count;
-	int failed_count;
-	int wait_condition;	/* 0 no, 1 fencing, 2 quorum, 3 fs */
-	int wait_messages;	/* 0 no, 1 yes */
-	uint32_t seq;
-	uint32_t combined_seq;
-};
-
-#define GFSC_MF_JOINING			0x00000001
-#define GFSC_MF_LEAVING			0x00000002
-#define GFSC_MF_KERNEL_STOPPED		0x00000004
-#define GFSC_MF_KERNEL_MOUNT_DONE	0x00000008
-#define GFSC_MF_KERNEL_MOUNT_ERROR	0x00000010
-#define GFSC_MF_FIRST_RECOVERY_NEEDED	0x00000020
-#define GFSC_MF_FIRST_RECOVERY_MSG	0x00000040
-#define GFSC_MF_LOCAL_RECOVERY_BUSY	0x00000080
-
-struct gfsc_mountgroup {
-	int group_mode;
-	struct gfsc_change cg_prev;	/* completed change (started_change) */
-	struct gfsc_change cg_next;	/* in-progress change (changes list) */
-	int journals_need_recovery;	/* count of jounals need_recovery */
-	uint32_t flags;
-	uint32_t global_id;
-	char name[GFS_MOUNTGROUP_LEN+1];
-};
-
-/* gfsc_mountgroup_nodes() types
-
-   MEMBERS: members in completed (prev) change,
-            zero if there's no completed (prev) change
-   NEXT:    members in in-progress (next) change,
-            zero if there's no in-progress (next) change
-   ALL:     NEXT + nonmembers if there's an in-progress (next) change,
-            MEMBERS + nonmembers if there's no in-progress (next) change, but
-            there is a completed (prev) change
-            nonmembers if there's no in-progress (next) or completed (prev)
-            change (possible?)
-
-   gfsc_node_info() returns info for in-progress (next) change, if one exists,
-   otherwise it returns info for completed (prev) change.
-*/
-
-#define GFSC_NODES_ALL		1
-#define GFSC_NODES_MEMBERS	2
-#define GFSC_NODES_NEXT		3
-
-int gfsc_dump_debug(char *buf);
-int gfsc_dump_plocks(char *name, char *buf);
-int gfsc_mountgroup_info(char *mgname, struct gfsc_mountgroup *mg);
-int gfsc_node_info(char *mgname, int nodeid, struct gfsc_node *node);
-int gfsc_mountgroups(int max, int *count, struct gfsc_mountgroup *mgs);
-int gfsc_mountgroup_nodes(char *mgname, int type, int max, int *count,
-			 struct gfsc_node *nodes);
-
-struct gfsc_mount_args {
-	char dir[PATH_MAX];
-	char type[PATH_MAX];
-	char proto[PATH_MAX];
-	char table[PATH_MAX];
-	char options[PATH_MAX];
-	char dev[PATH_MAX];
-	char hostdata[PATH_MAX];
-};
-
-/*
- * mount.gfs connects to gfs_controld,
- * mount.gfs tells gfs_controld to do a join or remount,
- * mount.gfs reads the result of the join or remount from gfs_controld,
- * mount.gfs tells gfs_controld the result of the mount(2),
- * mount.gfs disconnects from gfs_controld
- */
-
-int gfsc_fs_connect(void);
-int gfsc_fs_join(int fd, struct gfsc_mount_args *ma);
-int gfsc_fs_remount(int fd, struct gfsc_mount_args *ma);
-int gfsc_fs_result(int fd, int *result, struct gfsc_mount_args *ma);
-int gfsc_fs_mount_done(int fd, struct gfsc_mount_args *ma, int result);
-void gfsc_fs_disconnect(int fd);
-
-/*
- * mount.gfs tells gfs_controld to do a leave (due to a mount failure)
- * for unmount, gfs_controld leaves due to a message from the kernel
- */
-
-int gfsc_fs_leave(struct gfsc_mount_args *ma, int reason);
-
-#endif
-
diff --git a/group/libgfscontrol/main.c b/group/libgfscontrol/main.c
deleted file mode 100644
index 96a8e03..0000000
--- a/group/libgfscontrol/main.c
+++ /dev/null
@@ -1,436 +0,0 @@
-#include "clusterautoconfig.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <limits.h>
-
-#include "libgfscontrol.h"
-#include "gfs_controld.h"
-
-static int do_read(int fd, void *buf, size_t count)
-{
-	int rv, off = 0;
-
-	while (off < count) {
-		rv = read(fd, (char *)buf + off, count - off);
-		if (rv == 0)
-			return -1;
-		if (rv == -1 && errno == EINTR)
-			continue;
-		if (rv == -1)
-			return -1;
-		off += rv;
-	}
-	return 0;
-}
-
-static int do_write(int fd, void *buf, size_t count)
-{
-	int rv, off = 0;
-
- retry:
-	rv = write(fd, (char *)buf + off, count);
-	if (rv == -1 && errno == EINTR)
-		goto retry;
-	if (rv < 0) {
-		return rv;
-	}
-
-	if (rv != count) {
-		count -= rv;
-		off += rv;
-		goto retry;
-	}
-	return 0;
-}
-
-static int do_connect(const char *sock_path)
-{
-	struct sockaddr_un sun;
-	socklen_t addrlen;
-	int rv, fd;
-
-	fd = socket(PF_UNIX, SOCK_STREAM, 0);
-	if (fd < 0)
-		goto out;
-
-	memset(&sun, 0, sizeof(sun));
-	sun.sun_family = AF_UNIX;
-	strcpy(&sun.sun_path[1], sock_path);
-	addrlen = sizeof(sa_family_t) + strlen(sun.sun_path+1) + 1;
-
-	rv = connect(fd, (struct sockaddr *) &sun, addrlen);
-	if (rv < 0) {
-		close(fd);
-		fd = rv;
-	}
- out:
-	return fd;
-}
-
-static void init_header(struct gfsc_header *h, int cmd, char *name,
-			int extra_len)
-{
-	memset(h, 0, sizeof(struct gfsc_header));
-
-	h->magic = GFSC_MAGIC;
-	h->version = GFSC_VERSION;
-	h->len = sizeof(struct gfsc_header) + extra_len;
-	h->command = cmd;
-
-	if (name)
-		strncpy(h->name, name, GFS_MOUNTGROUP_LEN);
-}
-
-static int do_dump(int cmd, char *name, char *buf)
-{
-	struct gfsc_header h, *rh;
-	char *reply;
-	int reply_len;
-	int fd, rv;
-
-	init_header(&h, cmd, name, 0);
-
-	reply_len = sizeof(struct gfsc_header) + GFSC_DUMP_SIZE;
-	reply = malloc(reply_len);
-	if (!reply) {
-		rv = -1;
-		goto out;
-	}
-	memset(reply, 0, reply_len);
-
-	fd = do_connect(GFSC_QUERY_SOCK_PATH);
-	if (fd < 0) {
-		rv = fd;
-		goto out_free;
-	}
-
-	rv = do_write(fd, &h, sizeof(h));
-	if (rv < 0)
-		goto out_close;
-
-	/* won't always get back the full reply_len */
-	do_read(fd, reply, reply_len);
-
-	rh = (struct gfsc_header *)reply;
-	rv = rh->data;
-	if (rv < 0)
-		goto out_close;
-
-	memcpy(buf, (char *)reply + sizeof(struct gfsc_header),
-	       GFSC_DUMP_SIZE);
- out_close:
-	close(fd);
- out_free:
-	free(reply);
- out:
-	return rv;
-}
-
-int gfsc_dump_debug(char *buf)
-{
-	return do_dump(GFSC_CMD_DUMP_DEBUG, NULL, buf);
-}
-
-int gfsc_dump_plocks(char *name, char *buf)
-{
-	return do_dump(GFSC_CMD_DUMP_PLOCKS, name, buf);
-}
-
-int gfsc_node_info(char *name, int nodeid, struct gfsc_node *node)
-{
-	struct gfsc_header h, *rh;
-	char reply[sizeof(struct gfsc_header) + sizeof(struct gfsc_node)];
-	int fd, rv;
-
-	init_header(&h, GFSC_CMD_NODE_INFO, name, 0);
-	h.data = nodeid;
-
-	memset(reply, 0, sizeof(reply));
-
-	fd = do_connect(GFSC_QUERY_SOCK_PATH);
-	if (fd < 0) {
-		rv = fd;
-		goto out;
-	}
-
-	rv = do_write(fd, &h, sizeof(h));
-	if (rv < 0)
-		goto out_close;
-
-	rv = do_read(fd, reply, sizeof(reply));
-	if (rv < 0)
-		goto out_close;
-
-	rh = (struct gfsc_header *)reply;
-	rv = rh->data;
-	if (rv < 0)
-		goto out_close;
-
-	memcpy(node, (char *)reply + sizeof(struct gfsc_header),
-	       sizeof(struct gfsc_node));
- out_close:
-	close(fd);
- out:
-	return rv;
-}
-
-int gfsc_mountgroup_info(char *name, struct gfsc_mountgroup *mountgroup)
-{
-	struct gfsc_header h, *rh;
-	char reply[sizeof(struct gfsc_header) + sizeof(struct gfsc_mountgroup)];
-	int fd, rv;
-
-	init_header(&h, GFSC_CMD_MOUNTGROUP_INFO, name, 0);
-
-	memset(reply, 0, sizeof(reply));
-
-	fd = do_connect(GFSC_QUERY_SOCK_PATH);
-	if (fd < 0) {
-		rv = fd;
-		goto out;
-	}
-
-	rv = do_write(fd, &h, sizeof(h));
-	if (rv < 0)
-		goto out_close;
-
-	rv = do_read(fd, reply, sizeof(reply));
-	if (rv < 0)
-		goto out_close;
-
-	rh = (struct gfsc_header *)reply;
-	rv = rh->data;
-	if (rv < 0)
-		goto out_close;
-
-	memcpy(mountgroup, (char *)reply + sizeof(struct gfsc_header),
-	       sizeof(struct gfsc_mountgroup));
- out_close:
-	close(fd);
- out:
-	return rv;
-}
-
-int gfsc_mountgroups(int max, int *count, struct gfsc_mountgroup *mgs)
-{
-	struct gfsc_header h, *rh;
-	char *reply;
-	int reply_len;
-	int fd, rv, result, mg_count;
-
-	init_header(&h, GFSC_CMD_MOUNTGROUPS, NULL, 0);
-	h.data = max;
-
-	reply_len = sizeof(struct gfsc_header) +
-		    (max * sizeof(struct gfsc_mountgroup));
-	reply = malloc(reply_len);
-	if (!reply) {
-		rv = -1;
-		goto out;
-	}
-	memset(reply, 0, reply_len);
-
-	fd = do_connect(GFSC_QUERY_SOCK_PATH);
-	if (fd < 0) {
-		rv = fd;
-		goto out_free;
-	}
-
-	rv = do_write(fd, &h, sizeof(h));
-	if (rv < 0)
-		goto out_close;
-
-	/* won't usually get back the full reply_len */
-	do_read(fd, reply, reply_len);
-
-	rh = (struct gfsc_header *)reply;
-	result = rh->data;
-	if (result < 0 && result != -E2BIG) {
-		rv = result;
-		goto out_close;
-	}
-
-	if (result == -E2BIG) {
-		*count = -E2BIG;
-		mg_count = max;
-	} else {
-		*count = result;
-		mg_count = result;
-	}
-	rv = 0;
-
-	memcpy(mgs, (char *)reply + sizeof(struct gfsc_header),
-	       mg_count * sizeof(struct gfsc_mountgroup));
- out_close:
-	close(fd);
- out_free:
-	free(reply);
- out:
-	return rv;
-}
-
-int gfsc_mountgroup_nodes(char *name, int type, int max, int *count,
-			 struct gfsc_node *nodes)
-{
-	struct gfsc_header h, *rh;
-	char *reply;
-	int reply_len;
-	int fd, rv, result, node_count;
-
-	init_header(&h, GFSC_CMD_MOUNTGROUP_NODES, name, 0);
-	h.option = type;
-	h.data = max;
-
-	reply_len = sizeof(struct gfsc_header) +
-		    (max * sizeof(struct gfsc_node));
-	reply = malloc(reply_len);
-	if (!reply) {
-		rv = -1;
-		goto out;
-	}
-	memset(reply, 0, reply_len);
-
-	fd = do_connect(GFSC_QUERY_SOCK_PATH);
-	if (fd < 0) {
-		rv = fd;
-		goto out_free;
-	}
-
-	rv = do_write(fd, &h, sizeof(h));
-	if (rv < 0)
-		goto out_close;
-
-	/* won't usually get back the full reply_len */
-	do_read(fd, reply, reply_len);
-
-	rh = (struct gfsc_header *)reply;
-	result = rh->data;
-	if (result < 0 && result != -E2BIG) {
-		rv = result;
-		goto out_close;
-	}
-
-	if (result == -E2BIG) {
-		*count = -E2BIG;
-		node_count = max;
-	} else {
-		*count = result;
-		node_count = result;
-	}
-	rv = 0;
-
-	memcpy(nodes, (char *)reply + sizeof(struct gfsc_header),
-	       node_count * sizeof(struct gfsc_node));
- out_close:
-	close(fd);
- out_free:
-	free(reply);
- out:
-	return rv;
-}
-
-int gfsc_fs_connect(void)
-{
-	return do_connect(GFSC_SOCK_PATH);
-}
-
-void gfsc_fs_disconnect(int fd)
-{
-	close(fd);
-}
-
-int gfsc_fs_join(int fd, struct gfsc_mount_args *ma)
-{
-	char msg[sizeof(struct gfsc_header) + sizeof(struct gfsc_mount_args)];
-	struct gfsc_header *h = (struct gfsc_header *)msg;
-	char *name = strstr(ma->table, ":") + 1;
-
-	init_header(h, GFSC_CMD_FS_JOIN, name, sizeof(struct gfsc_mount_args));
-
-	memcpy(msg + sizeof(struct gfsc_header), ma,
-	       sizeof(struct gfsc_mount_args));
-
-	return do_write(fd, msg, sizeof(msg));
-}
-
-int gfsc_fs_remount(int fd, struct gfsc_mount_args *ma)
-{
-	char msg[sizeof(struct gfsc_header) + sizeof(struct gfsc_mount_args)];
-	struct gfsc_header *h = (struct gfsc_header *)msg;
-	char *name = strstr(ma->table, ":") + 1;
-
-	init_header(h, GFSC_CMD_FS_REMOUNT, name,
-		    sizeof(struct gfsc_mount_args));
-
-	memcpy(msg + sizeof(struct gfsc_header), ma,
-	       sizeof(struct gfsc_mount_args));
-
-	return do_write(fd, msg, sizeof(msg));
-}
-
-int gfsc_fs_result(int fd, int *result, struct gfsc_mount_args *ma)
-{
-	char reply[sizeof(struct gfsc_header) + sizeof(struct gfsc_mount_args)];
-	struct gfsc_header *h = (struct gfsc_header *)reply;
-	int rv;
-
-	rv = do_read(fd, reply, sizeof(reply));
-	if (rv < 0)
-		goto out;
-
-	*result = h->data;
-
-	memcpy(ma, reply + sizeof(struct gfsc_header),
-	       sizeof(struct gfsc_mount_args));
- out:
-	return rv;
-}
-
-int gfsc_fs_mount_done(int fd, struct gfsc_mount_args *ma, int result)
-{
-	char msg[sizeof(struct gfsc_header) + sizeof(struct gfsc_mount_args)];
-	struct gfsc_header *h = (struct gfsc_header *)msg;
-	char *name = strstr(ma->table, ":") + 1;
-
-	init_header(h, GFSC_CMD_FS_MOUNT_DONE, name,
-		    sizeof(struct gfsc_mount_args));
-
-	h->data = result;
-
-	memcpy(msg + sizeof(struct gfsc_header), ma,
-	       sizeof(struct gfsc_mount_args));
-
-	return do_write(fd, msg, sizeof(msg));
-}
-
-int gfsc_fs_leave(struct gfsc_mount_args *ma, int reason)
-{
-	char msg[sizeof(struct gfsc_header) + sizeof(struct gfsc_mount_args)];
-	struct gfsc_header *h = (struct gfsc_header *)msg;
-	char *name = strstr(ma->table, ":") + 1;
-	int fd, err;
-
-	init_header(h, GFSC_CMD_FS_LEAVE, name,
-		    sizeof(struct gfsc_mount_args));
-
-	h->data = reason;
-
-	memcpy(msg + sizeof(struct gfsc_header), ma,
-	       sizeof(struct gfsc_mount_args));
-
-	fd = do_connect(GFSC_SOCK_PATH);
-	if (fd < 0)
-		return fd;
-
-	err = do_write(fd, msg, sizeof(msg));
-	close(fd);
-	return err;
-}
-
diff --git a/group/libgfscontrol/target.mk b/group/libgfscontrol/target.mk
deleted file mode 100644
index 23bd3ac..0000000
--- a/group/libgfscontrol/target.mk
+++ /dev/null
@@ -1,3 +0,0 @@
-
-$(eval $(call make-library,libgfscontrol.a))
-
diff --git a/group/man/Makefile.am b/group/man/Makefile.am
deleted file mode 100644
index 87d81e5..0000000
--- a/group/man/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-MAINTAINERCLEANFILES	= Makefile.in
-
-dist_man_MANS		= gfs_controld.8
diff --git a/group/man/gfs_controld.8 b/group/man/gfs_controld.8
deleted file mode 100644
index 3d053ee..0000000
--- a/group/man/gfs_controld.8
+++ /dev/null
@@ -1,122 +0,0 @@
-.TH gfs_controld 8
-
-.SH NAME
-gfs_controld - daemon that manages mounting, unmounting, recovery and
-posix locks
-
-.SH SYNOPSIS
-.B
-gfs_controld
-[\fIOPTION\fR]...
-
-.SH DESCRIPTION
-GFS lives in the kernel, and the cluster infrastructure (cluster
-membership and group management) lives in user space.  GFS in the kernel
-needs to adjust/recover for certain cluster events.  It's the job of
-gfs_controld to receive these events and reconfigure gfs as needed.
-gfs_controld controls and configures gfs through sysfs files that are
-considered gfs-internal interfaces; not a general API/ABI.
-
-Mounting, unmounting and node failure are the main cluster events that
-gfs_controld controls.  It also manages the assignment of journals to
-different nodes.  The mount.gfs and umount.gfs programs communicate with
-gfs_controld to join/leave the mount group and receive the necessary
-options for the kernel mount.
-
-GFS also sends all posix lock operations to gfs_controld for processing.
-gfs_controld manages cluster-wide posix locks for gfs and passes results
-back to gfs in the kernel.
-
-.SH CONFIGURATION FILE
-
-Optional cluster.conf settings are placed in the <gfs_controld> section.
-
-.SS Posix locks
-
-Heavy use of plocks can result in high network load.  The rate at which
-plocks are processed are limited by the
-.I plock_rate_limit
-setting, which limits the maximum plock performance, and limits potentially
-excessive network load.  This value is the maximum number of plock operations
-a single node will process every second.  To achieve maximum posix locking
-performance, the rate limiting should be disabled by setting it to 0.  The
-default value is 100.
-
-  <gfs_controld plock_rate_limit="100"/>
-
-To optimize performance for repeated locking of the same locks by
-processes on a single node,
-.I plock_ownership
-can be set to 1.  The default is 0.  If this is enabled, gfs_controld
-cannot interoperate with older versions that did not support this option.
-
-  <gfs_controld plock_ownership="1"/>
-
-Three options can be used to tune the behavior of the plock_ownership
-optimization.  All three relate to the caching of lock ownership state.
-Specifically, they define how aggressively cached ownership state is dropped.
-More caching of ownership state can result in better performance, at the
-expense of more memory usage.
-
-.I drop_resources_time
-is the frequency of drop attempts in milliseconds.  Default 10000 (10 sec).
-
-.I drop_resources_count
-is the maximum number of items to drop from the cache each time.  Default 10.
-
-.I drop_resources_age
-is the time in milliseconds a cached item should be unused before being
-considered for dropping.  Default 10000 (10 sec).
-
-  <gfs_controld drop_resources_time="10000" drop_resources_count="10"
-   drop_resources_age="10000"/>
-
-
-.SH OPTIONS
-.TP
-\fB-D\fP
-Run the daemon in the foreground and print debug statements to stdout.
-.TP
-\fB-P\fP
-Enable posix lock debugging messages.
-.TP
-\fB-w\fP
-Disable the "withdraw" feature.
-.TP
-\fB-p\fP
-Disable posix lock handling.
-.TP
-\fB-l\fP <num>
-Limit the rate at which posix lock messages are sent to <num> messages per
-second.  0 disables the limit and results in the maximum performance of
-posix locks. Default 100.
-.TP
-\fB-o\fP <num>
-Enable (1) or disable (0) plock ownership optimization. Default 0.  All
-nodes must run with the same value.
-.TP
-\fB-t\fP <ms>
-Ownership cache tuning, drop resources time (milliseconds). Default 10000.
-.TP
-\fB-c\fP <ms>
-Ownership cache tuning, drop resources count. Default 10.
-.TP
-\fB-a\fP <ms>
-Ownership cache tuning, drop resources age (milliseconds). Default 10000.
-.TP
-\fB-h\fP 
-Print out a help message describing available options, then exit.
-.TP
-\fB-V\fP
-Print the version information and exit.
-
-.SH DEBUGGING 
-The gfs_controld daemon keeps a circular buffer of debug messages that can
-be dumped with the 'group_tool dump gfs' command.
-
-The state of all gfs posix locks can also be dumped from gfs_controld with
-the 'group_tool dump plocks <fsname>' command.
-
-.SH SEE ALSO
-groupd(8), group_tool(8)
-
diff --git a/group/man/target.mk b/group/man/target.mk
deleted file mode 100644
index 2567e81..0000000
--- a/group/man/target.mk
+++ /dev/null
@@ -1,3 +0,0 @@
-
-manpages += *.[0-9]
-
diff --git a/tests/Makefile.am b/tests/Makefile.am
index a97b8c5..af01c49 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,5 +1,6 @@
 TESTS_ENVIRONMENT	= TOPBUILDDIR=$(top_builddir)
 EXTRA_DIST		= tool_tests.sh
+CLEANFILES		= tests.log
 
 if BUILD_TESTS
 check_PROGRAMS		= check_libgfs2


More information about the cluster-commits mailing list