Gitweb:
http://git.fedorahosted.org/git/dlm.git?p=dlm.git;a=commitdiff;h=42a8ff3e...
Commit: 42a8ff3ed7d88a7e7a81d12bb905512f5e9c4a27
Parent: 2b5e4f87ca394113fd54395d5b5b3ab1e410dbb2
Author: David Teigland <teigland(a)redhat.com>
AuthorDate: Fri Sep 30 13:24:54 2011 -0500
Committer: David Teigland <teigland(a)redhat.com>
CommitterDate: Fri Sep 30 13:24:54 2011 -0500
dlm_controld: move source files
Signed-off-by: David Teigland <teigland(a)redhat.com>
---
dlm_controld/Makefile | 50 +
dlm_controld/action.c | 1084 +++++++++++++++
dlm_controld/config.c | 93 ++
dlm_controld/config.h | 47 +
dlm_controld/cpg.c | 2606 +++++++++++++++++++++++++++++++++++++
dlm_controld/crc.c | 72 +
dlm_controld/deadlock.c | 1550 ++++++++++++++++++++++
dlm_controld/dlm_controld.8 | 313 +++++
dlm_controld/dlm_controld.h | 38 +
dlm_controld/dlm_daemon.h | 326 +++++
dlm_controld/endian.h | 68 +
dlm_controld/list.h | 336 +++++
dlm_controld/logging.c | 173 +++
dlm_controld/main.c | 1274 ++++++++++++++++++
dlm_controld/member_cman.c | 267 ++++
dlm_controld/netlink.c | 225 ++++
dlm_controld/plock.c | 2197 +++++++++++++++++++++++++++++++
dlm_controld/rbtree.c | 383 ++++++
dlm_controld/rbtree.h | 160 +++
group/dlm_controld/Makefile | 50 -
group/dlm_controld/action.c | 1084 ---------------
group/dlm_controld/config.c | 93 --
group/dlm_controld/config.h | 47 -
group/dlm_controld/cpg.c | 2606 -------------------------------------
group/dlm_controld/crc.c | 72 -
group/dlm_controld/deadlock.c | 1550 ----------------------
group/dlm_controld/dlm_controld.h | 38 -
group/dlm_controld/dlm_daemon.h | 326 -----
group/dlm_controld/logging.c | 173 ---
group/dlm_controld/main.c | 1274 ------------------
group/dlm_controld/member_cman.c | 267 ----
group/dlm_controld/netlink.c | 225 ----
group/dlm_controld/plock.c | 2197 -------------------------------
group/dlm_controld/rbtree.c | 383 ------
group/include/linux_endian.h | 68 -
group/include/list.h | 336 -----
group/include/rbtree.h | 160 ---
group/man/dlm_controld.8 | 313 -----
38 files changed, 11262 insertions(+), 11262 deletions(-)
diff --git a/dlm_controld/Makefile b/dlm_controld/Makefile
new file mode 100644
index 0000000..63dcb9b
--- /dev/null
+++ b/dlm_controld/Makefile
@@ -0,0 +1,50 @@
+TARGET = dlm_controld
+
+SOURCE = \
+ action.c \
+ cpg.c \
+ crc.c \
+ main.c \
+ plock.c \
+ config.c \
+ member_cman.c \
+ logging.c \
+ rbtree.c
+
+CFLAGS += -D_GNU_SOURCE -g \
+ -Wall \
+ -Wformat \
+ -Wformat-security \
+ -Wmissing-prototypes \
+ -Wnested-externs \
+ -Wpointer-arith \
+ -Wextra -Wshadow \
+ -Wcast-align \
+ -Wwrite-strings \
+ -Waggregate-return \
+ -Wstrict-prototypes \
+ -Winline \
+ -Wredundant-decls \
+ -Wno-sign-compare \
+ -Wno-unused-parameter \
+ -Wp,-D_FORTIFY_SOURCE=2 \
+ -fexceptions \
+ -fasynchronous-unwind-tables \
+ -fdiagnostics-show-option \
+
+CFLAGS += -fPIE -DPIE
+CFLAGS += `xml2-config --cflags`
+CFLAGS += -I../include -I../../dlm/libdlm -I../../dlm/libdlmcontrol
+
+LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
+LDFLAGS += `xml2-config --libs`
+LDFLAGS += -lpthread -llogthread -lcpg -lconfdb -lcfg -lquorum -lfenced
+
+all: $(TARGET)
+
+$(TARGET): $(SOURCE)
+ $(CC) $(CFLAGS) $(LDFLAGS) $(SOURCE) -o $@ -L.
+
+clean:
+ rm -f *.o *.so *.so.* $(TARGET)
+
diff --git a/dlm_controld/action.c b/dlm_controld/action.c
new file mode 100644
index 0000000..e9148a1
--- /dev/null
+++ b/dlm_controld/action.c
@@ -0,0 +1,1084 @@
+#include "dlm_daemon.h"
+
+#include <corosync/corotypes.h>
+#include <corosync/confdb.h>
+
+static int dir_members[MAX_NODES];
+static int dir_members_count;
+static int comms_nodes[MAX_NODES];
+static int comms_nodes_count;
+static char mg_name[DLM_LOCKSPACE_LEN+1];
+
+#define DLM_SYSFS_DIR "/sys/kernel/dlm"
+#define CLUSTER_DIR "/sys/kernel/config/dlm/cluster"
+#define SPACES_DIR "/sys/kernel/config/dlm/cluster/spaces"
+#define COMMS_DIR "/sys/kernel/config/dlm/cluster/comms"
+
+static int detect_protocol(void)
+{
+ confdb_handle_t handle;
+ hdb_handle_t totem_handle;
+ char key_value[256];
+ size_t value_len;
+ int rv, proto = -1;
+ confdb_callbacks_t callbacks = {
+ .confdb_key_change_notify_fn = NULL,
+ .confdb_object_create_change_notify_fn = NULL,
+ .confdb_object_delete_change_notify_fn = NULL
+ };
+
+ rv = confdb_initialize(&handle, &callbacks);
+ if (rv != CS_OK) {
+ log_error("confdb_initialize error %d", rv);
+ return -1;
+ }
+
+ rv = confdb_object_find_start(handle, OBJECT_PARENT_HANDLE);
+ if (rv != CS_OK) {
+ log_error("confdb_object_find_start error %d", rv);
+ goto out;
+ }
+
+ rv = confdb_object_find(handle, OBJECT_PARENT_HANDLE,
+ "totem", strlen("totem"), &totem_handle);
+ if (rv != CS_OK) {
+ log_error("confdb_object_find error %d", rv);
+ goto out;
+ }
+
+ rv = confdb_key_get(handle, totem_handle,
+ "rrp_mode", strlen("rrp_mode"),
+ key_value, &value_len);
+ if (rv != CS_OK) {
+ log_error("confdb_key_get error %d", rv);
+ goto out;
+ }
+
+ key_value[value_len] = '\0';
+ log_debug("totem/rrp_mode = '%s'", key_value);
+
+ if (!strcmp(key_value, "none"))
+ proto = PROTO_TCP;
+ else
+ proto = PROTO_SCTP;
+ out:
+ confdb_finalize(handle);
+ return proto;
+}
+
+/* look for an id that matches in e.g. /sys/fs/gfs/bull\:x/lock_module/id
+ and then extract the "x" as the name */
+
+static int get_mountgroup_name(uint32_t mg_id)
+{
+ char path[PATH_MAX];
+ char *fsname;
+ const char *fsdir;
+ DIR *d;
+ FILE *file;
+ struct dirent *de;
+ uint32_t id;
+ int retry_gfs2 = 1;
+ int rv, error;
+
+ fsdir = "/sys/fs/gfs";
+ retry:
+ rv = -1;
+
+ d = opendir(fsdir);
+ if (!d) {
+ log_debug("%s: opendir failed: %d", path, errno);
+ goto out;
+ }
+
+ while ((de = readdir(d))) {
+ if (de->d_name[0] == '.')
+ continue;
+
+ id = 0;
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s/lock_module/id",
+ fsdir, de->d_name);
+
+ file = fopen(path, "r");
+ if (!file) {
+ log_error("can't open %s %d", path, errno);
+ continue;
+ }
+
+ error = fscanf(file, "%u", &id);
+ fclose(file);
+
+ if (error != 1) {
+ log_error("bad read %s %d", path, errno);
+ continue;
+ }
+ if (id != mg_id) {
+ log_debug("get_mountgroup_name skip %x %s",
+ id, de->d_name);
+ continue;
+ }
+
+ /* take the fsname out of clustername:fsname */
+ fsname = strstr(de->d_name, ":");
+ if (!fsname) {
+ log_debug("get_mountgroup_name skip2 %x %s",
+ id, de->d_name);
+ continue;
+ }
+ fsname++;
+
+ log_debug("get_mountgroup_name found %x %s %s",
+ id, de->d_name, fsname);
+ strncpy(mg_name, fsname, sizeof(mg_name));
+ rv = 0;
+ break;
+ }
+
+ closedir(d);
+
+ out:
+ if (rv && retry_gfs2) {
+ retry_gfs2 = 0;
+ fsdir = "/sys/fs/gfs2";
+ goto retry;
+ }
+
+ return rv;
+}
+
+/* This is for the case where dlm_controld exits/fails, abandoning dlm
+ lockspaces in the kernel, and then dlm_controld is restarted. When
+ dlm_controld exits and abandons lockspaces, that node needs to be
+ rebooted to clear the uncontrolled lockspaces from the kernel. */
+
+int check_uncontrolled_lockspaces(void)
+{
+ DIR *d;
+ struct dirent *de;
+ int count = 0;
+
+ d = opendir(DLM_SYSFS_DIR);
+ if (!d)
+ return 0;
+
+ while ((de = readdir(d))) {
+ if (de->d_name[0] == '.')
+ continue;
+
+ log_error("found uncontrolled lockspace %s", de->d_name);
+ count++;
+ }
+ closedir(d);
+
+ if (count) {
+ kick_node_from_cluster(our_nodeid);
+ return -1;
+ }
+ return 0;
+}
+
+/* find the mountgroup with "mg_id" in sysfs, get it's name, then look for
+ the ls with with the same name in lockspaces list, return its id */
+
+void set_associated_id(uint32_t mg_id)
+{
+ struct lockspace *ls;
+ int rv;
+
+ log_debug("set_associated_id mg_id %x %d", mg_id, mg_id);
+
+ memset(&mg_name, 0, sizeof(mg_name));
+
+ rv = get_mountgroup_name(mg_id);
+ if (rv) {
+ log_error("no mountgroup found with id %x", mg_id);
+ return;
+ }
+
+ ls = find_ls(mg_name);
+ if (!ls) {
+ log_error("no lockspace found with name %s for mg_id %x",
+ mg_name, mg_id);
+ return;
+ }
+
+ log_debug("set_associated_id mg %x is ls %x", mg_id, ls->global_id);
+
+ ls->associated_mg_id = mg_id;
+}
+
+static int do_sysfs(const char *name, const char *file, char *val)
+{
+ char fname[512];
+ int rv, fd;
+
+ sprintf(fname, "%s/%s/%s", DLM_SYSFS_DIR, name, file);
+
+ fd = open(fname, O_WRONLY);
+ if (fd < 0) {
+ log_error("open \"%s\" error %d %d", fname, fd, errno);
+ return -1;
+ }
+
+ log_debug("write \"%s\" to \"%s\"", val, fname);
+
+ rv = do_write(fd, val, strlen(val) + 1);
+ close(fd);
+ return rv;
+}
+
+int set_sysfs_control(char *name, int val)
+{
+ char buf[32];
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%d", val);
+
+ return do_sysfs(name, "control", buf);
+}
+
+int set_sysfs_event_done(char *name, int val)
+{
+ char buf[32];
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%d", val);
+
+ return do_sysfs(name, "event_done", buf);
+}
+
+int set_sysfs_id(char *name, uint32_t id)
+{
+ char buf[32];
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%u", id);
+
+ return do_sysfs(name, "id", buf);
+}
+
+static int update_dir_members(char *name)
+{
+ char path[PATH_MAX];
+ DIR *d;
+ struct dirent *de;
+ int i = 0;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s/nodes", SPACES_DIR, name);
+
+ d = opendir(path);
+ if (!d) {
+ log_debug("%s: opendir failed: %d", path, errno);
+ return -1;
+ }
+
+ memset(dir_members, 0, sizeof(dir_members));
+ dir_members_count = 0;
+
+ /* FIXME: we should probably read the nodeid in each dir instead */
+
+ while ((de = readdir(d))) {
+ if (de->d_name[0] == '.')
+ continue;
+ dir_members[i++] = atoi(de->d_name);
+ log_debug("dir_member %d", dir_members[i-1]);
+ }
+ closedir(d);
+
+ dir_members_count = i;
+ return 0;
+}
+
+static int id_exists(int id, int count, int *array)
+{
+ int i;
+ for (i = 0; i < count; i++) {
+ if (array[i] == id)
+ return 1;
+ }
+ return 0;
+}
+
+static int create_path(const char *path)
+{
+ mode_t old_umask;
+ int rv;
+
+ old_umask = umask(0022);
+ rv = mkdir(path, 0777);
+ if (rv < 0 && errno == EEXIST)
+ rv = 0;
+ if (rv < 0)
+ log_error("%s: mkdir failed: %d", path, errno);
+ umask(old_umask);
+ return rv;
+}
+
+int path_exists(const char *path)
+{
+ struct stat buf;
+
+ if (stat(path, &buf) < 0) {
+ if (errno != ENOENT)
+ log_error("%s: stat failed: %d", path, errno);
+ return 0;
+ }
+ return 1;
+}
+
+/* The "renew" nodes are those that have left and rejoined since the last
+ call to set_members(). We rmdir/mkdir for these nodes so dlm-kernel
+ can notice they've left and rejoined. */
+
+int set_configfs_members(char *name, int new_count, int *new_members,
+ int renew_count, int *renew_members)
+{
+ char path[PATH_MAX];
+ char buf[32];
+ int i, w, fd, rv, id, old_count, *old_members;
+ int do_renew;
+
+ /*
+ * create lockspace dir if it doesn't exist yet
+ */
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s", SPACES_DIR, name);
+
+ if (!path_exists(path)) {
+ if (create_path(path))
+ return -1;
+ }
+
+ /*
+ * remove/add lockspace members
+ */
+
+ rv = update_dir_members(name);
+ if (rv)
+ return rv;
+
+ old_members = dir_members;
+ old_count = dir_members_count;
+
+ for (i = 0; i < old_count; i++) {
+ id = old_members[i];
+ if (id_exists(id, new_count, new_members))
+ continue;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s/nodes/%d",
+ SPACES_DIR, name, id);
+
+ log_debug("set_members rmdir \"%s\"", path);
+
+ rv = rmdir(path);
+ if (rv) {
+ log_error("%s: rmdir failed: %d", path, errno);
+ goto out;
+ }
+ }
+
+ /*
+ * remove lockspace dir after we've removed all the nodes
+ * (when we're shutting down and adding no new nodes)
+ */
+
+ if (!new_count) {
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s", SPACES_DIR, name);
+
+ log_debug("set_members lockspace rmdir \"%s\"", path);
+
+ rv = rmdir(path);
+ if (rv)
+ log_error("%s: rmdir failed: %d", path, errno);
+ }
+
+ for (i = 0; i < new_count; i++) {
+ id = new_members[i];
+
+ do_renew = 0;
+
+ if (id_exists(id, renew_count, renew_members))
+ do_renew = 1;
+ else if (id_exists(id, old_count, old_members))
+ continue;
+
+ if (!is_cluster_member(id))
+ update_cluster();
+ /*
+ * create node's dir
+ */
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s/nodes/%d",
+ SPACES_DIR, name, id);
+
+ if (do_renew) {
+ log_debug("set_members renew rmdir \"%s\"", path);
+ rv = rmdir(path);
+ if (rv) {
+ log_error("%s: renew rmdir failed: %d",
+ path, errno);
+ goto out;
+ }
+ }
+
+ log_debug("set_members mkdir \"%s\"", path);
+
+ rv = create_path(path);
+ if (rv)
+ goto out;
+
+ /*
+ * set node's nodeid
+ */
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s/nodes/%d/nodeid",
+ SPACES_DIR, name, id);
+
+ rv = fd = open(path, O_WRONLY);
+ if (rv < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ goto out;
+ }
+
+ memset(buf, 0, 32);
+ snprintf(buf, 32, "%d", id);
+
+ rv = do_write(fd, buf, strlen(buf));
+ if (rv < 0) {
+ log_error("%s: write failed: %d, %s", path, errno, buf);
+ close(fd);
+ goto out;
+ }
+ close(fd);
+
+ /*
+ * set node's weight
+ */
+
+ w = get_weight(id, name);
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s/nodes/%d/weight",
+ SPACES_DIR, name, id);
+
+ rv = fd = open(path, O_WRONLY);
+ if (rv < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ goto out;
+ }
+
+ memset(buf, 0, 32);
+ snprintf(buf, 32, "%d", w);
+
+ rv = do_write(fd, buf, strlen(buf));
+ if (rv < 0) {
+ log_error("%s: write failed: %d, %s", path, errno, buf);
+ close(fd);
+ goto out;
+ }
+ close(fd);
+ }
+
+ rv = 0;
+ out:
+ return rv;
+}
+
+#if 0
+char *str_ip(char *addr)
+{
+ static char ip[256];
+ struct sockaddr_in *sin = (struct sockaddr_in *) addr;
+ memset(ip, 0, sizeof(ip));
+ inet_ntop(AF_INET, &sin->sin_addr, ip, 256);
+ return ip;
+}
+#endif
+
+static char *str_ip(char *addr)
+{
+ static char str_ip_buf[INET6_ADDRSTRLEN];
+ struct sockaddr_storage *ss = (struct sockaddr_storage *)addr;
+ struct sockaddr_in *sin = (struct sockaddr_in *)addr;
+ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
+ void *saddr;
+
+ if (ss->ss_family == AF_INET6)
+ saddr = &sin6->sin6_addr;
+ else
+ saddr = &sin->sin_addr;
+
+ inet_ntop(ss->ss_family, saddr, str_ip_buf, sizeof(str_ip_buf));
+ return str_ip_buf;
+}
+
+/* record the nodeids that are currently listed under
+ config/dlm/cluster/comms/ so that we can remove all of them */
+
+static int update_comms_nodes(void)
+{
+ char path[PATH_MAX];
+ DIR *d;
+ struct dirent *de;
+ int i = 0;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, COMMS_DIR);
+
+ d = opendir(path);
+ if (!d) {
+ log_debug("%s: opendir failed: %d", path, errno);
+ return -1;
+ }
+
+ memset(comms_nodes, 0, sizeof(comms_nodes));
+ comms_nodes_count = 0;
+
+ while ((de = readdir(d))) {
+ if (de->d_name[0] == '.')
+ continue;
+ comms_nodes[i++] = atoi(de->d_name);
+ }
+ closedir(d);
+
+ comms_nodes_count = i;
+ return 0;
+}
+
+/* clear out everything under config/dlm/cluster/comms/ */
+
+static void clear_configfs_comms(void)
+{
+ char path[PATH_MAX];
+ int i, rv;
+
+ rv = update_comms_nodes();
+ if (rv < 0)
+ return;
+
+ for (i = 0; i < comms_nodes_count; i++) {
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%d", COMMS_DIR, comms_nodes[i]);
+
+ log_debug("clear_configfs_nodes rmdir \"%s\"", path);
+
+ rv = rmdir(path);
+ if (rv)
+ log_error("%s: rmdir failed: %d", path, errno);
+ }
+}
+
+static void clear_configfs_space_nodes(char *name)
+{
+ char path[PATH_MAX];
+ int i, rv;
+
+ rv = update_dir_members(name);
+ if (rv < 0)
+ return;
+
+ for (i = 0; i < dir_members_count; i++) {
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s/nodes/%d",
+ SPACES_DIR, name, dir_members[i]);
+
+ log_debug("clear_configfs_space_nodes rmdir \"%s\"", path);
+
+ rv = rmdir(path);
+ if (rv)
+ log_error("%s: rmdir failed: %d", path, errno);
+ }
+}
+
+/* clear out everything under config/dlm/cluster/spaces/ */
+
+static void clear_configfs_spaces(void)
+{
+ char path[PATH_MAX];
+ DIR *d;
+ struct dirent *de;
+ int rv;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s", SPACES_DIR);
+
+ d = opendir(path);
+ if (!d) {
+ log_debug("%s: opendir failed: %d", path, errno);
+ return;
+ }
+
+ while ((de = readdir(d))) {
+ if (de->d_name[0] == '.')
+ continue;
+
+ clear_configfs_space_nodes(de->d_name);
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%s", SPACES_DIR, de->d_name);
+
+ log_debug("clear_configfs_spaces rmdir \"%s\"", path);
+
+ rv = rmdir(path);
+ if (rv)
+ log_error("%s: rmdir failed: %d", path, errno);
+ }
+ closedir(d);
+}
+
+static int add_configfs_base(void)
+{
+ int rv = 0;
+
+ if (!path_exists("/sys/kernel/config")) {
+ log_error("No /sys/kernel/config, is configfs loaded?");
+ return -1;
+ }
+
+ if (!path_exists("/sys/kernel/config/dlm")) {
+ log_error("No /sys/kernel/config/dlm, is the dlm loaded?");
+ return -1;
+ }
+
+ if (!path_exists("/sys/kernel/config/dlm/cluster"))
+ rv = create_path("/sys/kernel/config/dlm/cluster");
+
+ return rv;
+}
+
+int add_configfs_node(int nodeid, char *addr, int addrlen, int local)
+{
+ char path[PATH_MAX];
+ char padded_addr[sizeof(struct sockaddr_storage)];
+ char buf[32];
+ int rv, fd;
+
+ log_debug("set_configfs_node %d %s local %d",
+ nodeid, str_ip(addr), local);
+
+ /*
+ * create comm dir for this node
+ */
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%d", COMMS_DIR, nodeid);
+
+ rv = create_path(path);
+ if (rv)
+ return -1;
+
+ /*
+ * set the nodeid
+ */
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%d/nodeid", COMMS_DIR, nodeid);
+
+ fd = open(path, O_WRONLY);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return -1;
+ }
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%d", nodeid);
+
+ rv = do_write(fd, buf, strlen(buf));
+ if (rv < 0) {
+ log_error("%s: write failed: %d, %s", path, errno, buf);
+ close(fd);
+ return -1;
+ }
+ close(fd);
+
+ /*
+ * set the address
+ */
+
+ memset(padded_addr, 0, sizeof(padded_addr));
+ memcpy(padded_addr, addr, addrlen);
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%d/addr", COMMS_DIR, nodeid);
+
+ fd = open(path, O_WRONLY);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return -1;
+ }
+
+ rv = do_write(fd, padded_addr, sizeof(struct sockaddr_storage));
+ if (rv < 0) {
+ log_error("%s: write failed: %d %d", path, errno, rv);
+ close(fd);
+ return -1;
+ }
+ close(fd);
+
+ /*
+ * set local
+ */
+
+ if (!local)
+ goto out;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%d/local", COMMS_DIR, nodeid);
+
+ fd = open(path, O_WRONLY);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return -1;
+ }
+
+ rv = do_write(fd, (void *)"1", strlen("1"));
+ if (rv < 0) {
+ log_error("%s: write failed: %d", path, errno);
+ close(fd);
+ return -1;
+ }
+ close(fd);
+ out:
+ return 0;
+}
+
+void del_configfs_node(int nodeid)
+{
+ char path[PATH_MAX];
+ int rv;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/%d", COMMS_DIR, nodeid);
+
+ log_debug("del_configfs_node rmdir \"%s\"", path);
+
+ rv = rmdir(path);
+ if (rv)
+ log_error("%s: rmdir failed: %d", path, errno);
+}
+
+static int set_configfs_protocol(int proto)
+{
+ char path[PATH_MAX];
+ char buf[32];
+ int fd, rv;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/protocol", CLUSTER_DIR);
+
+ fd = open(path, O_WRONLY);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return fd;
+ }
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%d", proto);
+
+ rv = do_write(fd, buf, strlen(buf));
+ if (rv < 0) {
+ log_error("%s: write failed: %d", path, errno);
+ return rv;
+ }
+ close(fd);
+ log_debug("set protocol %d", proto);
+ return 0;
+}
+
+static int set_configfs_timewarn(int cs)
+{
+ char path[PATH_MAX];
+ char buf[32];
+ int fd, rv;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/timewarn_cs", CLUSTER_DIR);
+
+ fd = open(path, O_WRONLY);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return fd;
+ }
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%d", cs);
+
+ rv = do_write(fd, buf, strlen(buf));
+ if (rv < 0) {
+ log_error("%s: write failed: %d", path, errno);
+ return rv;
+ }
+ close(fd);
+ log_debug("set timewarn_cs %d", cs);
+ return 0;
+}
+
+static int set_configfs_debug(int val)
+{
+ char path[PATH_MAX];
+ char buf[32];
+ int fd, rv;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "%s/log_debug", CLUSTER_DIR);
+
+ fd = open(path, O_WRONLY);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return fd;
+ }
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%d", val);
+
+ rv = do_write(fd, buf, strlen(buf));
+ if (rv < 0) {
+ log_error("%s: write failed: %d", path, errno);
+ return rv;
+ }
+ close(fd);
+ log_debug("set log_debug %d", val);
+ return 0;
+}
+
+#define NET_RMEM_DEFAULT 4194304
+#define NET_RMEM_MAX 4194304
+
+static int set_proc_rmem(void)
+{
+ char path[PATH_MAX];
+ char buf[32];
+ int fd, rv;
+
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "/proc/sys/net/core/rmem_default");
+
+ fd = open(path, O_RDWR);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return fd;
+ }
+
+ memset(buf, 0, sizeof(buf));
+
+ rv = read(fd, buf, sizeof(buf));
+ if (rv < 0) {
+ log_error("%s: read failed: %d", path, errno);
+ close(fd);
+ return rv;
+ }
+
+ if (atoi(buf) >= NET_RMEM_DEFAULT) {
+ close(fd);
+ goto next;
+ }
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%d", NET_RMEM_DEFAULT);
+
+ rv = do_write(fd, buf, strlen(buf));
+ if (rv < 0) {
+ log_error("%s: write failed: %d", path, errno);
+ close(fd);
+ return rv;
+ }
+
+ close(fd);
+ log_debug("set %s %s", path, buf);
+
+ next:
+ memset(path, 0, PATH_MAX);
+ snprintf(path, PATH_MAX, "/proc/sys/net/core/rmem_max");
+
+ fd = open(path, O_RDWR);
+ if (fd < 0) {
+ log_error("%s: open failed: %d", path, errno);
+ return fd;
+ }
+
+ memset(buf, 0, sizeof(buf));
+
+ rv = read(fd, buf, sizeof(buf));
+ if (rv < 0) {
+ log_error("%s: read failed: %d", path, errno);
+ close(fd);
+ return rv;
+ }
+
+ if (atoi(buf) >= NET_RMEM_MAX) {
+ close(fd);
+ goto out;
+ }
+
+ memset(buf, 0, sizeof(buf));
+ snprintf(buf, 32, "%d", NET_RMEM_MAX);
+
+ rv = do_write(fd, buf, strlen(buf));
+ if (rv < 0) {
+ log_error("%s: write failed: %d", path, errno);
+ close(fd);
+ return rv;
+ }
+
+ close(fd);
+ log_debug("set %s %s", path, buf);
+ out:
+ return 0;
+}
+
+void clear_configfs(void)
+{
+ clear_configfs_comms();
+ clear_configfs_spaces();
+ rmdir("/sys/kernel/config/dlm/cluster");
+}
+
+int setup_configfs(void)
+{
+ int rv;
+
+ clear_configfs();
+
+ rv = add_configfs_base();
+ if (rv < 0)
+ return rv;
+
+ /* add configfs entries for existing nodes */
+ update_cluster();
+
+ /* the kernel has its own defaults for these values which we
+ don't want to change unless these have been set; -1 means
+ they have not been set on command line or config file */
+
+ if (cfgk_debug != -1)
+ set_configfs_debug(cfgk_debug);
+ if (cfgk_timewarn != -1)
+ set_configfs_timewarn(cfgk_timewarn);
+
+ if (cfgk_protocol == PROTO_DETECT) {
+ rv = detect_protocol();
+ if (rv == PROTO_TCP || rv == PROTO_SCTP)
+ cfgk_protocol = rv;
+ }
+
+ if (cfgk_protocol == PROTO_TCP || cfgk_protocol == PROTO_SCTP)
+ set_configfs_protocol(cfgk_protocol);
+
+ if (cfgk_protocol == PROTO_SCTP)
+ set_proc_rmem();
+
+ return 0;
+}
+
+static void find_minors(void)
+{
+ FILE *fl;
+ char name[256];
+ uint32_t number;
+ int found = 0;
+ int c;
+
+ control_minor = 0;
+ monitor_minor = 0;
+ plock_minor = 0;
+ old_plock_minor = 0;
+
+ if (!(fl = fopen("/proc/misc", "r"))) {
+ log_error("/proc/misc fopen failed: %s", strerror(errno));
+ return;
+ }
+
+ while (!feof(fl)) {
+ if (fscanf(fl, "%d %255s\n", &number, &name[0]) == 2) {
+
+ if (!strcmp(name, "dlm-control")) {
+ control_minor = number;
+ found++;
+ } else if (!strcmp(name, "dlm-monitor")) {
+ monitor_minor = number;
+ found++;
+ } else if (!strcmp(name, "dlm_plock")) {
+ plock_minor = number;
+ found++;
+ } else if (!strcmp(name, "lock_dlm_plock")) {
+ old_plock_minor = number;
+ found++;
+ }
+
+ } else do {
+ c = fgetc(fl);
+ } while (c != EOF && c != '\n');
+
+ if (found == 3)
+ break;
+ }
+ fclose(fl);
+
+ if (!found)
+ log_error("Is dlm missing from kernel? No misc devices found.");
+}
+
+static int find_udev_device(const char *path, uint32_t minor)
+{
+ struct stat st;
+ int i;
+
+ for (i = 0; i < 10; i++) {
+ if (stat(path, &st) == 0 && minor(st.st_rdev) == minor)
+ return 0;
+ sleep(1);
+ }
+
+ log_error("cannot find device %s with minor %d", path, minor);
+ return -1;
+}
+
+int setup_misc_devices(void)
+{
+ int rv;
+
+ find_minors();
+
+ if (control_minor) {
+ rv = find_udev_device("/dev/misc/dlm-control", control_minor);
+ if (rv < 0)
+ return rv;
+ log_debug("found /dev/misc/dlm-control minor %u",
+ control_minor);
+ }
+
+ if (monitor_minor) {
+ rv = find_udev_device("/dev/misc/dlm-monitor", monitor_minor);
+ if (rv < 0)
+ return rv;
+ log_debug("found /dev/misc/dlm-monitor minor %u",
+ monitor_minor);
+ }
+
+ if (plock_minor) {
+ rv = find_udev_device("/dev/misc/dlm_plock", plock_minor);
+ if (rv < 0)
+ return rv;
+ log_debug("found /dev/misc/dlm_plock minor %u",
+ plock_minor);
+ }
+
+ if (!plock_minor && old_plock_minor) {
+ rv = find_udev_device("/dev/misc/lock_dlm_plock",
+ old_plock_minor);
+ if (rv < 0)
+ return rv;
+ log_debug("found /dev/misc/lock_dlm_plock minor %u",
+ old_plock_minor);
+ }
+
+ return 0;
+}
+
diff --git a/dlm_controld/config.c b/dlm_controld/config.c
new file mode 100644
index 0000000..40b0c87
--- /dev/null
+++ b/dlm_controld/config.c
@@ -0,0 +1,93 @@
+#include "dlm_daemon.h"
+#include <libxml/tree.h>
+
+/* TODO:
+ <dlm>
+ <lockspace name="foo" nodir="1">
+ <master nodeid="1" weight="2"/>
+ <master nodeid="2" weight="3"/>
+ </lockspace>
+ </dlm>
+*/
+
+int get_weight(int nodeid, char *lockspace)
+{
+ /* default weight is 1 */
+ return 1;
+}
+
+static void proto_val(char *str, int *val)
+{
+ if (!strncasecmp(str, "tcp", 3))
+ *val = PROTO_TCP;
+ else if (!strncasecmp(str, "sctp", 4))
+ *val = PROTO_SCTP;
+ else if (!strncasecmp(str, "detect", 6))
+ *val = PROTO_DETECT;
+ else {
+ log_error("invalid protocol value %s", str);
+ }
+}
+
+static void set_val(xmlNode *root, const char *name, int *opt, int *val)
+{
+ xmlChar *str;
+
+ str = xmlGetProp(root, BAD_CAST name);
+ if (str && !(*opt)) {
+ *val = atoi((char *)str);
+ log_debug("config %s = %d", name, *val);
+ }
+}
+
+void setup_config(int update)
+{
+ xmlDoc *doc;
+ xmlNode *root;
+ xmlChar *str;
+
+ if (!path_exists(DLM_CONFIG_FILE))
+ return;
+
+ doc = xmlParseFile(DLM_CONFIG_FILE);
+ if (!doc) {
+ log_error("xml parse error %d %s", errno, DLM_CONFIG_FILE);
+ return;
+ }
+
+ root = xmlDocGetRootElement(doc);
+ if (!root) {
+ log_error("xml root error %d %s", errno, DLM_CONFIG_FILE);
+ xmlFreeDoc(doc);
+ return;
+ }
+
+ if (update)
+ goto do_update;
+
+ /* These config values are set from dlm.conf only if they haven't
+ already been set on the command line. */
+
+ str = xmlGetProp(root, BAD_CAST "protocol");
+ if (str && !optk_protocol) {
+ proto_val((char *)str, &cfgk_protocol);
+ log_debug("config protocol = %d", cfgk_protocol);
+ }
+
+ set_val(root, "log_debug", &optk_debug, &cfgk_debug);
+ set_val(root, "timewarn", &optk_timewarn, &cfgk_timewarn);
+ set_val(root, "enable_fencing", &optd_enable_fencing,
&cfgd_enable_fencing);
+ set_val(root, "enable_quorum", &optd_enable_quorum,
&cfgd_enable_quorum);
+ set_val(root, "enable_plock", &optd_enable_plock,
&cfgd_enable_plock);
+ set_val(root, "plock_ownership", &optd_plock_ownership,
&cfgd_plock_ownership);
+ do_update:
+ /* The following can be changed while running */
+ set_val(root, "plock_debug", &optd_plock_debug, &cfgd_plock_debug);
+ set_val(root, "plock_rate_limit", &optd_plock_rate_limit,
&cfgd_plock_rate_limit);
+ set_val(root, "drop_resources_time", &optd_drop_resources_time,
&cfgd_drop_resources_time);
+ set_val(root, "drop_resources_count", &optd_drop_resources_count,
&cfgd_drop_resources_count);
+ set_val(root, "drop_resources_age", &optd_drop_resources_age,
&cfgd_drop_resources_age);
+
+ xmlFreeDoc(doc);
+}
+
diff --git a/dlm_controld/config.h b/dlm_controld/config.h
new file mode 100644
index 0000000..ab37eed
--- /dev/null
+++ b/dlm_controld/config.h
@@ -0,0 +1,47 @@
+#ifndef __CONFIG_DOT_H__
+#define __CONFIG_DOT_H__
+
+/* the kernel has default values for debug, timewarn and protocol;
+ we only change them if new values are given on command line or in ccs */
+
+#define DEFAULT_DEBUG_LOGFILE 0
+#define DEFAULT_ENABLE_FENCING 1
+#define DEFAULT_ENABLE_QUORUM 0
+#define DEFAULT_ENABLE_PLOCK 1
+#define DEFAULT_PLOCK_DEBUG 0
+#define DEFAULT_PLOCK_RATE_LIMIT 0
+#define DEFAULT_PLOCK_OWNERSHIP 0
+#define DEFAULT_DROP_RESOURCES_TIME 10000 /* 10 sec */
+#define DEFAULT_DROP_RESOURCES_COUNT 10
+#define DEFAULT_DROP_RESOURCES_AGE 10000 /* 10 sec */
+
+extern int optk_debug;
+extern int optk_timewarn;
+extern int optk_protocol;
+extern int optd_debug_logfile;
+extern int optd_enable_fencing;
+extern int optd_enable_quorum;
+extern int optd_enable_plock;
+extern int optd_plock_debug;
+extern int optd_plock_rate_limit;
+extern int optd_plock_ownership;
+extern int optd_drop_resources_time;
+extern int optd_drop_resources_count;
+extern int optd_drop_resources_age;
+
+extern int cfgk_debug;
+extern int cfgk_timewarn;
+extern int cfgk_protocol;
+extern int cfgd_debug_logfile;
+extern int cfgd_enable_fencing;
+extern int cfgd_enable_quorum;
+extern int cfgd_enable_plock;
+extern int cfgd_plock_debug;
+extern int cfgd_plock_rate_limit;
+extern int cfgd_plock_ownership;
+extern int cfgd_drop_resources_time;
+extern int cfgd_drop_resources_count;
+extern int cfgd_drop_resources_age;
+
+#endif
+
diff --git a/dlm_controld/cpg.c b/dlm_controld/cpg.c
new file mode 100644
index 0000000..b3c1454
--- /dev/null
+++ b/dlm_controld/cpg.c
@@ -0,0 +1,2606 @@
+#include "dlm_daemon.h"
+
+struct protocol_version {
+ uint16_t major;
+ uint16_t minor;
+ uint16_t patch;
+ uint16_t flags;
+};
+
+struct protocol {
+ union {
+ struct protocol_version dm_ver;
+ uint16_t daemon_max[4];
+ };
+ union {
+ struct protocol_version km_ver;
+ uint16_t kernel_max[4];
+ };
+ union {
+ struct protocol_version dr_ver;
+ uint16_t daemon_run[4];
+ };
+ union {
+ struct protocol_version kr_ver;
+ uint16_t kernel_run[4];
+ };
+};
+
+struct member {
+ struct list_head list;
+ int nodeid;
+ int start; /* 1 if we received a start message for this change */
+ int added; /* 1 if added by this change */
+ int failed; /* 1 if failed in this change */
+ int disallowed;
+ uint32_t start_flags;
+};
+
+struct node {
+ struct list_head list;
+ int nodeid;
+ int check_fencing;
+ int check_quorum;
+ int check_fs;
+ int fs_notified;
+ uint64_t add_time;
+ uint64_t fail_time;
+ uint64_t fence_time; /* for debug */
+ uint64_t cluster_add_time;
+ uint64_t cluster_remove_time;
+ uint32_t fence_queries; /* for debug */
+ uint32_t added_seq; /* for queries */
+ uint32_t removed_seq; /* for queries */
+ int failed_reason; /* for queries */
+
+ struct protocol proto;
+};
+
+/* One of these change structs is created for every confchg a cpg gets. */
+
+#define CGST_WAIT_CONDITIONS 1
+#define CGST_WAIT_MESSAGES 2
+
+struct change {
+ struct list_head list;
+ struct list_head members;
+ struct list_head removed; /* nodes removed by this change */
+ int member_count;
+ int joined_count;
+ int remove_count;
+ int failed_count;
+ int state;
+ int we_joined;
+ uint32_t seq; /* used as a reference for debugging, and for queries */
+ uint32_t combined_seq; /* for queries */
+ uint64_t create_time;
+};
+
+struct ls_info {
+ uint32_t ls_info_size;
+ uint32_t id_info_size;
+ uint32_t id_info_count;
+
+ uint32_t started_count;
+
+ int member_count;
+ int joined_count;
+ int remove_count;
+ int failed_count;
+};
+
+struct id_info {
+ int nodeid;
+};
+
+int message_flow_control_on;
+static cpg_handle_t cpg_handle_daemon;
+static int cpg_fd_daemon;
+static struct protocol our_protocol;
+static struct list_head daemon_nodes;
+static struct cpg_address daemon_member[MAX_NODES];
+static int daemon_member_count;
+
+static void log_config(const struct cpg_name *group_name,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries)
+{
+ char m_buf[128];
+ char j_buf[32];
+ char l_buf[32];
+ size_t i, len, pos;
+ int ret;
+
+ memset(m_buf, 0, sizeof(m_buf));
+ memset(j_buf, 0, sizeof(j_buf));
+ memset(l_buf, 0, sizeof(l_buf));
+
+ len = sizeof(m_buf);
+ pos = 0;
+ for (i = 0; i < member_list_entries; i++) {
+ ret = snprintf(m_buf + pos, len - pos, " %d",
+ member_list[i].nodeid);
+ if (ret >= len - pos)
+ break;
+ pos += ret;
+ }
+
+ len = sizeof(j_buf);
+ pos = 0;
+ for (i = 0; i < joined_list_entries; i++) {
+ ret = snprintf(j_buf + pos, len - pos, " %d",
+ joined_list[i].nodeid);
+ if (ret >= len - pos)
+ break;
+ pos += ret;
+ }
+
+ len = sizeof(l_buf);
+ pos = 0;
+ for (i = 0; i < left_list_entries; i++) {
+ ret = snprintf(l_buf + pos, len - pos, " %d",
+ left_list[i].nodeid);
+ if (ret >= len - pos)
+ break;
+ pos += ret;
+ }
+
+ log_debug("%s conf %zu %zu %zu memb%s join%s left%s", group_name->value,
+ member_list_entries, joined_list_entries, left_list_entries,
+ m_buf, j_buf, l_buf);
+}
+
+static void ls_info_in(struct ls_info *li)
+{
+ li->ls_info_size = le32_to_cpu(li->ls_info_size);
+ li->id_info_size = le32_to_cpu(li->id_info_size);
+ li->id_info_count = le32_to_cpu(li->id_info_count);
+ li->started_count = le32_to_cpu(li->started_count);
+ li->member_count = le32_to_cpu(li->member_count);
+ li->joined_count = le32_to_cpu(li->joined_count);
+ li->remove_count = le32_to_cpu(li->remove_count);
+ li->failed_count = le32_to_cpu(li->failed_count);
+}
+
+static void id_info_in(struct id_info *id)
+{
+ id->nodeid = le32_to_cpu(id->nodeid);
+}
+
+static void ids_in(struct ls_info *li, struct id_info *ids)
+{
+ struct id_info *id;
+ int i;
+
+ id = ids;
+ for (i = 0; i < li->id_info_count; i++) {
+ id_info_in(id);
+ id = (struct id_info *)((char *)id + li->id_info_size);
+ }
+}
+
+const char *msg_name(int type)
+{
+ switch (type) {
+ case DLM_MSG_PROTOCOL:
+ return "protocol";
+ case DLM_MSG_START:
+ return "start";
+ case DLM_MSG_PLOCK:
+ return "plock";
+ case DLM_MSG_PLOCK_OWN:
+ return "plock_own";
+ case DLM_MSG_PLOCK_DROP:
+ return "plock_drop";
+ case DLM_MSG_PLOCK_SYNC_LOCK:
+ return "plock_sync_lock";
+ case DLM_MSG_PLOCK_SYNC_WAITER:
+ return "plock_sync_waiter";
+ case DLM_MSG_PLOCKS_DATA:
+ return "plocks_data";
+ case DLM_MSG_PLOCKS_DONE:
+ return "plocks_done";
+ case DLM_MSG_DEADLK_CYCLE_START:
+ return "deadlk_cycle_start";
+ case DLM_MSG_DEADLK_CYCLE_END:
+ return "deadlk_cycle_end";
+ case DLM_MSG_DEADLK_CHECKPOINT_READY:
+ return "deadlk_checkpoint_ready";
+ case DLM_MSG_DEADLK_CANCEL_LOCK:
+ return "deadlk_cancel_lock";
+ default:
+ return "unknown";
+ }
+}
+
+static int _send_message(cpg_handle_t h, void *buf, int len, int type)
+{
+ struct iovec iov;
+ cpg_error_t error;
+ int retries = 0;
+
+ iov.iov_base = buf;
+ iov.iov_len = len;
+
+ retry:
+ error = cpg_mcast_joined(h, CPG_TYPE_AGREED, &iov, 1);
+ if (error == CPG_ERR_TRY_AGAIN) {
+ retries++;
+ usleep(1000);
+ if (!(retries % 100))
+ log_error("cpg_mcast_joined retry %d %s",
+ retries, msg_name(type));
+ goto retry;
+ }
+ if (error != CPG_OK) {
+ log_error("cpg_mcast_joined error %d handle %llx %s",
+ error, (unsigned long long)h, msg_name(type));
+ return -1;
+ }
+
+ if (retries)
+ log_debug("cpg_mcast_joined retried %d %s",
+ retries, msg_name(type));
+
+ return 0;
+}
+
+/* header fields caller needs to set: type, to_nodeid, flags, msgdata */
+
+void dlm_send_message(struct lockspace *ls, char *buf, int len)
+{
+ struct dlm_header *hd = (struct dlm_header *) buf;
+ int type = hd->type;
+
+ hd->version[0] = cpu_to_le16(our_protocol.daemon_run[0]);
+ hd->version[1] = cpu_to_le16(our_protocol.daemon_run[1]);
+ hd->version[2] = cpu_to_le16(our_protocol.daemon_run[2]);
+ hd->type = cpu_to_le16(hd->type);
+ hd->nodeid = cpu_to_le32(our_nodeid);
+ hd->to_nodeid = cpu_to_le32(hd->to_nodeid);
+ hd->global_id = cpu_to_le32(ls->global_id);
+ hd->flags = cpu_to_le32(hd->flags);
+ hd->msgdata = cpu_to_le32(hd->msgdata);
+ hd->msgdata2 = cpu_to_le32(hd->msgdata2);
+
+ _send_message(ls->cpg_handle, buf, len, type);
+}
+
+static struct member *find_memb(struct change *cg, int nodeid)
+{
+ struct member *memb;
+
+ list_for_each_entry(memb, &cg->members, list) {
+ if (memb->nodeid == nodeid)
+ return memb;
+ }
+ return NULL;
+}
+
+static struct lockspace *find_ls_handle(cpg_handle_t h)
+{
+ struct lockspace *ls;
+
+ list_for_each_entry(ls, &lockspaces, list) {
+ if (ls->cpg_handle == h)
+ return ls;
+ }
+ return NULL;
+}
+
+static struct lockspace *find_ls_ci(int ci)
+{
+ struct lockspace *ls;
+
+ list_for_each_entry(ls, &lockspaces, list) {
+ if (ls->cpg_client == ci)
+ return ls;
+ }
+ return NULL;
+}
+
+static void free_cg(struct change *cg)
+{
+ struct member *memb, *safe;
+
+ list_for_each_entry_safe(memb, safe, &cg->members, list) {
+ list_del(&memb->list);
+ free(memb);
+ }
+ list_for_each_entry_safe(memb, safe, &cg->removed, list) {
+ list_del(&memb->list);
+ free(memb);
+ }
+ free(cg);
+}
+
+static void free_ls(struct lockspace *ls)
+{
+ struct change *cg, *cg_safe;
+ struct node *node, *node_safe;
+
+ list_for_each_entry_safe(cg, cg_safe, &ls->changes, list) {
+ list_del(&cg->list);
+ free_cg(cg);
+ }
+
+ if (ls->started_change)
+ free_cg(ls->started_change);
+
+ list_for_each_entry_safe(node, node_safe, &ls->node_history, list) {
+ list_del(&node->list);
+ free(node);
+ }
+
+ free(ls);
+}
+
+
+/* Problem scenario:
+ nodes A,B,C are in fence domain
+ node C has gfs foo mounted
+ node C fails
+ nodes A,B begin fencing C (slow, not completed)
+ node B mounts gfs foo
+
+ We may end up having gfs foo mounted and being used on B before
+ C has been fenced. C could wake up corrupt fs.
+
+ So, we need to prevent any new gfs mounts while there are any
+ outstanding, incomplete fencing operations.
+
+ We also need to check that the specific failed nodes we know about have
+ been fenced (since fenced may not even have been notified that the node
+ has failed yet).
+
+ So, check that:
+ 1. has fenced fenced the node since we saw it fail?
+ 2. fenced has no outstanding fencing ops
+
+ For 1:
+ - node X fails
+ - we see node X fail and X has non-zero add_time,
+ set check_fencing and record the fail time
+ - wait for X to be removed from all dlm cpg's (probably not necessary)
+ - check that the fencing time is later than the recorded time above
+
+ Tracking fencing state when there are spurious partitions/merges...
+
+ from a spurious leave/join of node X, a lockspace will see:
+ - node X is a lockspace member
+ - node X fails, may be waiting for all cpgs to see failure or for fencing to
+ complete
+ - node X joins the lockspace - we want to process the change as usual, but
+ don't want to disrupt the code waiting for the fencing, and we want to
+ continue running properly once the remerged node is properly reset
+
+ ls->node_history
+ when we see a node not in this list, add entry for it with zero add_time
+ record the time we get a good start message from the node, add_time
+ clear add_time if the node leaves
+ if node fails with non-zero add_time, set check_fencing
+ when a node is fenced, clear add_time and clear check_fencing
+ if a node remerges after this, no good start message, no new add_time set
+ if a node fails with zero add_time, it doesn't need fencing
+ if a node remerges before it's been fenced, no good start message, no new
+ add_time set
+*/
+
+static struct node *get_node_history(struct lockspace *ls, int nodeid)
+{
+ struct node *node;
+
+ list_for_each_entry(node, &ls->node_history, list) {
+ if (node->nodeid == nodeid)
+ return node;
+ }
+ return NULL;
+}
+
+static void node_history_init(struct lockspace *ls, int nodeid,
+ struct change *cg)
+{
+ struct node *node;
+
+ node = get_node_history(ls, nodeid);
+ if (node)
+ goto out;
+
+ node = malloc(sizeof(struct node));
+ if (!node)
+ return;
+ memset(node, 0, sizeof(struct node));
+
+ node->nodeid = nodeid;
+ node->add_time = 0;
+ list_add_tail(&node->list, &ls->node_history);
+ out:
+ if (cg)
+ node->added_seq = cg->seq; /* for queries */
+}
+
+void node_history_cluster_add(int nodeid)
+{
+ struct lockspace *ls;
+ struct node *node;
+
+ list_for_each_entry(ls, &lockspaces, list) {
+ node_history_init(ls, nodeid, NULL);
+
+ node = get_node_history(ls, nodeid);
+ if (!node) {
+ log_error("node_history_cluster_add no nodeid %d",
+ nodeid);
+ return;
+ }
+
+ node->cluster_add_time = time(NULL);
+ }
+}
+
+void node_history_cluster_remove(int nodeid)
+{
+ struct lockspace *ls;
+ struct node *node;
+
+ list_for_each_entry(ls, &lockspaces, list) {
+ node = get_node_history(ls, nodeid);
+ if (!node) {
+ log_error("node_history_cluster_remove no nodeid %d",
+ nodeid);
+ return;
+ }
+
+ node->cluster_remove_time = time(NULL);
+ }
+}
+
+static void node_history_start(struct lockspace *ls, int nodeid)
+{
+ struct node *node;
+
+ node = get_node_history(ls, nodeid);
+ if (!node) {
+ log_error("node_history_start no nodeid %d", nodeid);
+ return;
+ }
+
+ node->add_time = time(NULL);
+}
+
+static void node_history_left(struct lockspace *ls, int nodeid,
+ struct change *cg)
+{
+ struct node *node;
+
+ node = get_node_history(ls, nodeid);
+ if (!node) {
+ log_error("node_history_left no nodeid %d", nodeid);
+ return;
+ }
+
+ node->add_time = 0;
+ node->removed_seq = cg->seq; /* for queries */
+}
+
+static void node_history_fail(struct lockspace *ls, int nodeid,
+ struct change *cg, int reason)
+{
+ struct node *node;
+
+ node = get_node_history(ls, nodeid);
+ if (!node) {
+ log_error("node_history_fail no nodeid %d", nodeid);
+ return;
+ }
+
+ if (cfgd_enable_fencing && node->add_time) {
+ node->check_fencing = 1;
+ node->fence_time = 0;
+ node->fence_queries = 0;
+ node->fail_time = time(NULL);
+ }
+
+ /* fenced will take care of making sure the quorum value
+ is adjusted for all the failures */
+
+ if (cfgd_enable_quorum && !cfgd_enable_fencing)
+ node->check_quorum = 1;
+
+ if (ls->fs_registered) {
+ log_group(ls, "check_fs nodeid %d set", nodeid);
+ node->check_fs = 1;
+ }
+
+ node->removed_seq = cg->seq; /* for queries */
+ node->failed_reason = reason; /* for queries */
+}
+
+static int check_fencing_done(struct lockspace *ls)
+{
+ struct node *node;
+ uint64_t last_fenced_time;
+ int in_progress, wait_count = 0;
+ int rv;
+
+ if (!cfgd_enable_fencing) {
+ log_group(ls, "check_fencing disabled");
+ return 1;
+ }
+
+ list_for_each_entry(node, &ls->node_history, list) {
+ if (!node->check_fencing)
+ continue;
+
+ /* check with fenced to see if the node has been
+ fenced since node->add_time */
+
+ rv = fence_node_time(node->nodeid, &last_fenced_time);
+ if (rv < 0)
+ log_error("fenced_node_info error %d", rv);
+
+ /* need >= not just > because in at least one case
+ we've seen fenced_time within the same second as
+ fail_time: with external fencing, e.g. fence_node */
+
+ if (last_fenced_time >= node->fail_time) {
+ log_group(ls, "check_fencing %d done "
+ "add %llu fail %llu last %llu",
+ node->nodeid,
+ (unsigned long long)node->add_time,
+ (unsigned long long)node->fail_time,
+ (unsigned long long)last_fenced_time);
+ node->check_fencing = 0;
+ node->add_time = 0;
+ node->fence_time = last_fenced_time;
+ } else {
+ if (!node->fence_queries ||
+ node->fence_time != last_fenced_time) {
+ log_group(ls, "check_fencing %d wait "
+ "add %llu fail %llu last %llu",
+ node->nodeid,
+ (unsigned long long)node->add_time,
+ (unsigned long long)node->fail_time,
+ (unsigned long long)last_fenced_time);
+ node->fence_queries++;
+ node->fence_time = last_fenced_time;
+ }
+ wait_count++;
+ }
+ }
+
+ if (wait_count)
+ return 0;
+
+ /* now check if there are any outstanding fencing ops (for nodes
+ we may not have seen in any lockspace), and return 0 if there
+ are any */
+
+ rv = fence_in_progress(&in_progress);
+ if (rv < 0) {
+ log_error("fenced_domain_info error %d", rv);
+ return 0;
+ }
+
+ if (in_progress)
+ return 0;
+
+ log_group(ls, "check_fencing done");
+ return 1;
+}
+
+static int check_quorum_done(struct lockspace *ls)
+{
+ struct node *node;
+ int wait_count = 0;
+
+ if (!cfgd_enable_quorum) {
+ log_group(ls, "check_quorum disabled");
+ return 1;
+ }
+
+ /* wait for quorum system (cman) to see all the same nodes failed, so
+ we know that cluster_quorate is adjusted for the same failures we've
+ seen (see comment in fenced about the assumption here) */
+
+ list_for_each_entry(node, &ls->node_history, list) {
+ if (!node->check_quorum)
+ continue;
+
+ if (!is_cluster_member(node->nodeid)) {
+ node->check_quorum = 0;
+ } else {
+ log_group(ls, "check_quorum nodeid %d is_cluster_member",
+ node->nodeid);
+ wait_count++;
+ }
+ }
+
+ if (wait_count)
+ return 0;
+
+ if (!cluster_quorate) {
+ log_group(ls, "check_quorum not quorate");
+ return 0;
+ }
+
+ log_group(ls, "check_quorum done");
+ return 1;
+}
+
+/* wait for local fs_controld to ack each failed node */
+
+static int check_fs_done(struct lockspace *ls)
+{
+ struct node *node;
+ int wait_count = 0;
+
+ /* no corresponding fs for this lockspace */
+ if (!ls->fs_registered) {
+ log_group(ls, "check_fs none registered");
+ return 1;
+ }
+
+ list_for_each_entry(node, &ls->node_history, list) {
+ if (!node->check_fs)
+ continue;
+
+ if (node->fs_notified) {
+ log_group(ls, "check_fs nodeid %d clear", node->nodeid);
+ node->check_fs = 0;
+ node->fs_notified = 0;
+ } else {
+ log_group(ls, "check_fs nodeid %d needs fs notify",
+ node->nodeid);
+ wait_count++;
+ }
+ }
+
+ if (wait_count)
+ return 0;
+
+ log_group(ls, "check_fs done");
+ return 1;
+}
+
+static int member_ids[MAX_NODES];
+static int member_count;
+static int renew_ids[MAX_NODES];
+static int renew_count;
+
+static void format_member_ids(struct lockspace *ls)
+{
+ struct change *cg = list_first_entry(&ls->changes, struct change, list);
+ struct member *memb;
+
+ memset(member_ids, 0, sizeof(member_ids));
+ member_count = 0;
+
+ list_for_each_entry(memb, &cg->members, list)
+ member_ids[member_count++] = memb->nodeid;
+}
+
+/* list of nodeids that have left and rejoined since last start_kernel;
+ is any member of startcg in the left list of any other cg's?
+ (if it is, then it presumably must be flagged added in another) */
+
+static void format_renew_ids(struct lockspace *ls)
+{
+ struct change *cg, *startcg;
+ struct member *memb, *leftmemb;
+
+ startcg = list_first_entry(&ls->changes, struct change, list);
+
+ memset(renew_ids, 0, sizeof(renew_ids));
+ renew_count = 0;
+
+ list_for_each_entry(memb, &startcg->members, list) {
+ list_for_each_entry(cg, &ls->changes, list) {
+ if (cg == startcg)
+ continue;
+ list_for_each_entry(leftmemb, &cg->removed, list) {
+ if (memb->nodeid == leftmemb->nodeid) {
+ renew_ids[renew_count++] = memb->nodeid;
+ }
+ }
+ }
+ }
+
+}
+
+static void start_kernel(struct lockspace *ls)
+{
+ struct change *cg = list_first_entry(&ls->changes, struct change, list);
+
+ if (!ls->kernel_stopped) {
+ log_error("start_kernel cg %u not stopped", cg->seq);
+ return;
+ }
+
+ log_group(ls, "start_kernel cg %u member_count %d",
+ cg->seq, cg->member_count);
+
+ /* needs to happen before setting control which starts recovery */
+ if (ls->joining)
+ set_sysfs_id(ls->name, ls->global_id);
+
+ format_member_ids(ls);
+ format_renew_ids(ls);
+ set_configfs_members(ls->name, member_count, member_ids,
+ renew_count, renew_ids);
+ set_sysfs_control(ls->name, 1);
+ ls->kernel_stopped = 0;
+
+ if (ls->joining) {
+ set_sysfs_event_done(ls->name, 0);
+ ls->joining = 0;
+ }
+}
+
+static void stop_kernel(struct lockspace *ls, uint32_t seq)
+{
+ if (!ls->kernel_stopped) {
+ log_group(ls, "stop_kernel cg %u", seq);
+ set_sysfs_control(ls->name, 0);
+ ls->kernel_stopped = 1;
+ }
+}
+
+/* the first condition is that the local lockspace is stopped which we
+ don't need to check for because stop_kernel(), which is synchronous,
+ was done when the change was created */
+
+static int wait_conditions_done(struct lockspace *ls)
+{
+ /* the fencing/quorum/fs conditions need to account for all the changes
+ that have occured since the last change applied to dlm-kernel, not
+ just the latest change */
+
+ if (!check_fencing_done(ls)) {
+ poll_fencing++;
+ return 0;
+ }
+
+ /* fencing waits for quorum, so we don't need to check quorum for any
+ reasons related to safety or protection, so enable_quorum defaults
+ to 0. This does mean that lockspaces (and cluster fs's) can be
+ started/enabled in an inquorate cluster if there are no outstanding
+ fencing operations. Some users or apps may want lockspaces/fs's to
+ only be enabled in a quorate cluster; enable_quorum can be set to 1
+ to get that behavior. The main advantage of not waiting for quorum
+ here is to allow lockspaces to be shut down (and cluster fs's
+ unmounted) in an inquorate cluster. */
+
+ if (!check_quorum_done(ls)) {
+ poll_quorum++;
+ return 0;
+ }
+
+ if (!check_fs_done(ls)) {
+ poll_fs++;
+ return 0;
+ }
+
+ return 1;
+}
+
+static int wait_messages_done(struct lockspace *ls)
+{
+ struct change *cg = list_first_entry(&ls->changes, struct change, list);
+ struct member *memb;
+ int need = 0, total = 0;
+
+ list_for_each_entry(memb, &cg->members, list) {
+ if (!memb->start)
+ need++;
+ total++;
+ }
+
+ if (need) {
+ log_group(ls, "wait_messages cg %u need %d of %d",
+ cg->seq, need, total);
+ return 0;
+ }
+
+ log_group(ls, "wait_messages cg %u got all %d", cg->seq, total);
+ return 1;
+}
+
+static void cleanup_changes(struct lockspace *ls)
+{
+ struct change *cg = list_first_entry(&ls->changes, struct change, list);
+ struct change *safe;
+
+ list_del(&cg->list);
+ if (ls->started_change)
+ free_cg(ls->started_change);
+ ls->started_change = cg;
+
+ ls->started_count++;
+ if (!ls->started_count)
+ ls->started_count++;
+
+ cg->combined_seq = cg->seq; /* for queries */
+
+ list_for_each_entry_safe(cg, safe, &ls->changes, list) {
+ ls->started_change->combined_seq = cg->seq; /* for queries */
+ list_del(&cg->list);
+ free_cg(cg);
+ }
+}
+
+/* There's a stream of confchg and messages. At one of these
+ messages, the low node needs to store plocks and new nodes
+ need to begin saving plock messages. A second message is
+ needed to say that the plocks are ready to be read.
+
+ When the last start message is recvd for a change, the low node
+ stores plocks and the new nodes begin saving messages. When the
+ store is done, low node sends plocks_stored message. When
+ new nodes recv this, they read the plocks and their saved messages.
+ plocks_stored message should identify a specific change, like start
+ messages do; if it doesn't match ls->started_change, then it's ignored.
+
+ If a confchg adding a new node arrives after plocks are stored but
+ before plocks_stored msg recvd, then the message is ignored. The low
+ node will send another plocks_stored message for the latest change
+ (although it may be able to reuse the ckpt if no plock state has changed).
+*/
+
+static void set_plock_data_node(struct lockspace *ls)
+{
+ struct change *cg = list_first_entry(&ls->changes, struct change, list);
+ struct member *memb;
+ int low = 0;
+
+ list_for_each_entry(memb, &cg->members, list) {
+ if (!(memb->start_flags & DLM_MFLG_HAVEPLOCK))
+ continue;
+
+ if (!low || memb->nodeid < low)
+ low = memb->nodeid;
+ }
+
+ log_dlock(ls, "set_plock_data_node from %d to %d",
+ ls->plock_data_node, low);
+
+ ls->plock_data_node = low;
+}
+
+static struct id_info *get_id_struct(struct id_info *ids, int count, int size,
+ int nodeid)
+{
+ struct id_info *id = ids;
+ int i;
+
+ for (i = 0; i < count; i++) {
+ if (id->nodeid == nodeid)
+ return id;
+ id = (struct id_info *)((char *)id + size);
+ }
+ return NULL;
+}
+
+/* do the change details in the message match the details of the given change */
+
+static int match_change(struct lockspace *ls, struct change *cg,
+ struct dlm_header *hd, struct ls_info *li,
+ struct id_info *ids)
+{
+ struct id_info *id;
+ struct member *memb;
+ struct node *node;
+ uint32_t seq = hd->msgdata;
+ int i, members_mismatch;
+
+ /* We can ignore messages if we're not in the list of members.
+ The one known time this will happen is after we've joined
+ the cpg, we can get messages for changes prior to the change
+ in which we're added. */
+
+ id = get_id_struct(ids, li->id_info_count, li->id_info_size,our_nodeid);
+
+ if (!id) {
+ log_group(ls, "match_change %d:%u skip %u we are not in members",
+ hd->nodeid, seq, cg->seq);
+ return 0;
+ }
+
+ memb = find_memb(cg, hd->nodeid);
+ if (!memb) {
+ log_group(ls, "match_change %d:%u skip %u sender not member",
+ hd->nodeid, seq, cg->seq);
+ return 0;
+ }
+
+ if (memb->start_flags & DLM_MFLG_NACK) {
+ log_group(ls, "match_change %d:%u skip %u is nacked",
+ hd->nodeid, seq, cg->seq);
+ return 0;
+ }
+
+ if (memb->start && hd->type == DLM_MSG_START) {
+ log_group(ls, "match_change %d:%u skip %u already start",
+ hd->nodeid, seq, cg->seq);
+ return 0;
+ }
+
+ /* a node's start can't match a change if the node joined the cluster
+ more recently than the change was created */
+
+ node = get_node_history(ls, hd->nodeid);
+ if (!node) {
+ log_group(ls, "match_change %d:%u skip cg %u no node history",
+ hd->nodeid, seq, cg->seq);
+ return 0;
+ }
+
+ if (node->cluster_add_time > cg->create_time) {
+ log_group(ls, "match_change %d:%u skip cg %u created %llu "
+ "cluster add %llu", hd->nodeid, seq, cg->seq,
+ (unsigned long long)cg->create_time,
+ (unsigned long long)node->cluster_add_time);
+ return 0;
+ }
+
+ /* verify this is the right change by matching the counts
+ and the nodeids of the current members */
+
+ if (li->member_count != cg->member_count ||
+ li->joined_count != cg->joined_count ||
+ li->remove_count != cg->remove_count ||
+ li->failed_count != cg->failed_count) {
+ log_group(ls, "match_change %d:%u skip %u expect counts "
+ "%d %d %d %d", hd->nodeid, seq, cg->seq,
+ cg->member_count, cg->joined_count,
+ cg->remove_count, cg->failed_count);
+ return 0;
+ }
+
+ members_mismatch = 0;
+ id = ids;
+
+ for (i = 0; i < li->id_info_count; i++) {
+ memb = find_memb(cg, id->nodeid);
+ if (!memb) {
+ log_group(ls, "match_change %d:%u skip %u no memb %d",
+ hd->nodeid, seq, cg->seq, id->nodeid);
+ members_mismatch = 1;
+ break;
+ }
+ id = (struct id_info *)((char *)id + li->id_info_size);
+ }
+
+ if (members_mismatch)
+ return 0;
+
+ log_group(ls, "match_change %d:%u matches cg %u", hd->nodeid, seq,
+ cg->seq);
+ return 1;
+}
+
+/* Unfortunately, there's no really simple way to match a message with the
+ specific change that it was sent for. We hope that by passing all the
+ details of the change in the message, we will be able to uniquely match the
+ it to the correct change. */
+
+/* A start message will usually be for the first (current) change on our list.
+ In some cases it will be for a non-current change, and we can ignore it:
+
+ 1. A,B,C get confchg1 adding C
+ 2. C sends start for confchg1
+ 3. A,B,C get confchg2 adding D
+ 4. A,B,C,D recv start from C for confchg1 - ignored
+ 5. C,D send start for confchg2
+ 6. A,B send start for confchg2
+ 7. A,B,C,D recv all start messages for confchg2, and start kernel
+
+ In step 4, how do the nodes know whether the start message from C is
+ for confchg1 or confchg2? Hopefully by comparing the counts and members. */
+
+static struct change *find_change(struct lockspace *ls, struct dlm_header *hd,
+ struct ls_info *li, struct id_info *ids)
+{
+ struct change *cg;
+
+ list_for_each_entry_reverse(cg, &ls->changes, list) {
+ if (!match_change(ls, cg, hd, li, ids))
+ continue;
+ return cg;
+ }
+
+ log_group(ls, "find_change %d:%u no match", hd->nodeid, hd->msgdata);
+ return NULL;
+}
+
+static int is_added(struct lockspace *ls, int nodeid)
+{
+ struct change *cg;
+ struct member *memb;
+
+ list_for_each_entry(cg, &ls->changes, list) {
+ memb = find_memb(cg, nodeid);
+ if (memb && memb->added)
+ return 1;
+ }
+ return 0;
+}
+
+static void receive_start(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ struct change *cg;
+ struct member *memb;
+ struct ls_info *li;
+ struct id_info *ids;
+ uint32_t seq = hd->msgdata;
+ int added;
+
+ log_group(ls, "receive_start %d:%u len %d", hd->nodeid, seq, len);
+
+ li = (struct ls_info *)((char *)hd + sizeof(struct dlm_header));
+ ids = (struct id_info *)((char *)li + sizeof(struct ls_info));
+
+ ls_info_in(li);
+ ids_in(li, ids);
+
+ cg = find_change(ls, hd, li, ids);
+ if (!cg)
+ return;
+
+ memb = find_memb(cg, hd->nodeid);
+ if (!memb) {
+ /* this should never happen since match_change checks it */
+ log_error("receive_start no member %d", hd->nodeid);
+ return;
+ }
+
+ memb->start_flags = hd->flags;
+
+ added = is_added(ls, hd->nodeid);
+
+ if (added && li->started_count && ls->started_count) {
+ log_error("receive_start %d:%u add node with started_count %u",
+ hd->nodeid, seq, li->started_count);
+
+ /* see comment in fence/fenced/cpg.c */
+ memb->disallowed = 1;
+ return;
+ }
+
+ if (memb->start_flags & DLM_MFLG_NACK) {
+ log_group(ls, "receive_start %d:%u is NACK", hd->nodeid, seq);
+ return;
+ }
+
+ node_history_start(ls, hd->nodeid);
+ memb->start = 1;
+}
+
+static void receive_plocks_done(struct lockspace *ls, struct dlm_header *hd,
+ int len)
+{
+ struct ls_info *li;
+ struct id_info *ids;
+
+ log_dlock(ls, "receive_plocks_done %d:%u flags %x plocks_data %u need %d save
%d",
+ hd->nodeid, hd->msgdata, hd->flags, hd->msgdata2,
+ ls->need_plocks, ls->save_plocks);
+
+ if (!ls->need_plocks)
+ return;
+
+ if (ls->need_plocks && !ls->save_plocks)
+ return;
+
+ if (!ls->started_change) {
+ /* don't think this should happen */
+ log_elock(ls, "receive_plocks_done %d:%u no started_change",
+ hd->nodeid, hd->msgdata);
+ return;
+ }
+
+ li = (struct ls_info *)((char *)hd + sizeof(struct dlm_header));
+ ids = (struct id_info *)((char *)li + sizeof(struct ls_info));
+ ls_info_in(li);
+ ids_in(li, ids);
+
+ if (!match_change(ls, ls->started_change, hd, li, ids)) {
+ /* don't think this should happen */
+ log_elock(ls, "receive_plocks_done %d:%u no match_change",
+ hd->nodeid, hd->msgdata);
+
+ /* remove/free anything we've saved from
+ receive_plocks_data messages that weren't for us */
+ clear_plocks_data(ls);
+ return;
+ }
+
+ if (ls->recv_plocks_data_count != hd->msgdata2) {
+ log_elock(ls, "receive_plocks_done plocks_data %u recv %u",
+ hd->msgdata2, ls->recv_plocks_data_count);
+ }
+
+ process_saved_plocks(ls);
+ ls->need_plocks = 0;
+ ls->save_plocks = 0;
+
+ log_dlock(ls, "receive_plocks_done %d:%u plocks_data_count %u",
+ hd->nodeid, hd->msgdata, ls->recv_plocks_data_count);
+}
+
+static void send_info(struct lockspace *ls, struct change *cg, int type,
+ uint32_t flags, uint32_t msgdata2)
+{
+ struct dlm_header *hd;
+ struct ls_info *li;
+ struct id_info *id;
+ struct member *memb;
+ char *buf;
+ int len, id_count;
+
+ id_count = cg->member_count;
+
+ len = sizeof(struct dlm_header) + sizeof(struct ls_info) +
+ id_count * sizeof(struct id_info);
+
+ buf = malloc(len);
+ if (!buf) {
+ log_error("send_info len %d no mem", len);
+ return;
+ }
+ memset(buf, 0, len);
+
+ hd = (struct dlm_header *)buf;
+ li = (struct ls_info *)(buf + sizeof(*hd));
+ id = (struct id_info *)(buf + sizeof(*hd) + sizeof(*li));
+
+ /* fill in header (dlm_send_message handles part of header) */
+
+ hd->type = type;
+ hd->msgdata = cg->seq;
+ hd->flags = flags;
+ hd->msgdata2 = msgdata2;
+
+ if (ls->joining)
+ hd->flags |= DLM_MFLG_JOINING;
+ if (!ls->need_plocks)
+ hd->flags |= DLM_MFLG_HAVEPLOCK;
+
+ /* fill in ls_info */
+
+ li->ls_info_size = cpu_to_le32(sizeof(struct ls_info));
+ li->id_info_size = cpu_to_le32(sizeof(struct id_info));
+ li->id_info_count = cpu_to_le32(id_count);
+ li->started_count = cpu_to_le32(ls->started_count);
+ li->member_count = cpu_to_le32(cg->member_count);
+ li->joined_count = cpu_to_le32(cg->joined_count);
+ li->remove_count = cpu_to_le32(cg->remove_count);
+ li->failed_count = cpu_to_le32(cg->failed_count);
+
+ /* fill in id_info entries */
+
+ list_for_each_entry(memb, &cg->members, list) {
+ id->nodeid = cpu_to_le32(memb->nodeid);
+ id++;
+ }
+
+ dlm_send_message(ls, buf, len);
+
+ free(buf);
+}
+
+static void send_start(struct lockspace *ls, struct change *cg)
+{
+ log_group(ls, "send_start %d:%u counts %u %d %d %d %d",
+ our_nodeid, cg->seq, ls->started_count,
+ cg->member_count, cg->joined_count, cg->remove_count,
+ cg->failed_count);
+
+ send_info(ls, cg, DLM_MSG_START, 0, 0);
+}
+
+static void send_plocks_done(struct lockspace *ls, struct change *cg, uint32_t
plocks_data)
+{
+ log_dlock(ls, "send_plocks_done %d:%u counts %u %d %d %d %d plocks_data %u",
+ our_nodeid, cg->seq, ls->started_count,
+ cg->member_count, cg->joined_count, cg->remove_count,
+ cg->failed_count, plocks_data);
+
+ send_info(ls, cg, DLM_MSG_PLOCKS_DONE, 0, plocks_data);
+}
+
+static int same_members(struct change *cg1, struct change *cg2)
+{
+ struct member *memb;
+
+ list_for_each_entry(memb, &cg1->members, list) {
+ if (!find_memb(cg2, memb->nodeid))
+ return 0;
+ }
+ return 1;
+}
+
+static void send_nacks(struct lockspace *ls, struct change *startcg)
+{
+ struct change *cg;
+
+ list_for_each_entry(cg, &ls->changes, list) {
+ if (cg->seq < startcg->seq &&
+ cg->member_count == startcg->member_count &&
+ cg->joined_count == startcg->joined_count &&
+ cg->remove_count == startcg->remove_count &&
+ cg->failed_count == startcg->failed_count &&
+ same_members(cg, startcg)) {
+ log_group(ls, "send nack old cg %u new cg %u",
+ cg->seq, startcg->seq);
+ send_info(ls, cg, DLM_MSG_START, DLM_MFLG_NACK, 0);
+ }
+ }
+}
+
+static int nodes_added(struct lockspace *ls)
+{
+ struct change *cg;
+
+ list_for_each_entry(cg, &ls->changes, list) {
+ if (cg->joined_count)
+ return 1;
+ }
+ return 0;
+}
+
+static void prepare_plocks(struct lockspace *ls)
+{
+ struct change *cg = list_first_entry(&ls->changes, struct change, list);
+ struct member *memb;
+ uint32_t plocks_data;
+
+ if (!cfgd_enable_plock || ls->disable_plock)
+ return;
+
+ log_dlock(ls, "prepare_plocks");
+
+ /* if we're the only node in the lockspace, then we are the data_node
+ and we don't need plocks */
+
+ if (cg->member_count == 1) {
+ list_for_each_entry(memb, &cg->members, list) {
+ if (memb->nodeid != our_nodeid) {
+ log_elock(ls, "prepare_plocks other member %d",
+ memb->nodeid);
+ }
+ }
+ ls->plock_data_node = our_nodeid;
+ ls->need_plocks = 0;
+ return;
+ }
+
+ /* the low node that indicated it had plock state in its last
+ start message is the data_node */
+
+ set_plock_data_node(ls);
+
+ /* there is no node with plock state, so there's no syncing to do */
+
+ if (!ls->plock_data_node) {
+ ls->need_plocks = 0;
+ ls->save_plocks = 0;
+ return;
+ }
+
+ /* We save all plock messages received after our own confchg and
+ apply them after we receive the plocks_done message from the
+ data_node. */
+
+ if (ls->need_plocks) {
+ log_dlock(ls, "save_plocks start");
+ ls->save_plocks = 1;
+ return;
+ }
+
+ if (ls->plock_data_node != our_nodeid)
+ return;
+
+ if (nodes_added(ls))
+ send_all_plocks_data(ls, cg->seq, &plocks_data);
+
+ send_plocks_done(ls, cg, plocks_data);
+}
+
+static void apply_changes(struct lockspace *ls)
+{
+ struct change *cg;
+
+ if (list_empty(&ls->changes))
+ return;
+ cg = list_first_entry(&ls->changes, struct change, list);
+
+ switch (cg->state) {
+
+ case CGST_WAIT_CONDITIONS:
+ if (wait_conditions_done(ls)) {
+ send_nacks(ls, cg);
+ send_start(ls, cg);
+ cg->state = CGST_WAIT_MESSAGES;
+ }
+ break;
+
+ case CGST_WAIT_MESSAGES:
+ if (wait_messages_done(ls)) {
+ start_kernel(ls);
+ prepare_plocks(ls);
+ cleanup_changes(ls);
+ }
+ break;
+
+ default:
+ log_error("apply_changes invalid state %d", cg->state);
+ }
+}
+
+void process_lockspace_changes(void)
+{
+ struct lockspace *ls, *safe;
+
+ poll_fencing = 0;
+ poll_quorum = 0;
+ poll_fs = 0;
+
+ list_for_each_entry_safe(ls, safe, &lockspaces, list) {
+ if (!list_empty(&ls->changes))
+ apply_changes(ls);
+ }
+}
+
+static int add_change(struct lockspace *ls,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries,
+ struct change **cg_out)
+{
+ struct change *cg;
+ struct member *memb;
+ int i, error;
+
+ cg = malloc(sizeof(struct change));
+ if (!cg)
+ goto fail_nomem;
+ memset(cg, 0, sizeof(struct change));
+ INIT_LIST_HEAD(&cg->members);
+ INIT_LIST_HEAD(&cg->removed);
+ cg->state = CGST_WAIT_CONDITIONS;
+ cg->create_time = time(NULL);
+ cg->seq = ++ls->change_seq;
+ if (!cg->seq)
+ cg->seq = ++ls->change_seq;
+
+ cg->member_count = member_list_entries;
+ cg->joined_count = joined_list_entries;
+ cg->remove_count = left_list_entries;
+
+ for (i = 0; i < member_list_entries; i++) {
+ memb = malloc(sizeof(struct member));
+ if (!memb)
+ goto fail_nomem;
+ memset(memb, 0, sizeof(struct member));
+ memb->nodeid = member_list[i].nodeid;
+ list_add_tail(&memb->list, &cg->members);
+ }
+
+ for (i = 0; i < left_list_entries; i++) {
+ memb = malloc(sizeof(struct member));
+ if (!memb)
+ goto fail_nomem;
+ memset(memb, 0, sizeof(struct member));
+ memb->nodeid = left_list[i].nodeid;
+ if (left_list[i].reason == CPG_REASON_NODEDOWN ||
+ left_list[i].reason == CPG_REASON_PROCDOWN) {
+ memb->failed = 1;
+ cg->failed_count++;
+ }
+ list_add_tail(&memb->list, &cg->removed);
+
+ if (memb->failed)
+ node_history_fail(ls, memb->nodeid, cg,
+ left_list[i].reason);
+ else
+ node_history_left(ls, memb->nodeid, cg);
+
+ log_group(ls, "add_change cg %u remove nodeid %d reason %d",
+ cg->seq, memb->nodeid, left_list[i].reason);
+
+ if (left_list[i].reason == CPG_REASON_PROCDOWN)
+ kick_node_from_cluster(memb->nodeid);
+ }
+
+ for (i = 0; i < joined_list_entries; i++) {
+ memb = find_memb(cg, joined_list[i].nodeid);
+ if (!memb) {
+ log_error("no member %d", joined_list[i].nodeid);
+ error = -ENOENT;
+ goto fail;
+ }
+ memb->added = 1;
+
+ if (memb->nodeid == our_nodeid)
+ cg->we_joined = 1;
+ else
+ node_history_init(ls, memb->nodeid, cg);
+
+ log_group(ls, "add_change cg %u joined nodeid %d", cg->seq,
+ memb->nodeid);
+ }
+
+ if (cg->we_joined) {
+ log_group(ls, "add_change cg %u we joined", cg->seq);
+ list_for_each_entry(memb, &cg->members, list)
+ node_history_init(ls, memb->nodeid, cg);
+ }
+
+ log_group(ls, "add_change cg %u counts member %d joined %d remove %d "
+ "failed %d", cg->seq, cg->member_count, cg->joined_count,
+ cg->remove_count, cg->failed_count);
+
+ list_add(&cg->list, &ls->changes);
+ *cg_out = cg;
+ return 0;
+
+ fail_nomem:
+ log_error("no memory");
+ error = -ENOMEM;
+ fail:
+ free_cg(cg);
+ return error;
+}
+
+static int we_left(const struct cpg_address *left_list,
+ size_t left_list_entries)
+{
+ int i;
+
+ for (i = 0; i < left_list_entries; i++) {
+ if (left_list[i].nodeid == our_nodeid)
+ return 1;
+ }
+ return 0;
+}
+
+static void confchg_cb(cpg_handle_t handle,
+ const struct cpg_name *group_name,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries)
+{
+ struct lockspace *ls;
+ struct change *cg;
+ struct member *memb;
+ int rv;
+
+ log_config(group_name, member_list, member_list_entries,
+ left_list, left_list_entries,
+ joined_list, joined_list_entries);
+
+ ls = find_ls_handle(handle);
+ if (!ls) {
+ log_error("confchg_cb no lockspace for cpg %s",
+ group_name->value);
+ return;
+ }
+
+ if (ls->leaving && we_left(left_list, left_list_entries)) {
+ /* we called cpg_leave(), and this should be the final
+ cpg callback we receive */
+ log_group(ls, "confchg for our leave");
+ stop_kernel(ls, 0);
+ set_configfs_members(ls->name, 0, NULL, 0, NULL);
+ set_sysfs_event_done(ls->name, 0);
+ cpg_finalize(ls->cpg_handle);
+ client_dead(ls->cpg_client);
+ purge_plocks(ls, our_nodeid, 1);
+ list_del(&ls->list);
+ free_ls(ls);
+ return;
+ }
+
+ rv = add_change(ls, member_list, member_list_entries,
+ left_list, left_list_entries,
+ joined_list, joined_list_entries, &cg);
+ if (rv)
+ return;
+
+ stop_kernel(ls, cg->seq);
+
+ list_for_each_entry(memb, &cg->removed, list)
+ purge_plocks(ls, memb->nodeid, 0);
+
+ apply_changes(ls);
+
+#if 0
+ deadlk_confchg(ls, member_list, member_list_entries,
+ left_list, left_list_entries,
+ joined_list, joined_list_entries);
+#endif
+}
+
+static void dlm_header_in(struct dlm_header *hd)
+{
+ hd->version[0] = le16_to_cpu(hd->version[0]);
+ hd->version[1] = le16_to_cpu(hd->version[1]);
+ hd->version[2] = le16_to_cpu(hd->version[2]);
+ hd->type = le16_to_cpu(hd->type);
+ hd->nodeid = le32_to_cpu(hd->nodeid);
+ hd->to_nodeid = le32_to_cpu(hd->to_nodeid);
+ hd->global_id = le32_to_cpu(hd->global_id);
+ hd->flags = le32_to_cpu(hd->flags);
+ hd->msgdata = le32_to_cpu(hd->msgdata);
+ hd->msgdata2 = le32_to_cpu(hd->msgdata2);
+}
+
+/* after our join confchg, we want to ignore plock messages (see need_plocks
+ checks below) until the point in time where the ckpt_node saves plock
+ state (final start message received); at this time we want to shift from
+ ignoring plock messages to saving plock messages to apply on top of the
+ plock state that we read. */
+
+static void deliver_cb(cpg_handle_t handle,
+ const struct cpg_name *group_name,
+ uint32_t nodeid, uint32_t pid,
+ void *data, size_t len)
+{
+ struct lockspace *ls;
+ struct dlm_header *hd;
+ int ignore_plock;
+
+ ls = find_ls_handle(handle);
+ if (!ls) {
+ log_error("deliver_cb no ls for cpg %s", group_name->value);
+ return;
+ }
+
+ if (len < sizeof(*hd)) {
+ log_error("deliver_cb short message %zd", len);
+ return;
+ }
+
+ hd = (struct dlm_header *)data;
+ dlm_header_in(hd);
+
+ if (hd->version[0] != our_protocol.daemon_run[0] ||
+ hd->version[1] != our_protocol.daemon_run[1]) {
+ log_error("reject message from %d version %u.%u.%u vs %u.%u.%u",
+ nodeid, hd->version[0], hd->version[1],
+ hd->version[2], our_protocol.daemon_run[0],
+ our_protocol.daemon_run[1],
+ our_protocol.daemon_run[2]);
+ return;
+ }
+
+ if (hd->nodeid != nodeid) {
+ log_error("bad msg nodeid %d %d", hd->nodeid, nodeid);
+ return;
+ }
+
+ ignore_plock = 0;
+
+ switch (hd->type) {
+ case DLM_MSG_START:
+ receive_start(ls, hd, len);
+ break;
+
+ case DLM_MSG_PLOCK:
+ if (ls->disable_plock)
+ break;
+ if (ls->need_plocks && !ls->save_plocks) {
+ ignore_plock = 1;
+ break;
+ }
+ if (cfgd_enable_plock)
+ receive_plock(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_plock %d",
+ hd->type, nodeid, cfgd_enable_plock);
+ break;
+
+ case DLM_MSG_PLOCK_OWN:
+ if (ls->disable_plock)
+ break;
+ if (ls->need_plocks && !ls->save_plocks) {
+ ignore_plock = 1;
+ break;
+ }
+ if (cfgd_enable_plock && cfgd_plock_ownership)
+ receive_own(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_plock %d owner %d",
+ hd->type, nodeid, cfgd_enable_plock,
+ cfgd_plock_ownership);
+ break;
+
+ case DLM_MSG_PLOCK_DROP:
+ if (ls->disable_plock)
+ break;
+ if (ls->need_plocks && !ls->save_plocks) {
+ ignore_plock = 1;
+ break;
+ }
+ if (cfgd_enable_plock && cfgd_plock_ownership)
+ receive_drop(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_plock %d owner %d",
+ hd->type, nodeid, cfgd_enable_plock,
+ cfgd_plock_ownership);
+ break;
+
+ case DLM_MSG_PLOCK_SYNC_LOCK:
+ case DLM_MSG_PLOCK_SYNC_WAITER:
+ if (ls->disable_plock)
+ break;
+ if (ls->need_plocks && !ls->save_plocks) {
+ ignore_plock = 1;
+ break;
+ }
+ if (cfgd_enable_plock && cfgd_plock_ownership)
+ receive_sync(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_plock %d owner %d",
+ hd->type, nodeid, cfgd_enable_plock,
+ cfgd_plock_ownership);
+ break;
+
+ case DLM_MSG_PLOCKS_DATA:
+ if (ls->disable_plock)
+ break;
+ if (cfgd_enable_plock)
+ receive_plocks_data(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_plock %d",
+ hd->type, nodeid, cfgd_enable_plock);
+ break;
+
+ case DLM_MSG_PLOCKS_DONE:
+ if (ls->disable_plock)
+ break;
+ if (cfgd_enable_plock)
+ receive_plocks_done(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_plock %d",
+ hd->type, nodeid, cfgd_enable_plock);
+ break;
+
+#if 0
+ case DLM_MSG_DEADLK_CYCLE_START:
+ if (cfgd_enable_deadlk)
+ receive_cycle_start(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_deadlk %d",
+ hd->type, nodeid, cfgd_enable_deadlk);
+ break;
+
+ case DLM_MSG_DEADLK_CYCLE_END:
+ if (cfgd_enable_deadlk)
+ receive_cycle_end(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_deadlk %d",
+ hd->type, nodeid, cfgd_enable_deadlk);
+ break;
+
+ case DLM_MSG_DEADLK_CHECKPOINT_READY:
+ if (cfgd_enable_deadlk)
+ receive_checkpoint_ready(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_deadlk %d",
+ hd->type, nodeid, cfgd_enable_deadlk);
+ break;
+
+ case DLM_MSG_DEADLK_CANCEL_LOCK:
+ if (cfgd_enable_deadlk)
+ receive_cancel_lock(ls, hd, len);
+ else
+ log_error("msg %d nodeid %d enable_deadlk %d",
+ hd->type, nodeid, cfgd_enable_deadlk);
+ break;
+#endif
+
+ default:
+ log_error("unknown msg type %d", hd->type);
+ }
+
+ if (ignore_plock)
+ log_plock(ls, "msg %s nodeid %d need_plock ignore",
+ msg_name(hd->type), nodeid);
+
+ apply_changes(ls);
+}
+
+static cpg_callbacks_t cpg_callbacks = {
+ .cpg_deliver_fn = deliver_cb,
+ .cpg_confchg_fn = confchg_cb,
+};
+
+void update_flow_control_status(void)
+{
+ cpg_flow_control_state_t flow_control_state;
+ cpg_error_t error;
+
+ error = cpg_flow_control_state_get(cpg_handle_daemon,
+ &flow_control_state);
+ if (error != CPG_OK) {
+ log_error("cpg_flow_control_state_get %d", error);
+ return;
+ }
+
+ if (flow_control_state == CPG_FLOW_CONTROL_ENABLED) {
+ if (message_flow_control_on == 0) {
+ log_debug("flow control on");
+ }
+ message_flow_control_on = 1;
+ } else {
+ if (message_flow_control_on) {
+ log_debug("flow control off");
+ }
+ message_flow_control_on = 0;
+ }
+}
+
+static void process_cpg_lockspace(int ci)
+{
+ struct lockspace *ls;
+ cpg_error_t error;
+
+ ls = find_ls_ci(ci);
+ if (!ls) {
+ log_error("process_lockspace_cpg no lockspace for ci %d", ci);
+ return;
+ }
+
+ error = cpg_dispatch(ls->cpg_handle, CPG_DISPATCH_ALL);
+ if (error != CPG_OK) {
+ log_error("cpg_dispatch error %d", error);
+ return;
+ }
+
+ update_flow_control_status();
+}
+
+/* received an "online" uevent from dlm-kernel */
+
+int dlm_join_lockspace(struct lockspace *ls)
+{
+ cpg_error_t error;
+ cpg_handle_t h;
+ struct cpg_name name;
+ int i = 0, fd, ci, rv;
+ int unused;
+
+ rv = fence_in_progress(&unused);
+ if (cfgd_enable_fencing && rv < 0) {
+ log_error("dlm_join_lockspace no fence domain");
+ rv = -1;
+ goto fail_free;
+ }
+
+ error = cpg_initialize(&h, &cpg_callbacks);
+ if (error != CPG_OK) {
+ log_error("cpg_initialize error %d", error);
+ rv = -1;
+ goto fail_free;
+ }
+
+ cpg_fd_get(h, &fd);
+
+ ci = client_add(fd, process_cpg_lockspace, NULL);
+
+ list_add(&ls->list, &lockspaces);
+
+ ls->cpg_handle = h;
+ ls->cpg_client = ci;
+ ls->cpg_fd = fd;
+ ls->kernel_stopped = 1;
+ ls->need_plocks = 1;
+ ls->joining = 1;
+
+ memset(&name, 0, sizeof(name));
+ sprintf(name.value, "dlm:ls:%s", ls->name);
+ name.length = strlen(name.value) + 1;
+
+ /* TODO: allow global_id to be set in cluster.conf? */
+ ls->global_id = cpgname_to_crc(name.value, name.length);
+
+ retry:
+ error = cpg_join(h, &name);
+ if (error == CPG_ERR_TRY_AGAIN) {
+ sleep(1);
+ if (!(++i % 10))
+ log_error("cpg_join error retrying");
+ goto retry;
+ }
+ if (error != CPG_OK) {
+ log_error("cpg_join error %d", error);
+ cpg_finalize(h);
+ rv = -1;
+ goto fail;
+ }
+
+ return 0;
+
+ fail:
+ list_del(&ls->list);
+ client_dead(ci);
+ cpg_finalize(h);
+ fail_free:
+ set_sysfs_event_done(ls->name, rv);
+ free_ls(ls);
+ return rv;
+}
+
+/* received an "offline" uevent from dlm-kernel */
+
+int dlm_leave_lockspace(struct lockspace *ls)
+{
+ cpg_error_t error;
+ struct cpg_name name;
+ int i = 0;
+
+ ls->leaving = 1;
+
+ memset(&name, 0, sizeof(name));
+ sprintf(name.value, "dlm:ls:%s", ls->name);
+ name.length = strlen(name.value) + 1;
+
+ retry:
+ error = cpg_leave(ls->cpg_handle, &name);
+ if (error == CPG_ERR_TRY_AGAIN) {
+ sleep(1);
+ if (!(++i % 10))
+ log_error("cpg_leave error retrying");
+ goto retry;
+ }
+ if (error != CPG_OK)
+ log_error("cpg_leave error %d", error);
+
+ return 0;
+}
+
+static struct node *get_node_daemon(int nodeid)
+{
+ struct node *node;
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (node->nodeid == nodeid)
+ return node;
+ }
+ return NULL;
+}
+
+static void add_node_daemon(int nodeid)
+{
+ struct node *node;
+
+ if (get_node_daemon(nodeid))
+ return;
+
+ node = malloc(sizeof(struct node));
+ if (!node) {
+ log_error("add_node_daemon no mem");
+ return;
+ }
+ memset(node, 0, sizeof(struct node));
+ node->nodeid = nodeid;
+ list_add_tail(&node->list, &daemon_nodes);
+}
+
+static void pv_in(struct protocol_version *pv)
+{
+ pv->major = le16_to_cpu(pv->major);
+ pv->minor = le16_to_cpu(pv->minor);
+ pv->patch = le16_to_cpu(pv->patch);
+ pv->flags = le16_to_cpu(pv->flags);
+}
+
+static void pv_out(struct protocol_version *pv)
+{
+ pv->major = cpu_to_le16(pv->major);
+ pv->minor = cpu_to_le16(pv->minor);
+ pv->patch = cpu_to_le16(pv->patch);
+ pv->flags = cpu_to_le16(pv->flags);
+}
+
+static void protocol_in(struct protocol *proto)
+{
+ pv_in(&proto->dm_ver);
+ pv_in(&proto->km_ver);
+ pv_in(&proto->dr_ver);
+ pv_in(&proto->kr_ver);
+}
+
+static void protocol_out(struct protocol *proto)
+{
+ pv_out(&proto->dm_ver);
+ pv_out(&proto->km_ver);
+ pv_out(&proto->dr_ver);
+ pv_out(&proto->kr_ver);
+}
+
+/* go through member list saved in last confchg, see if we have received a
+ proto message from each */
+
+static int all_protocol_messages(void)
+{
+ struct node *node;
+ int i;
+
+ if (!daemon_member_count)
+ return 0;
+
+ for (i = 0; i < daemon_member_count; i++) {
+ node = get_node_daemon(daemon_member[i].nodeid);
+ if (!node) {
+ log_error("all_protocol_messages no node %d",
+ daemon_member[i].nodeid);
+ return 0;
+ }
+
+ if (!node->proto.daemon_max[0])
+ return 0;
+ }
+ return 1;
+}
+
+static int pick_min_protocol(struct protocol *proto)
+{
+ uint16_t mind[4];
+ uint16_t mink[4];
+ struct node *node;
+ int i;
+
+ memset(&mind, 0, sizeof(mind));
+ memset(&mink, 0, sizeof(mink));
+
+ /* first choose the minimum major */
+
+ for (i = 0; i < daemon_member_count; i++) {
+ node = get_node_daemon(daemon_member[i].nodeid);
+ if (!node) {
+ log_error("pick_min_protocol no node %d",
+ daemon_member[i].nodeid);
+ return -1;
+ }
+
+ if (!mind[0] || node->proto.daemon_max[0] < mind[0])
+ mind[0] = node->proto.daemon_max[0];
+
+ if (!mink[0] || node->proto.kernel_max[0] < mink[0])
+ mink[0] = node->proto.kernel_max[0];
+ }
+
+ if (!mind[0] || !mink[0]) {
+ log_error("pick_min_protocol zero major number");
+ return -1;
+ }
+
+ /* second pick the minimum minor with the chosen major */
+
+ for (i = 0; i < daemon_member_count; i++) {
+ node = get_node_daemon(daemon_member[i].nodeid);
+ if (!node)
+ continue;
+
+ if (mind[0] == node->proto.daemon_max[0]) {
+ if (!mind[1] || node->proto.daemon_max[1] < mind[1])
+ mind[1] = node->proto.daemon_max[1];
+ }
+
+ if (mink[0] == node->proto.kernel_max[0]) {
+ if (!mink[1] || node->proto.kernel_max[1] < mink[1])
+ mink[1] = node->proto.kernel_max[1];
+ }
+ }
+
+ if (!mind[1] || !mink[1]) {
+ log_error("pick_min_protocol zero minor number");
+ return -1;
+ }
+
+ /* third pick the minimum patch with the chosen major.minor */
+
+ for (i = 0; i < daemon_member_count; i++) {
+ node = get_node_daemon(daemon_member[i].nodeid);
+ if (!node)
+ continue;
+
+ if (mind[0] == node->proto.daemon_max[0] &&
+ mind[1] == node->proto.daemon_max[1]) {
+ if (!mind[2] || node->proto.daemon_max[2] < mind[2])
+ mind[2] = node->proto.daemon_max[2];
+ }
+
+ if (mink[0] == node->proto.kernel_max[0] &&
+ mink[1] == node->proto.kernel_max[1]) {
+ if (!mink[2] || node->proto.kernel_max[2] < mink[2])
+ mink[2] = node->proto.kernel_max[2];
+ }
+ }
+
+ if (!mind[2] || !mink[2]) {
+ log_error("pick_min_protocol zero patch number");
+ return -1;
+ }
+
+ memcpy(&proto->daemon_run, &mind, sizeof(mind));
+ memcpy(&proto->kernel_run, &mink, sizeof(mink));
+ return 0;
+}
+
+static void receive_protocol(struct dlm_header *hd, int len)
+{
+ struct protocol *p;
+ struct node *node;
+
+ p = (struct protocol *)((char *)hd + sizeof(struct dlm_header));
+ protocol_in(p);
+
+ if (len < sizeof(struct dlm_header) + sizeof(struct protocol)) {
+ log_error("receive_protocol invalid len %d from %d",
+ len, hd->nodeid);
+ return;
+ }
+
+ /* zero is an invalid version value */
+
+ if (!p->daemon_max[0] || !p->daemon_max[1] || !p->daemon_max[2] ||
+ !p->kernel_max[0] || !p->kernel_max[1] || !p->kernel_max[2]) {
+ log_error("receive_protocol invalid max value from %d "
+ "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
+ p->daemon_max[0], p->daemon_max[1], p->daemon_max[2],
+ p->kernel_max[0], p->kernel_max[1], p->kernel_max[2]);
+ return;
+ }
+
+ /* the run values will be zero until a version is set, after
+ which none of the run values can be zero */
+
+ if (p->daemon_run[0] && (!p->daemon_run[1] || !p->daemon_run[2] ||
+ !p->kernel_run[0] || !p->kernel_run[1] || !p->kernel_run[2])) {
+ log_error("receive_protocol invalid run value from %d "
+ "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
+ p->daemon_run[0], p->daemon_run[1], p->daemon_run[2],
+ p->kernel_run[0], p->kernel_run[1], p->kernel_run[2]);
+ return;
+ }
+
+ /* if we have zero run values, and this msg has non-zero run values,
+ then adopt them as ours; otherwise save this proto message */
+
+ if (our_protocol.daemon_run[0])
+ return;
+
+ if (p->daemon_run[0]) {
+ memcpy(&our_protocol.daemon_run, &p->daemon_run,
+ sizeof(struct protocol_version));
+ memcpy(&our_protocol.kernel_run, &p->kernel_run,
+ sizeof(struct protocol_version));
+ log_debug("run protocol from nodeid %d", hd->nodeid);
+ return;
+ }
+
+ /* save this node's proto so we can tell when we've got all, and
+ use it to select a minimum protocol from all */
+
+ node = get_node_daemon(hd->nodeid);
+ if (!node) {
+ log_error("receive_protocol no node %d", hd->nodeid);
+ return;
+ }
+ memcpy(&node->proto, p, sizeof(struct protocol));
+}
+
+static void send_protocol(struct protocol *proto)
+{
+ struct dlm_header *hd;
+ struct protocol *pr;
+ char *buf;
+ int len;
+
+ len = sizeof(struct dlm_header) + sizeof(struct protocol);
+ buf = malloc(len);
+ if (!buf) {
+ log_error("send_protocol no mem %d", len);
+ return;
+ }
+ memset(buf, 0, len);
+
+ hd = (struct dlm_header *)buf;
+ pr = (struct protocol *)(buf + sizeof(*hd));
+
+ hd->type = cpu_to_le16(DLM_MSG_PROTOCOL);
+ hd->nodeid = cpu_to_le32(our_nodeid);
+
+ memcpy(pr, proto, sizeof(struct protocol));
+ protocol_out(pr);
+
+ _send_message(cpg_handle_daemon, buf, len, DLM_MSG_PROTOCOL);
+}
+
+int set_protocol(void)
+{
+ struct protocol proto;
+ struct pollfd pollfd;
+ int sent_proposal = 0;
+ int rv;
+
+ memset(&pollfd, 0, sizeof(pollfd));
+ pollfd.fd = cpg_fd_daemon;
+ pollfd.events = POLLIN;
+
+ while (1) {
+ if (our_protocol.daemon_run[0])
+ break;
+
+ if (!sent_proposal && all_protocol_messages()) {
+ /* propose a protocol; look through info from all
+ nodes and pick the min for both daemon and kernel,
+ and propose that */
+
+ sent_proposal = 1;
+
+ /* copy our max values */
+ memcpy(&proto, &our_protocol, sizeof(struct protocol));
+
+ rv = pick_min_protocol(&proto);
+ if (rv < 0)
+ return rv;
+
+ log_debug("set_protocol member_count %d propose "
+ "daemon %u.%u.%u kernel %u.%u.%u",
+ daemon_member_count,
+ proto.daemon_run[0], proto.daemon_run[1],
+ proto.daemon_run[2], proto.kernel_run[0],
+ proto.kernel_run[1], proto.kernel_run[2]);
+
+ send_protocol(&proto);
+ }
+
+ /* only process messages/events from daemon cpg until protocol
+ is established */
+
+ rv = poll(&pollfd, 1, -1);
+ if (rv == -1 && errno == EINTR) {
+ if (daemon_quit)
+ return -1;
+ continue;
+ }
+ if (rv < 0) {
+ log_error("set_protocol poll errno %d", errno);
+ return -1;
+ }
+
+ if (pollfd.revents & POLLIN)
+ process_cpg_daemon(0);
+ if (pollfd.revents & (POLLERR | POLLHUP | POLLNVAL)) {
+ log_error("set_protocol poll revents %u",
+ pollfd.revents);
+ return -1;
+ }
+ }
+
+ if (our_protocol.daemon_run[0] != our_protocol.daemon_max[0] ||
+ our_protocol.daemon_run[1] > our_protocol.daemon_max[1]) {
+ log_error("incompatible daemon protocol run %u.%u.%u max %u.%u.%u",
+ our_protocol.daemon_run[0],
+ our_protocol.daemon_run[1],
+ our_protocol.daemon_run[2],
+ our_protocol.daemon_max[0],
+ our_protocol.daemon_max[1],
+ our_protocol.daemon_max[2]);
+ return -1;
+ }
+
+ if (our_protocol.kernel_run[0] != our_protocol.kernel_max[0] ||
+ our_protocol.kernel_run[1] > our_protocol.kernel_max[1]) {
+ log_error("incompatible kernel protocol run %u.%u.%u max %u.%u.%u",
+ our_protocol.kernel_run[0],
+ our_protocol.kernel_run[1],
+ our_protocol.kernel_run[2],
+ our_protocol.kernel_max[0],
+ our_protocol.kernel_max[1],
+ our_protocol.kernel_max[2]);
+ return -1;
+ }
+
+ log_debug("daemon run %u.%u.%u max %u.%u.%u "
+ "kernel run %u.%u.%u max %u.%u.%u",
+ our_protocol.daemon_run[0],
+ our_protocol.daemon_run[1],
+ our_protocol.daemon_run[2],
+ our_protocol.daemon_max[0],
+ our_protocol.daemon_max[1],
+ our_protocol.daemon_max[2],
+ our_protocol.kernel_run[0],
+ our_protocol.kernel_run[1],
+ our_protocol.kernel_run[2],
+ our_protocol.kernel_max[0],
+ our_protocol.kernel_max[1],
+ our_protocol.kernel_max[2]);
+
+ send_protocol(&our_protocol);
+ return 0;
+}
+
+static void deliver_cb_daemon(cpg_handle_t handle,
+ const struct cpg_name *group_name,
+ uint32_t nodeid, uint32_t pid,
+ void *data, size_t len)
+{
+ struct dlm_header *hd;
+
+ if (len < sizeof(*hd)) {
+ log_error("deliver_cb short message %zd", len);
+ return;
+ }
+
+ hd = (struct dlm_header *)data;
+ dlm_header_in(hd);
+
+ switch (hd->type) {
+ case DLM_MSG_PROTOCOL:
+ receive_protocol(hd, len);
+ break;
+ default:
+ log_error("deliver_cb_daemon unknown msg type %d", hd->type);
+ }
+}
+
+static void confchg_cb_daemon(cpg_handle_t handle,
+ const struct cpg_name *group_name,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries)
+{
+ int i;
+
+ log_config(group_name, member_list, member_list_entries,
+ left_list, left_list_entries,
+ joined_list, joined_list_entries);
+
+ if (joined_list_entries)
+ send_protocol(&our_protocol);
+
+ memset(&daemon_member, 0, sizeof(daemon_member));
+ daemon_member_count = member_list_entries;
+
+ for (i = 0; i < member_list_entries; i++) {
+ daemon_member[i] = member_list[i];
+ add_node_daemon(member_list[i].nodeid);
+ }
+}
+
+static cpg_callbacks_t cpg_callbacks_daemon = {
+ .cpg_deliver_fn = deliver_cb_daemon,
+ .cpg_confchg_fn = confchg_cb_daemon,
+};
+
+void process_cpg_daemon(int ci)
+{
+ cpg_error_t error;
+
+ error = cpg_dispatch(cpg_handle_daemon, CPG_DISPATCH_ALL);
+ if (error != CPG_OK)
+ log_error("daemon cpg_dispatch error %d", error);
+}
+
+int setup_cpg_daemon(void)
+{
+ cpg_error_t error;
+ struct cpg_name name;
+ int i = 0;
+
+ INIT_LIST_HEAD(&daemon_nodes);
+
+ memset(&our_protocol, 0, sizeof(our_protocol));
+ our_protocol.daemon_max[0] = 2;
+ our_protocol.daemon_max[1] = 1;
+ our_protocol.daemon_max[2] = 1;
+ our_protocol.kernel_max[0] = 1;
+ our_protocol.kernel_max[1] = 1;
+ our_protocol.kernel_max[2] = 1;
+
+ error = cpg_initialize(&cpg_handle_daemon, &cpg_callbacks_daemon);
+ if (error != CPG_OK) {
+ log_error("daemon cpg_initialize error %d", error);
+ return -1;
+ }
+
+ cpg_fd_get(cpg_handle_daemon, &cpg_fd_daemon);
+
+ memset(&name, 0, sizeof(name));
+ sprintf(name.value, "dlm:controld");
+ name.length = strlen(name.value) + 1;
+
+ retry:
+ error = cpg_join(cpg_handle_daemon, &name);
+ if (error == CPG_ERR_TRY_AGAIN) {
+ sleep(1);
+ if (!(++i % 10))
+ log_error("daemon cpg_join error retrying");
+ goto retry;
+ }
+ if (error != CPG_OK) {
+ log_error("daemon cpg_join error %d", error);
+ goto fail;
+ }
+
+ log_debug("setup_cpg_daemon %d", cpg_fd_daemon);
+ return cpg_fd_daemon;
+
+ fail:
+ cpg_finalize(cpg_handle_daemon);
+ return -1;
+}
+
+void close_cpg_daemon(void)
+{
+ struct lockspace *ls;
+ cpg_error_t error;
+ struct cpg_name name;
+ int i = 0;
+
+ if (!cpg_handle_daemon)
+ return;
+ if (cluster_down)
+ goto fin;
+
+ memset(&name, 0, sizeof(name));
+ sprintf(name.value, "dlm:controld");
+ name.length = strlen(name.value) + 1;
+
+ retry:
+ error = cpg_leave(cpg_handle_daemon, &name);
+ if (error == CPG_ERR_TRY_AGAIN) {
+ sleep(1);
+ if (!(++i % 10))
+ log_error("daemon cpg_leave error retrying");
+ goto retry;
+ }
+ if (error != CPG_OK)
+ log_error("daemon cpg_leave error %d", error);
+ fin:
+ list_for_each_entry(ls, &lockspaces, list) {
+ if (ls->cpg_handle)
+ cpg_finalize(ls->cpg_handle);
+ }
+ cpg_finalize(cpg_handle_daemon);
+}
+
+/* fs_controld has seen nodedown for nodeid; it's now ok for dlm to do
+ recovery for the failed node */
+
+int set_fs_notified(struct lockspace *ls, int nodeid)
+{
+ struct node *node;
+
+ /* this shouldn't happen */
+ node = get_node_history(ls, nodeid);
+ if (!node) {
+ log_error("set_fs_notified no nodeid %d", nodeid);
+ return -ESRCH;
+ }
+
+ if (!find_memb(ls->started_change, nodeid)) {
+ log_group(ls, "set_fs_notified %d not in ls", nodeid);
+ return 0;
+ }
+
+ /* this can happen, we haven't seen a nodedown for this node yet,
+ but we should soon */
+ if (!node->check_fs) {
+ log_group(ls, "set_fs_notified %d zero check_fs", nodeid);
+ return -EAGAIN;
+ }
+
+ log_group(ls, "set_fs_notified nodeid %d", nodeid);
+ node->fs_notified = 1;
+ return 0;
+}
+
+int set_lockspace_info(struct lockspace *ls, struct dlmc_lockspace *lockspace)
+{
+ struct change *cg, *last = NULL;
+
+ strncpy(lockspace->name, ls->name, DLM_LOCKSPACE_LEN);
+ lockspace->global_id = ls->global_id;
+
+ if (ls->joining)
+ lockspace->flags |= DLMC_LF_JOINING;
+ if (ls->leaving)
+ lockspace->flags |= DLMC_LF_LEAVING;
+ if (ls->kernel_stopped)
+ lockspace->flags |= DLMC_LF_KERNEL_STOPPED;
+ if (ls->fs_registered)
+ lockspace->flags |= DLMC_LF_FS_REGISTERED;
+ if (ls->need_plocks)
+ lockspace->flags |= DLMC_LF_NEED_PLOCKS;
+ if (ls->save_plocks)
+ lockspace->flags |= DLMC_LF_SAVE_PLOCKS;
+
+ if (!ls->started_change)
+ goto next;
+
+ cg = ls->started_change;
+
+ lockspace->cg_prev.member_count = cg->member_count;
+ lockspace->cg_prev.joined_count = cg->joined_count;
+ lockspace->cg_prev.remove_count = cg->remove_count;
+ lockspace->cg_prev.failed_count = cg->failed_count;
+ lockspace->cg_prev.combined_seq = cg->combined_seq;
+ lockspace->cg_prev.seq = cg->seq;
+
+ next:
+ if (list_empty(&ls->changes))
+ goto out;
+
+ list_for_each_entry(cg, &ls->changes, list)
+ last = cg;
+
+ cg = list_first_entry(&ls->changes, struct change, list);
+
+ lockspace->cg_next.member_count = cg->member_count;
+ lockspace->cg_next.joined_count = cg->joined_count;
+ lockspace->cg_next.remove_count = cg->remove_count;
+ lockspace->cg_next.failed_count = cg->failed_count;
+ lockspace->cg_next.combined_seq = last->seq;
+ lockspace->cg_next.seq = cg->seq;
+
+ if (cg->state == CGST_WAIT_CONDITIONS)
+ lockspace->cg_next.wait_condition = 4;
+ if (poll_fencing)
+ lockspace->cg_next.wait_condition = 1;
+ else if (poll_quorum)
+ lockspace->cg_next.wait_condition = 2;
+ else if (poll_fs)
+ lockspace->cg_next.wait_condition = 3;
+
+ if (cg->state == CGST_WAIT_MESSAGES)
+ lockspace->cg_next.wait_messages = 1;
+ out:
+ return 0;
+}
+
+static int _set_node_info(struct lockspace *ls, struct change *cg, int nodeid,
+ struct dlmc_node *node)
+{
+ struct member *m = NULL;
+ struct node *n;
+
+ node->nodeid = nodeid;
+
+ if (cg)
+ m = find_memb(cg, nodeid);
+ if (!m)
+ goto history;
+
+ node->flags |= DLMC_NF_MEMBER;
+
+ if (m->start)
+ node->flags |= DLMC_NF_START;
+ if (m->disallowed)
+ node->flags |= DLMC_NF_DISALLOWED;
+
+ history:
+ n = get_node_history(ls, nodeid);
+ if (!n)
+ goto out;
+
+ if (n->check_fencing)
+ node->flags |= DLMC_NF_CHECK_FENCING;
+ if (n->check_quorum)
+ node->flags |= DLMC_NF_CHECK_QUORUM;
+ if (n->check_fs)
+ node->flags |= DLMC_NF_CHECK_FS;
+
+ node->added_seq = n->added_seq;
+ node->removed_seq = n->removed_seq;
+ node->failed_reason = n->failed_reason;
+ out:
+ return 0;
+}
+
+int set_node_info(struct lockspace *ls, int nodeid, struct dlmc_node *node)
+{
+ struct change *cg;
+
+ if (!list_empty(&ls->changes)) {
+ cg = list_first_entry(&ls->changes, struct change, list);
+ return _set_node_info(ls, cg, nodeid, node);
+ }
+
+ return _set_node_info(ls, ls->started_change, nodeid, node);
+}
+
+int set_lockspaces(int *count, struct dlmc_lockspace **lss_out)
+{
+ struct lockspace *ls;
+ struct dlmc_lockspace *lss, *lsp;
+ int ls_count = 0;
+
+ list_for_each_entry(ls, &lockspaces, list)
+ ls_count++;
+
+ lss = malloc(ls_count * sizeof(struct dlmc_lockspace));
+ if (!lss)
+ return -ENOMEM;
+ memset(lss, 0, ls_count * sizeof(struct dlmc_lockspace));
+
+ lsp = lss;
+ list_for_each_entry(ls, &lockspaces, list) {
+ set_lockspace_info(ls, lsp++);
+ }
+
+ *count = ls_count;
+ *lss_out = lss;
+ return 0;
+}
+
+int set_lockspace_nodes(struct lockspace *ls, int option, int *node_count,
+ struct dlmc_node **nodes_out)
+{
+ struct change *cg;
+ struct node *n;
+ struct dlmc_node *nodes = NULL, *nodep;
+ struct member *memb;
+ int count = 0;
+
+ if (option == DLMC_NODES_ALL) {
+ if (!list_empty(&ls->changes))
+ cg = list_first_entry(&ls->changes, struct change,list);
+ else
+ cg = ls->started_change;
+
+ list_for_each_entry(n, &ls->node_history, list)
+ count++;
+
+ } else if (option == DLMC_NODES_MEMBERS) {
+ if (!ls->started_change)
+ goto out;
+ cg = ls->started_change;
+ count = cg->member_count;
+
+ } else if (option == DLMC_NODES_NEXT) {
+ if (list_empty(&ls->changes))
+ goto out;
+ cg = list_first_entry(&ls->changes, struct change, list);
+ count = cg->member_count;
+ } else
+ goto out;
+
+ nodes = malloc(count * sizeof(struct dlmc_node));
+ if (!nodes)
+ return -ENOMEM;
+ memset(nodes, 0, count * sizeof(struct dlmc_node));
+ nodep = nodes;
+
+ if (option == DLMC_NODES_ALL) {
+ list_for_each_entry(n, &ls->node_history, list)
+ _set_node_info(ls, cg, n->nodeid, nodep++);
+ } else {
+ list_for_each_entry(memb, &cg->members, list)
+ _set_node_info(ls, cg, memb->nodeid, nodep++);
+ }
+ out:
+ *node_count = count;
+ *nodes_out = nodes;
+ return 0;
+}
+
diff --git a/dlm_controld/crc.c b/dlm_controld/crc.c
new file mode 100644
index 0000000..ff8c1d3
--- /dev/null
+++ b/dlm_controld/crc.c
@@ -0,0 +1,72 @@
+#include "dlm_daemon.h"
+
+static const uint32_t crc_32_tab[] = {
+ 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535,
0x9e6495a3,
+ 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
0x90bf1d91,
+ 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551,
0x83d385c7,
+ 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63,
0x8d080df5,
+ 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd,
0xa50ab56b,
+ 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf,
0xabd13d59,
+ 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
0xb8bda50f,
+ 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab,
0xb6662d3d,
+ 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5,
0xe8b8d433,
+ 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97,
0xe6635c01,
+ 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1,
0xf50fc457,
+ 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
0xfbd44c65,
+ 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d,
0xd3d6f4fb,
+ 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f,
0xdd0d7cc9,
+ 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409,
0xce61e49f,
+ 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b,
0xc0ba6cad,
+ 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
0x73dc1683,
+ 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27,
0x7d079eb1,
+ 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671,
0x6e6b06e7,
+ 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43,
0x60b08ed5,
+ 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd,
0x48b2364b,
+ 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
0x4669be79,
+ 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9,
0x5505262f,
+ 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b,
0x5bdeae1d,
+ 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785,
0x05005713,
+ 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7,
0x0bdbdf21,
+ 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
0x18b74777,
+ 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3,
0x166ccf45,
+ 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d,
0x3e6e77db,
+ 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f,
0x30b5ffe9,
+ 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729,
0x23d967bf,
+ 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
0x2d02ef8d
+};
+
+/**
+ *
+ * Copied from:
+ *
+ * gfs2_disk_hash - hash an array of data
+ * @data: the data to be hashed
+ * @len: the length of data to be hashed
+ *
+ * This function must produce the same results as the one in the kernel:
+ * crc32_le(0xFFFFFFFF, data, len) ^ 0xFFFFFFFF
+ *
+ * Take some data and convert it to a 32-bit hash.
+ *
+ * The hash function is a 32-bit CRC of the data. The algorithm uses
+ * the crc_32_tab table above.
+ *
+ * This may not be the fastest hash function, but it does a fair bit better
+ * at providing uniform results than the others I've looked at. That's
+ * really important for efficient directories.
+ *
+ * Returns: the hash
+ */
+
+uint32_t cpgname_to_crc(const char *data, int len)
+{
+ uint32_t hash = 0xFFFFFFFF;
+
+ for (; len--; data++)
+ hash = crc_32_tab[(hash ^ *data) & 0xFF] ^ (hash >> 8);
+
+ hash = ~hash;
+
+ return hash;
+}
+
diff --git a/dlm_controld/deadlock.c b/dlm_controld/deadlock.c
new file mode 100644
index 0000000..bd1d68c
--- /dev/null
+++ b/dlm_controld/deadlock.c
@@ -0,0 +1,1550 @@
+#include "dlm_daemon.h"
+#include "config.h"
+#include "libdlm.h"
+
+static SaCkptHandleT global_ckpt_h;
+static SaCkptCallbacksT callbacks = { 0, 0 };
+static SaVersionT version = { 'B', 1, 1 };
+static char section_buf[10 * 1024 * 1024]; /* 10MB of pack_lock's enough? */
+static uint32_t section_len;
+static uint32_t section_max;
+
+struct node {
+ struct list_head list;
+ int nodeid;
+ int checkpoint_ready; /* we've read its ckpt */
+ int in_cycle; /* participating in cycle */
+};
+
+enum {
+ LOCAL_COPY = 1,
+ MASTER_COPY = 2,
+};
+
+/* from linux/fs/dlm/dlm_internal.h */
+#define DLM_LKSTS_WAITING 1
+#define DLM_LKSTS_GRANTED 2
+#define DLM_LKSTS_CONVERT 3
+
+struct pack_lock {
+ uint64_t xid;
+ uint32_t id;
+ int nodeid;
+ uint32_t remid;
+ int ownpid;
+ uint32_t exflags;
+ uint32_t flags;
+ int8_t status;
+ int8_t grmode;
+ int8_t rqmode;
+ int8_t copy;
+};
+
+struct dlm_rsb {
+ struct list_head list;
+ struct list_head locks;
+ char name[DLM_RESNAME_MAXLEN];
+ int len;
+};
+
+/* information is saved in the lkb, and lkb->lock, from the perspective of the
+ local or master copy, not the process copy */
+
+struct dlm_lkb {
+ struct list_head list; /* r->locks */
+ struct pack_lock lock; /* data from debugfs/checkpoint */
+ int home; /* node where the lock owner lives*/
+ struct dlm_rsb *rsb; /* lock is on resource */
+ struct trans *trans; /* lock owned by this transaction */
+ struct list_head trans_list; /* tr->locks */
+ struct trans *waitfor_trans; /* the trans that's holding the
+ lock that's blocking us */
+};
+
+/* waitfor pointers alloc'ed 4 at at time */
+#define TR_NALLOC 4
+
+struct trans {
+ struct list_head list;
+ struct list_head locks;
+ uint64_t xid;
+ int others_waiting_on_us; /* count of trans's
+ pointing to us in
+ waitfor */
+ int waitfor_alloc;
+ int waitfor_count; /* count of in-use
+ waitfor slots and
+ num of trans's we're
+ waiting on */
+ struct trans **waitfor; /* waitfor_alloc trans
+ pointers */
+};
+
+static const int __dlm_compat_matrix[8][8] = {
+ /* UN NL CR CW PR PW EX PD */
+ {1, 1, 1, 1, 1, 1, 1, 0}, /* UN */
+ {1, 1, 1, 1, 1, 1, 1, 0}, /* NL */
+ {1, 1, 1, 1, 1, 1, 0, 0}, /* CR */
+ {1, 1, 1, 1, 0, 0, 0, 0}, /* CW */
+ {1, 1, 1, 0, 1, 0, 0, 0}, /* PR */
+ {1, 1, 1, 0, 0, 0, 0, 0}, /* PW */
+ {1, 1, 0, 0, 0, 0, 0, 0}, /* EX */
+ {0, 0, 0, 0, 0, 0, 0, 0} /* PD */
+};
+
+static inline int dlm_modes_compat(int mode1, int mode2)
+{
+ return __dlm_compat_matrix[mode1 + 1][mode2 + 1];
+}
+
+static const char *status_str(int lksts)
+{
+ switch (lksts) {
+ case DLM_LKSTS_WAITING:
+ return "W";
+ case DLM_LKSTS_GRANTED:
+ return "G";
+ case DLM_LKSTS_CONVERT:
+ return "C";
+ }
+ return "?";
+}
+
+static void free_resources(struct lockspace *ls)
+{
+ struct dlm_rsb *r, *r_safe;
+ struct dlm_lkb *lkb, *lkb_safe;
+
+ list_for_each_entry_safe(r, r_safe, &ls->resources, list) {
+ list_for_each_entry_safe(lkb, lkb_safe, &r->locks, list) {
+ list_del(&lkb->list);
+ if (!list_empty(&lkb->trans_list))
+ list_del(&lkb->trans_list);
+ free(lkb);
+ }
+ list_del(&r->list);
+ free(r);
+ }
+}
+
+static void free_transactions(struct lockspace *ls)
+{
+ struct trans *tr, *tr_safe;
+
+ list_for_each_entry_safe(tr, tr_safe, &ls->transactions, list) {
+ list_del(&tr->list);
+ if (tr->waitfor)
+ free(tr->waitfor);
+ free(tr);
+ }
+}
+
+static void disable_deadlock(void)
+{
+ log_error("FIXME: deadlock detection disabled");
+}
+
+void setup_deadlock(void)
+{
+ SaAisErrorT rv;
+
+ if (!cfgd_enable_deadlk)
+ return;
+
+ rv = saCkptInitialize(&global_ckpt_h, &callbacks, &version);
+ if (rv != SA_AIS_OK)
+ log_error("ckpt init error %d", rv);
+}
+
+static struct dlm_rsb *get_resource(struct lockspace *ls, char *name, int len)
+{
+ struct dlm_rsb *r;
+
+ list_for_each_entry(r, &ls->resources, list) {
+ if (r->len == len && !strncmp(r->name, name, len))
+ return r;
+ }
+
+ r = malloc(sizeof(struct dlm_rsb));
+ if (!r) {
+ log_error("get_resource: no memory");
+ disable_deadlock();
+ return NULL;
+ }
+ memset(r, 0, sizeof(struct dlm_rsb));
+ memcpy(r->name, name, len);
+ r->len = len;
+ INIT_LIST_HEAD(&r->locks);
+ list_add(&r->list, &ls->resources);
+ return r;
+}
+
+static struct dlm_lkb *create_lkb(void)
+{
+ struct dlm_lkb *lkb;
+
+ lkb = malloc(sizeof(struct dlm_lkb));
+ if (!lkb) {
+ log_error("create_lkb: no memory");
+ disable_deadlock();
+ } else {
+ memset(lkb, 0, sizeof(struct dlm_lkb));
+ INIT_LIST_HEAD(&lkb->list);
+ INIT_LIST_HEAD(&lkb->trans_list);
+ }
+ return lkb;
+}
+
+static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb)
+{
+ list_add(&lkb->list, &r->locks);
+ lkb->rsb = r;
+}
+
+/* from linux/fs/dlm/dlm_internal.h */
+#define IFL_MSTCPY 0x00010000
+
+/* called on a lock that's just been read from debugfs */
+
+static void set_copy(struct pack_lock *lock)
+{
+ uint32_t id, remid;
+
+ if (!lock->nodeid)
+ lock->copy = LOCAL_COPY;
+ else if (lock->flags & IFL_MSTCPY)
+ lock->copy = MASTER_COPY;
+ else {
+ /* process copy lock is converted to a partial master copy
+ lock that will be combined with the real master copy */
+ lock->copy = MASTER_COPY;
+ id = lock->id;
+ remid = lock->remid;
+ lock->id = remid;
+ lock->remid = id;
+ lock->nodeid = our_nodeid;
+ }
+}
+
+/* xid is always zero in the real master copy, xid should always be non-zero
+ in the partial master copy (what was a process copy) */
+/* TODO: confirm or enforce that the partial will always have non-zero xid */
+
+static int partial_master_copy(struct pack_lock *lock)
+{
+ return (lock->xid != 0);
+}
+
+static struct dlm_lkb *get_lkb(struct dlm_rsb *r, struct pack_lock *lock)
+{
+ struct dlm_lkb *lkb;
+
+ if (lock->copy != MASTER_COPY)
+ goto out;
+
+ list_for_each_entry(lkb, &r->locks, list) {
+ if (lkb->lock.nodeid == lock->nodeid &&
+ lkb->lock.id == lock->id)
+ return lkb;
+ }
+ out:
+ return create_lkb();
+}
+
+static struct dlm_lkb *add_lock(struct lockspace *ls, struct dlm_rsb *r,
+ int from_nodeid, struct pack_lock *lock)
+{
+ struct dlm_lkb *lkb;
+
+ lkb = get_lkb(r, lock);
+ if (!lkb)
+ return NULL;
+
+ switch (lock->copy) {
+ case LOCAL_COPY:
+ lkb->lock.xid = lock->xid;
+ lkb->lock.nodeid = lock->nodeid;
+ lkb->lock.id = lock->id;
+ lkb->lock.remid = lock->remid;
+ lkb->lock.ownpid = lock->ownpid;
+ lkb->lock.exflags = lock->exflags;
+ lkb->lock.flags = lock->flags;
+ lkb->lock.status = lock->status;
+ lkb->lock.grmode = lock->grmode;
+ lkb->lock.rqmode = lock->rqmode;
+ lkb->lock.copy = LOCAL_COPY;
+ lkb->home = from_nodeid;
+
+ log_group(ls, "add %s local nodeid %d id %x remid %x xid %llx",
+ r->name, lock->nodeid, lock->id, lock->remid,
+ (unsigned long long)lock->xid);
+ break;
+
+ case MASTER_COPY:
+ if (partial_master_copy(lock)) {
+ lkb->lock.xid = lock->xid;
+ lkb->lock.nodeid = lock->nodeid;
+ lkb->lock.id = lock->id;
+ lkb->lock.remid = lock->remid;
+ lkb->lock.copy = MASTER_COPY;
+ } else {
+ /* only set xid from partial master copy above */
+ lkb->lock.nodeid = lock->nodeid;
+ lkb->lock.id = lock->id;
+ lkb->lock.remid = lock->remid;
+ lkb->lock.copy = MASTER_COPY;
+ /* set other fields from real master copy */
+ lkb->lock.ownpid = lock->ownpid;
+ lkb->lock.exflags = lock->exflags;
+ lkb->lock.flags = lock->flags;
+ lkb->lock.status = lock->status;
+ lkb->lock.grmode = lock->grmode;
+ lkb->lock.rqmode = lock->rqmode;
+ }
+ lkb->home = lock->nodeid;
+
+ log_group(ls, "add %s master nodeid %d id %x remid %x xid %llx",
+ r->name, lock->nodeid, lock->id, lock->remid,
+ (unsigned long long)lock->xid);
+ break;
+ }
+
+ if (list_empty(&lkb->list))
+ add_lkb(r, lkb);
+ return lkb;
+}
+
+static void parse_r_name(char *line, char *name)
+{
+ char *p;
+ int i = 0;
+ int begin = 0;
+
+ for (p = line; ; p++) {
+ if (*p == '"') {
+ if (begin)
+ break;
+ begin = 1;
+ continue;
+ }
+ if (begin)
+ name[i++] = *p;
+ }
+}
+
+#define LOCK_LINE_MAX 1024
+
+static int read_debugfs_locks(struct lockspace *ls)
+{
+ FILE *file;
+ char path[PATH_MAX];
+ char line[LOCK_LINE_MAX];
+ struct dlm_rsb *r;
+ struct pack_lock lock;
+ char r_name[65];
+ unsigned long long xid;
+ unsigned int waiting;
+ int r_nodeid;
+ int r_len;
+ int rv;
+
+ snprintf(path, PATH_MAX, "/sys/kernel/debug/dlm/%s_locks", ls->name);
+
+ file = fopen(path, "r");
+ if (!file)
+ return -1;
+
+ /* skip the header on the first line */
+ if (!fgets(line, LOCK_LINE_MAX, file)) {
+ log_error("Unable to read %s: %d", path, errno);
+ goto out;
+ }
+
+ while (fgets(line, LOCK_LINE_MAX, file)) {
+ memset(&lock, 0, sizeof(struct pack_lock));
+
+ rv = sscanf(line, "%x %d %x %u %llu %x %x %hhd %hhd %hhd %u %d %d",
+ &lock.id,
+ &lock.nodeid,
+ &lock.remid,
+ &lock.ownpid,
+ &xid,
+ &lock.exflags,
+ &lock.flags,
+ &lock.status,
+ &lock.grmode,
+ &lock.rqmode,
+ &waiting,
+ &r_nodeid,
+ &r_len);
+
+ lock.xid = xid; /* hack to avoid warning */
+
+ if (rv != 13) {
+ log_error("invalid debugfs line %d: %s", rv, line);
+ goto out;
+ }
+
+ memset(r_name, 0, sizeof(r_name));
+ parse_r_name(line, r_name);
+
+ r = get_resource(ls, r_name, r_len);
+ if (!r)
+ break;
+
+ set_copy(&lock);
+ add_lock(ls, r, our_nodeid, &lock);
+ }
+ out:
+ fclose(file);
+ return 0;
+}
+
+static int read_checkpoint_locks(struct lockspace *ls, int from_nodeid,
+ char *numbuf, int buflen)
+{
+ struct dlm_rsb *r;
+ struct pack_lock *lock;
+ int count = section_len / sizeof(struct pack_lock);
+ int i;
+
+ r = get_resource(ls, numbuf, buflen - 1);
+ if (!r)
+ return -1;
+
+ lock = (struct pack_lock *) §ion_buf;
+
+ for (i = 0; i < count; i++) {
+ lock->xid = le64_to_cpu(lock->xid);
+ lock->id = le32_to_cpu(lock->id);
+ lock->nodeid = le32_to_cpu(lock->nodeid);
+ lock->remid = le32_to_cpu(lock->remid);
+ lock->ownpid = le32_to_cpu(lock->ownpid);
+ lock->exflags = le32_to_cpu(lock->exflags);
+ lock->flags = le32_to_cpu(lock->flags);
+
+ add_lock(ls, r, from_nodeid, lock);
+ lock++;
+ }
+ return 0;
+}
+
+static int pack_lkb_list(struct list_head *q, struct pack_lock **lockp)
+{
+ struct dlm_lkb *lkb;
+ struct pack_lock *lock = *lockp;
+ int count = 0;
+
+ list_for_each_entry(lkb, q, list) {
+ if (count + 1 > section_max) {
+ log_error("too many locks %d for ckpt buf", count);
+ break;
+ }
+
+ lock->xid = cpu_to_le64(lkb->lock.xid);
+ lock->id = cpu_to_le32(lkb->lock.id);
+ lock->nodeid = cpu_to_le32(lkb->lock.nodeid);
+ lock->remid = cpu_to_le32(lkb->lock.remid);
+ lock->ownpid = cpu_to_le32(lkb->lock.ownpid);
+ lock->exflags = cpu_to_le32(lkb->lock.exflags);
+ lock->flags = cpu_to_le32(lkb->lock.flags);
+ lock->status = lkb->lock.status;
+ lock->grmode = lkb->lock.grmode;
+ lock->rqmode = lkb->lock.rqmode;
+ lock->copy = lkb->lock.copy;
+
+ lock++;
+ count++;
+ }
+ return count;
+}
+
+static void pack_section_buf(struct lockspace *ls, struct dlm_rsb *r)
+{
+ struct pack_lock *lock;
+ int count;
+
+ memset(§ion_buf, 0, sizeof(section_buf));
+ section_max = sizeof(section_buf) / sizeof(struct pack_lock);
+
+ lock = (struct pack_lock *) §ion_buf;
+
+ count = pack_lkb_list(&r->locks, &lock);
+
+ section_len = count * sizeof(struct pack_lock);
+}
+
+static int _unlink_checkpoint(struct lockspace *ls, SaNameT *name)
+{
+ SaCkptCheckpointHandleT h;
+ SaCkptCheckpointDescriptorT s;
+ SaAisErrorT rv;
+ int ret = 0;
+ int retries;
+
+ h = (SaCkptCheckpointHandleT) ls->deadlk_ckpt_handle;
+ log_group(ls, "unlink ckpt %llx", (unsigned long long)h);
+
+ retries = 0;
+ unlink_retry:
+ rv = saCkptCheckpointUnlink(global_ckpt_h, name);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "unlink ckpt retry");
+ sleep(1);
+ if (retries++ < 10)
+ goto unlink_retry;
+ }
+ if (rv == SA_AIS_OK)
+ goto out_close;
+ if (!h)
+ goto out;
+
+ log_error("unlink ckpt error %d %s", rv, ls->name);
+ ret = -1;
+
+ retries = 0;
+ status_retry:
+ rv = saCkptCheckpointStatusGet(h, &s);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "unlink ckpt status retry");
+ sleep(1);
+ if (retries++ < 10)
+ goto status_retry;
+ }
+ if (rv != SA_AIS_OK) {
+ log_error("unlink ckpt status error %d %s", rv, ls->name);
+ goto out_close;
+ }
+
+ log_group(ls, "unlink ckpt status: size %llu, max sections %u, "
+ "max section size %llu, section count %u, mem %u",
+ (unsigned long long)s.checkpointCreationAttributes.checkpointSize,
+ s.checkpointCreationAttributes.maxSections,
+ (unsigned long long)s.checkpointCreationAttributes.maxSectionSize,
+ s.numberOfSections, s.memoryUsed);
+
+ out_close:
+ retries = 0;
+ close_retry:
+ rv = saCkptCheckpointClose(h);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "unlink ckpt close retry");
+ sleep(1);
+ if (retries++ < 10)
+ goto close_retry;
+ }
+ if (rv != SA_AIS_OK) {
+ log_error("unlink ckpt %llx close err %d %s",
+ (unsigned long long)h, rv, ls->name);
+ }
+ out:
+ ls->deadlk_ckpt_handle = 0;
+ return ret;
+}
+
+static int unlink_checkpoint(struct lockspace *ls)
+{
+ SaNameT name;
+ int len;
+
+ len = snprintf((char *)name.value, SA_MAX_NAME_LENGTH, "dlmdeadlk.%s.%d",
+ ls->name, our_nodeid);
+ name.length = len;
+
+ return _unlink_checkpoint(ls, &name);
+}
+
+static void read_checkpoint(struct lockspace *ls, int nodeid)
+{
+ SaCkptCheckpointHandleT h;
+ SaCkptSectionIterationHandleT itr;
+ SaCkptSectionDescriptorT desc;
+ SaCkptIOVectorElementT iov;
+ SaNameT name;
+ SaAisErrorT rv;
+ char buf[DLM_RESNAME_MAXLEN];
+ int len;
+ int retries;
+
+ if (nodeid == our_nodeid)
+ return;
+
+ log_group(ls, "read_checkpoint %d", nodeid);
+
+ len = snprintf((char *)name.value, SA_MAX_NAME_LENGTH, "dlmdeadlk.%s.%d",
+ ls->name, nodeid);
+ name.length = len;
+
+ retries = 0;
+ open_retry:
+ rv = saCkptCheckpointOpen(global_ckpt_h, &name, NULL,
+ SA_CKPT_CHECKPOINT_READ, 0, &h);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "read_checkpoint: %d ckpt open retry", nodeid);
+ sleep(1);
+ if (retries++ < 10)
+ goto open_retry;
+ }
+ if (rv != SA_AIS_OK) {
+ log_error("read_checkpoint: %d ckpt open error %d", nodeid, rv);
+ return;
+ }
+
+ retries = 0;
+ init_retry:
+ rv = saCkptSectionIterationInitialize(h, SA_CKPT_SECTIONS_ANY, 0, &itr);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "read_checkpoint: ckpt iterinit retry");
+ sleep(1);
+ if (retries++ < 10)
+ goto init_retry;
+ }
+ if (rv != SA_AIS_OK) {
+ log_error("read_checkpoint: %d ckpt iterinit error %d", nodeid, rv);
+ goto out;
+ }
+
+ while (1) {
+ retries = 0;
+ next_retry:
+ rv = saCkptSectionIterationNext(itr, &desc);
+ if (rv == SA_AIS_ERR_NO_SECTIONS)
+ break;
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "read_checkpoint: ckpt iternext retry");
+ sleep(1);
+ if (retries++ < 10)
+ goto next_retry;
+ }
+ if (rv != SA_AIS_OK) {
+ log_error("read_checkpoint: %d ckpt iternext error %d",
+ nodeid, rv);
+ goto out_it;
+ }
+
+ if (!desc.sectionSize)
+ continue;
+
+ iov.sectionId = desc.sectionId;
+ iov.dataBuffer = §ion_buf;
+ iov.dataSize = desc.sectionSize;
+ iov.dataOffset = 0;
+
+ memset(&buf, 0, sizeof(buf));
+ snprintf(buf, sizeof(buf), "%s", desc.sectionId.id);
+
+ log_group(ls, "read_checkpoint: section size %llu id %u \"%s\"",
+ (unsigned long long)iov.dataSize,
+ iov.sectionId.idLen, buf);
+
+ retries = 0;
+ read_retry:
+ rv = saCkptCheckpointRead(h, &iov, 1, NULL);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "read_checkpoint: ckpt read retry");
+ sleep(1);
+ if (retries++ < 10)
+ goto read_retry;
+ }
+ if (rv != SA_AIS_OK) {
+ log_error("read_checkpoint: %d ckpt read error %d",
+ nodeid, rv);
+ goto out_it;
+ }
+
+ section_len = iov.readSize;
+
+ if (!section_len)
+ continue;
+
+ if (section_len % sizeof(struct pack_lock)) {
+ log_error("read_checkpoint: %d bad section len %d",
+ nodeid, section_len);
+ continue;
+ }
+
+ read_checkpoint_locks(ls, nodeid, (char *)desc.sectionId.id,
+ desc.sectionId.idLen);
+ }
+
+ out_it:
+ saCkptSectionIterationFinalize(itr);
+ retries = 0;
+ out:
+ rv = saCkptCheckpointClose(h);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "read_checkpoint: unlink ckpt close retry");
+ sleep(1);
+ if (retries++ < 10)
+ goto out;
+ }
+ if (rv != SA_AIS_OK)
+ log_error("read_checkpoint: %d close error %d", nodeid, rv);
+}
+
+static void write_checkpoint(struct lockspace *ls)
+{
+ SaCkptCheckpointCreationAttributesT attr;
+ SaCkptCheckpointHandleT h;
+ SaCkptSectionIdT section_id;
+ SaCkptSectionCreationAttributesT section_attr;
+ SaCkptCheckpointOpenFlagsT flags;
+ SaNameT name;
+ SaAisErrorT rv;
+ char buf[DLM_RESNAME_MAXLEN];
+ struct dlm_rsb *r;
+ struct dlm_lkb *lkb;
+ int r_count, lock_count, total_size, section_size, max_section_size;
+ int len;
+
+ len = snprintf((char *)name.value, SA_MAX_NAME_LENGTH, "dlmdeadlk.%s.%d",
+ ls->name, our_nodeid);
+ name.length = len;
+
+ /* unlink an old checkpoint before we create a new one */
+ if (ls->deadlk_ckpt_handle) {
+ log_error("write_checkpoint: old ckpt");
+ if (_unlink_checkpoint(ls, &name))
+ return;
+ }
+
+ /* loop through all locks to figure out sizes to set in
+ the attr fields */
+
+ r_count = 0;
+ lock_count = 0;
+ total_size = 0;
+ max_section_size = 0;
+
+ list_for_each_entry(r, &ls->resources, list) {
+ r_count++;
+ section_size = 0;
+ list_for_each_entry(lkb, &r->locks, list) {
+ section_size += sizeof(struct pack_lock);
+ lock_count++;
+ }
+ total_size += section_size;
+ if (section_size > max_section_size)
+ max_section_size = section_size;
+ }
+
+ log_group(ls, "write_checkpoint: r_count %d, lock_count %d",
+ r_count, lock_count);
+
+ log_group(ls, "write_checkpoint: total %d bytes, max_section %d bytes",
+ total_size, max_section_size);
+
+ attr.creationFlags = SA_CKPT_WR_ALL_REPLICAS;
+ attr.checkpointSize = total_size;
+ attr.retentionDuration = SA_TIME_MAX;
+ attr.maxSections = r_count + 1; /* don't know why we need +1 */
+ attr.maxSectionSize = max_section_size;
+ attr.maxSectionIdSize = DLM_RESNAME_MAXLEN;
+
+ flags = SA_CKPT_CHECKPOINT_READ |
+ SA_CKPT_CHECKPOINT_WRITE |
+ SA_CKPT_CHECKPOINT_CREATE;
+
+ open_retry:
+ rv = saCkptCheckpointOpen(global_ckpt_h, &name, &attr, flags, 0, &h);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "write_checkpoint: ckpt open retry");
+ sleep(1);
+ goto open_retry;
+ }
+ if (rv == SA_AIS_ERR_EXIST) {
+ log_group(ls, "write_checkpoint: ckpt already exists");
+ return;
+ }
+ if (rv != SA_AIS_OK) {
+ log_group(ls, "write_checkpoint: ckpt open error %d", rv);
+ return;
+ }
+
+ log_group(ls, "write_checkpoint: open ckpt handle %llx",
+ (unsigned long long)h);
+ ls->deadlk_ckpt_handle = (uint64_t) h;
+
+ list_for_each_entry(r, &ls->resources, list) {
+ memset(buf, 0, sizeof(buf));
+ len = snprintf(buf, sizeof(buf), "%s", r->name);
+
+ section_id.id = (void *)buf;
+ section_id.idLen = len + 1;
+ section_attr.sectionId = §ion_id;
+ section_attr.expirationTime = SA_TIME_END;
+
+ pack_section_buf(ls, r);
+
+ log_group(ls, "write_checkpoint: section size %u id %u \"%s\"",
+ section_len, section_id.idLen, buf);
+
+ create_retry:
+ rv = saCkptSectionCreate(h, §ion_attr, §ion_buf,
+ section_len);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ log_group(ls, "write_checkpoint: ckpt create retry");
+ sleep(1);
+ goto create_retry;
+ }
+ if (rv == SA_AIS_ERR_EXIST) {
+ /* this shouldn't happen in general */
+ log_error("write_checkpoint: clearing old ckpt");
+ saCkptCheckpointClose(h);
+ _unlink_checkpoint(ls, &name);
+ goto open_retry;
+ }
+ if (rv != SA_AIS_OK) {
+ log_error("write_checkpoint: section create %d", rv);
+ break;
+ }
+ }
+}
+
+static void send_message(struct lockspace *ls, int type,
+ uint32_t to_nodeid, uint32_t msgdata)
+{
+ struct dlm_header *hd;
+ int len;
+ char *buf;
+
+ len = sizeof(struct dlm_header);
+ buf = malloc(len);
+ if (!buf) {
+ log_error("send_message: no memory");
+ disable_deadlock();
+ return;
+ }
+ memset(buf, 0, len);
+
+ hd = (struct dlm_header *)buf;
+ hd->type = type;
+ hd->to_nodeid = to_nodeid;
+ hd->msgdata = msgdata;
+
+ dlm_send_message(ls, buf, len);
+
+ free(buf);
+}
+
+static void send_checkpoint_ready(struct lockspace *ls)
+{
+ log_group(ls, "send_checkpoint_ready");
+ send_message(ls, DLM_MSG_DEADLK_CHECKPOINT_READY, 0, 0);
+}
+
+void send_cycle_start(struct lockspace *ls)
+{
+ log_group(ls, "send_cycle_start");
+ send_message(ls, DLM_MSG_DEADLK_CYCLE_START, 0, 0);
+}
+
+static void send_cycle_end(struct lockspace *ls)
+{
+ log_group(ls, "send_cycle_end");
+ send_message(ls, DLM_MSG_DEADLK_CYCLE_END, 0, 0);
+}
+
+static void send_cancel_lock(struct lockspace *ls, struct trans *tr,
+ struct dlm_lkb *lkb)
+{
+ int to_nodeid;
+ uint32_t lkid;
+
+ if (!lkb->lock.nodeid)
+ lkid = lkb->lock.id;
+ else
+ lkid = lkb->lock.remid;
+ to_nodeid = lkb->home;
+
+ log_group(ls, "send_cancel_lock to nodeid %d rsb %s id %x xid %llx",
+ to_nodeid, lkb->rsb->name, lkid,
+ (unsigned long long)lkb->lock.xid);
+
+ send_message(ls, DLM_MSG_DEADLK_CANCEL_LOCK, to_nodeid, lkid);
+}
+
+static void dump_resources(struct lockspace *ls)
+{
+ struct dlm_rsb *r;
+ struct dlm_lkb *lkb;
+
+ log_group(ls, "Resource dump:");
+
+ list_for_each_entry(r, &ls->resources, list) {
+ log_group(ls, "\"%s\" len %d", r->name, r->len);
+ list_for_each_entry(lkb, &r->locks, list) {
+ log_group(ls, " %s: nodeid %d id %08x remid %08x gr %s rq %s pid %u xid
%llx",
+ status_str(lkb->lock.status),
+ lkb->lock.nodeid,
+ lkb->lock.id,
+ lkb->lock.remid,
+ dlm_mode_str(lkb->lock.grmode),
+ dlm_mode_str(lkb->lock.rqmode),
+ lkb->lock.ownpid,
+ (unsigned long long)lkb->lock.xid);
+ }
+ }
+}
+
+static void find_deadlock(struct lockspace *ls);
+
+static void run_deadlock(struct lockspace *ls)
+{
+ struct node *node;
+ int not_ready = 0;
+ int low = -1;
+
+ if (ls->all_checkpoints_ready)
+ log_group(ls, "WARNING: run_deadlock all_checkpoints_ready");
+
+ list_for_each_entry(node, &ls->deadlk_nodes, list) {
+ if (!node->in_cycle)
+ continue;
+ if (!node->checkpoint_ready)
+ not_ready++;
+
+ log_group(ls, "nodeid %d checkpoint_ready = %d",
+ node->nodeid, node->checkpoint_ready);
+ }
+ if (not_ready)
+ return;
+
+ ls->all_checkpoints_ready = 1;
+
+ list_for_each_entry(node, &ls->deadlk_nodes, list) {
+ if (!node->in_cycle)
+ continue;
+ if (node->nodeid < low || low == -1)
+ low = node->nodeid;
+ }
+ ls->deadlk_low_nodeid = low;
+
+ if (low == our_nodeid)
+ find_deadlock(ls);
+ else
+ log_group(ls, "defer resolution to low nodeid %d", low);
+}
+
+void receive_checkpoint_ready(struct lockspace *ls, struct dlm_header *hd,
+ int len)
+{
+ struct node *node;
+ int nodeid = hd->nodeid;
+
+ log_group(ls, "receive_checkpoint_ready from %d", nodeid);
+
+ read_checkpoint(ls, nodeid);
+
+ list_for_each_entry(node, &ls->deadlk_nodes, list) {
+ if (node->nodeid == nodeid) {
+ node->checkpoint_ready = 1;
+ break;
+ }
+ }
+
+ run_deadlock(ls);
+}
+
+void receive_cycle_start(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ struct node *node;
+ int nodeid = hd->nodeid;
+ int rv;
+
+ log_group(ls, "receive_cycle_start from %d", nodeid);
+
+ if (ls->cycle_running) {
+ log_group(ls, "cycle already running");
+ return;
+ }
+ ls->cycle_running = 1;
+ gettimeofday(&ls->cycle_start_time, NULL);
+
+ list_for_each_entry(node, &ls->deadlk_nodes, list)
+ node->in_cycle = 1;
+
+ rv = read_debugfs_locks(ls);
+ if (rv < 0) {
+ log_error("can't read dlm debugfs file: %s", strerror(errno));
+ return;
+ }
+
+ write_checkpoint(ls);
+ send_checkpoint_ready(ls);
+}
+
+static uint64_t dt_usec(struct timeval *start, struct timeval *stop)
+{
+ uint64_t dt;
+
+ dt = stop->tv_sec - start->tv_sec;
+ dt *= 1000000;
+ dt += stop->tv_usec - start->tv_usec;
+ return dt;
+}
+
+/* TODO: nodes added during a cycle - what will they do with messages
+ they recv from other nodes running the cycle? */
+
+void receive_cycle_end(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ struct node *node;
+ int nodeid = hd->nodeid;
+ uint64_t usec;
+
+ if (!ls->cycle_running) {
+ log_error("receive_cycle_end %s from %d: no cycle running",
+ ls->name, nodeid);
+ return;
+ }
+
+ gettimeofday(&ls->cycle_end_time, NULL);
+ usec = dt_usec(&ls->cycle_start_time, &ls->cycle_end_time);
+ log_group(ls, "receive_cycle_end: from %d cycle time %.2f s",
+ nodeid, usec * 1.e-6);
+
+ ls->cycle_running = 0;
+ ls->all_checkpoints_ready = 0;
+
+ list_for_each_entry(node, &ls->deadlk_nodes, list)
+ node->checkpoint_ready = 0;
+
+ free_resources(ls);
+ free_transactions(ls);
+ unlink_checkpoint(ls);
+}
+
+void receive_cancel_lock(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ dlm_lshandle_t h;
+ int nodeid = hd->nodeid;
+ uint32_t lkid = hd->msgdata;
+ int rv;
+
+ if (nodeid != our_nodeid)
+ return;
+
+ h = dlm_open_lockspace(ls->name);
+ if (!h) {
+ log_error("deadlock cancel %x from %d can't open lockspace %s",
+ lkid, nodeid, ls->name);
+ return;
+ }
+
+ log_group(ls, "receive_cancel_lock %x from %d", lkid, nodeid);
+
+ rv = dlm_ls_deadlock_cancel(h, lkid, 0);
+ if (rv < 0) {
+ log_error("deadlock cancel %x from %x lib cancel errno %d",
+ lkid, nodeid, errno);
+ }
+
+ dlm_close_lockspace(h);
+}
+
+static void node_joined(struct lockspace *ls, int nodeid)
+{
+ struct node *node;
+
+ node = malloc(sizeof(struct node));
+ if (!node) {
+ log_error("node_joined: no memory");
+ disable_deadlock();
+ return;
+ }
+ memset(node, 0, sizeof(struct node));
+ node->nodeid = nodeid;
+ list_add_tail(&node->list, &ls->deadlk_nodes);
+ log_group(ls, "node %d joined deadlock cpg", nodeid);
+}
+
+static void node_left(struct lockspace *ls, int nodeid, int reason)
+{
+ struct node *node, *safe;
+
+ list_for_each_entry_safe(node, safe, &ls->deadlk_nodes, list) {
+ if (node->nodeid != nodeid)
+ continue;
+
+ list_del(&node->list);
+ free(node);
+ log_group(ls, "node %d left deadlock cpg", nodeid);
+ }
+}
+
+static void purge_locks(struct lockspace *ls, int nodeid);
+
+void deadlk_confchg(struct lockspace *ls,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries)
+{
+ int i;
+
+ if (!cfgd_enable_deadlk)
+ return;
+
+ if (!ls->deadlk_confchg_init) {
+ ls->deadlk_confchg_init = 1;
+ for (i = 0; i < member_list_entries; i++)
+ node_joined(ls, member_list[i].nodeid);
+ return;
+ }
+
+ /* nodes added during a cycle won't have node->in_cycle set so they
+ won't be included in any of the cycle processing */
+
+ for (i = 0; i < joined_list_entries; i++)
+ node_joined(ls, joined_list[i].nodeid);
+
+ for (i = 0; i < left_list_entries; i++)
+ node_left(ls, left_list[i].nodeid, left_list[i].reason);
+
+ if (!ls->cycle_running)
+ return;
+
+ if (!left_list_entries)
+ return;
+
+ if (!ls->all_checkpoints_ready) {
+ run_deadlock(ls);
+ return;
+ }
+
+ for (i = 0; i < left_list_entries; i++)
+ purge_locks(ls, left_list[i].nodeid);
+
+ for (i = 0; i < left_list_entries; i++) {
+ if (left_list[i].nodeid != ls->deadlk_low_nodeid)
+ continue;
+ /* this will set a new low node which will call find_deadlock */
+ run_deadlock(ls);
+ break;
+ }
+}
+
+/* would we ever call this after we've created the transaction lists?
+ I don't think so; I think it can only be called between reading
+ checkpoints */
+
+static void purge_locks(struct lockspace *ls, int nodeid)
+{
+ struct dlm_rsb *r;
+ struct dlm_lkb *lkb, *safe;
+
+ list_for_each_entry(r, &ls->resources, list) {
+ list_for_each_entry_safe(lkb, safe, &r->locks, list) {
+ if (lkb->home == nodeid) {
+ list_del(&lkb->list);
+ if (list_empty(&lkb->trans_list))
+ free(lkb);
+ else
+ log_group(ls, "purge %d %x on trans",
+ nodeid, lkb->lock.id);
+ }
+ }
+ }
+}
+
+static void add_lkb_trans(struct trans *tr, struct dlm_lkb *lkb)
+{
+ list_add(&lkb->trans_list, &tr->locks);
+ lkb->trans = tr;
+}
+
+static struct trans *get_trans(struct lockspace *ls, uint64_t xid)
+{
+ struct trans *tr;
+
+ list_for_each_entry(tr, &ls->transactions, list) {
+ if (tr->xid == xid)
+ return tr;
+ }
+
+ tr = malloc(sizeof(struct trans));
+ if (!tr) {
+ log_error("get_trans: no memory");
+ disable_deadlock();
+ return NULL;
+ }
+ memset(tr, 0, sizeof(struct trans));
+ tr->xid = xid;
+ tr->waitfor = NULL;
+ tr->waitfor_alloc = 0;
+ tr->waitfor_count = 0;
+ INIT_LIST_HEAD(&tr->locks);
+ list_add(&tr->list, &ls->transactions);
+ return tr;
+}
+
+/* for each rsb, for each lock, find/create trans, add lkb to the trans list */
+
+static void create_trans_list(struct lockspace *ls)
+{
+ struct dlm_rsb *r;
+ struct dlm_lkb *lkb;
+ struct trans *tr;
+ int r_count = 0, lkb_count = 0;
+
+ list_for_each_entry(r, &ls->resources, list) {
+ r_count++;
+ list_for_each_entry(lkb, &r->locks, list) {
+ lkb_count++;
+ tr = get_trans(ls, lkb->lock.xid);
+ if (!tr)
+ goto out;
+ add_lkb_trans(tr, lkb);
+ }
+ }
+ out:
+ log_group(ls, "create_trans_list: r_count %d lkb_count %d",
+ r_count, lkb_count);
+}
+
+static int locks_compat(struct dlm_lkb *waiting_lkb,
+ struct dlm_lkb *granted_lkb)
+{
+ if (waiting_lkb == granted_lkb) {
+ log_debug("waiting and granted same lock");
+ return 0;
+ }
+
+ if (waiting_lkb->trans->xid == granted_lkb->trans->xid) {
+ log_debug("waiting and granted same trans %llx",
+ (unsigned long long)waiting_lkb->trans->xid);
+ return 0;
+ }
+
+ return dlm_modes_compat(granted_lkb->lock.grmode,
+ waiting_lkb->lock.rqmode);
+}
+
+static int in_waitfor(struct trans *tr, struct trans *add_tr)
+{
+ int i;
+
+ for (i = 0; i < tr->waitfor_alloc; i++) {
+ if (!tr->waitfor[i])
+ continue;
+ if (tr->waitfor[i] == add_tr)
+ return 1;
+ }
+ return 0;
+}
+
+static void add_waitfor(struct lockspace *ls, struct dlm_lkb *waiting_lkb,
+ struct dlm_lkb *granted_lkb)
+{
+ struct trans *tr = waiting_lkb->trans;
+ int i;
+
+ if (locks_compat(waiting_lkb, granted_lkb))
+ return;
+
+ /* this shouldn't happen AFAIK */
+ if (tr == granted_lkb->trans) {
+ log_group(ls, "trans %llx waiting on self",
+ (unsigned long long)tr->xid);
+ return;
+ }
+
+ /* don't add the same trans to the waitfor list multiple times */
+ if (tr->waitfor_count && in_waitfor(tr, granted_lkb->trans)) {
+ log_group(ls, "trans %llx already waiting for trans %llx, "
+ "waiting %x %s, granted %x %s",
+ (unsigned long long)waiting_lkb->trans->xid,
+ (unsigned long long)granted_lkb->trans->xid,
+ waiting_lkb->lock.id, waiting_lkb->rsb->name,
+ granted_lkb->lock.id, granted_lkb->rsb->name);
+ return;
+ }
+
+ if (tr->waitfor_count == tr->waitfor_alloc) {
+ struct trans **old_waitfor = tr->waitfor;
+ tr->waitfor_alloc += TR_NALLOC;
+ tr->waitfor = malloc(tr->waitfor_alloc * sizeof(tr));
+ if (!tr->waitfor) {
+ log_error("add_waitfor no mem %u", tr->waitfor_alloc);
+ return;
+ }
+ memset(tr->waitfor, 0, tr->waitfor_alloc * sizeof(tr));
+
+ /* copy then free old set of pointers */
+ for (i = 0; i < tr->waitfor_count; i++)
+ tr->waitfor[i] = old_waitfor[i];
+ if (old_waitfor)
+ free(old_waitfor);
+ }
+
+ tr->waitfor[tr->waitfor_count++] = granted_lkb->trans;
+ granted_lkb->trans->others_waiting_on_us++;
+ waiting_lkb->waitfor_trans = granted_lkb->trans;
+}
+
+/* for each trans, for each waiting lock, go to rsb of the lock,
+ find granted locks on that rsb, then find the trans the
+ granted lock belongs to, add that trans to our waitfor list */
+
+static void create_waitfor_graph(struct lockspace *ls)
+{
+ struct dlm_lkb *waiting_lkb, *granted_lkb;
+ struct dlm_rsb *r;
+ struct trans *tr;
+ int depend_count = 0;
+
+ list_for_each_entry(tr, &ls->transactions, list) {
+ list_for_each_entry(waiting_lkb, &tr->locks, trans_list) {
+ if (waiting_lkb->lock.status == DLM_LKSTS_GRANTED)
+ continue;
+ /* waiting_lkb status is CONVERT or WAITING */
+
+ r = waiting_lkb->rsb;
+
+ list_for_each_entry(granted_lkb, &r->locks, list) {
+ if (granted_lkb->lock.status==DLM_LKSTS_WAITING)
+ continue;
+ /* granted_lkb status is GRANTED or CONVERT */
+ add_waitfor(ls, waiting_lkb, granted_lkb);
+ depend_count++;
+ }
+ }
+ }
+
+ log_group(ls, "create_waitfor_graph: depend_count %d", depend_count);
+}
+
+/* Assume a transaction that's not waiting on any locks will complete, release
+ all the locks it currently holds, and exit. Other transactions that were
+ blocked waiting on the removed transaction's now-released locks may now be
+ unblocked, complete, release all held locks and exit. Repeat this until
+ no more transactions can be removed. If there are transactions remaining,
+ then they are deadlocked. */
+
+static void remove_waitfor(struct trans *tr, struct trans *remove_tr)
+{
+ int i;
+
+ for (i = 0; i < tr->waitfor_alloc; i++) {
+ if (!tr->waitfor_count)
+ break;
+
+ if (!tr->waitfor[i])
+ continue;
+
+ if (tr->waitfor[i] == remove_tr) {
+ tr->waitfor[i] = NULL;
+ tr->waitfor_count--;
+ remove_tr->others_waiting_on_us--;
+ }
+ }
+}
+
+/* remove_tr is not waiting for anything, assume it completes and goes away
+ and remove it from any other transaction's waitfor list */
+
+static void remove_trans(struct lockspace *ls, struct trans *remove_tr)
+{
+ struct trans *tr;
+
+ list_for_each_entry(tr, &ls->transactions, list) {
+ if (tr == remove_tr)
+ continue;
+ if (!remove_tr->others_waiting_on_us)
+ break;
+ remove_waitfor(tr, remove_tr);
+ }
+
+ if (remove_tr->others_waiting_on_us)
+ log_group(ls, "trans %llx removed others waiting %d",
+ (unsigned long long)remove_tr->xid,
+ remove_tr->others_waiting_on_us);
+}
+
+static int reduce_waitfor_graph(struct lockspace *ls)
+{
+ struct trans *tr, *safe;
+ int blocked = 0;
+ int removed = 0;
+
+ list_for_each_entry_safe(tr, safe, &ls->transactions, list) {
+ if (tr->waitfor_count) {
+ blocked++;
+ continue;
+ }
+ remove_trans(ls, tr);
+ list_del(&tr->list);
+ if (tr->waitfor)
+ free(tr->waitfor);
+ free(tr);
+ removed++;
+ }
+
+ log_group(ls, "reduce_waitfor_graph: %d blocked, %d removed",
+ blocked, removed);
+ return removed;
+}
+
+static void reduce_waitfor_graph_loop(struct lockspace *ls)
+{
+ int removed;
+
+ while (1) {
+ removed = reduce_waitfor_graph(ls);
+ if (!removed)
+ break;
+ }
+}
+
+static struct trans *find_trans_to_cancel(struct lockspace *ls)
+{
+ struct trans *tr;
+
+ list_for_each_entry(tr, &ls->transactions, list) {
+ if (!tr->others_waiting_on_us)
+ continue;
+ return tr;
+ }
+ return NULL;
+}
+
+static void cancel_trans(struct lockspace *ls)
+{
+ struct trans *tr;
+ struct dlm_lkb *lkb;
+ int removed;
+
+ tr = find_trans_to_cancel(ls);
+ if (!tr) {
+ log_group(ls, "cancel_trans: no trans found");
+ return;
+ }
+
+ list_for_each_entry(lkb, &tr->locks, trans_list) {
+ if (lkb->lock.status == DLM_LKSTS_GRANTED)
+ continue;
+ send_cancel_lock(ls, tr, lkb);
+
+ /* When this canceled trans has multiple locks all blocked by
+ locks held by one other trans, that other trans is only
+ added to tr->waitfor once, and only one of these waiting
+ locks will have waitfor_trans set. So, the lkb with
+ non-null waitfor_trans was the first one responsible
+ for adding waitfor_trans to tr->waitfor.
+
+ We could potentially forget about keeping track of lkb->
+ waitfor_trans, forget about calling remove_waitfor()
+ here and just set tr->waitfor_count = 0 after this loop.
+ The loss would be that waitfor_trans->others_waiting_on_us
+ would not get decremented. */
+
+ if (lkb->waitfor_trans)
+ remove_waitfor(tr, lkb->waitfor_trans);
+ }
+
+ /* this shouldn't happen, if it does something's not working right */
+ if (tr->waitfor_count) {
+ log_group(ls, "cancel_trans: %llx non-zero waitfor_count %d",
+ (unsigned long long)tr->xid, tr->waitfor_count);
+ }
+
+ /* this should now remove the canceled trans since it now has a zero
+ waitfor_count */
+ removed = reduce_waitfor_graph(ls);
+
+ if (!removed)
+ log_group(ls, "canceled trans not removed from graph");
+
+ /* now call reduce_waitfor_graph() in another loop and it
+ should completely reduce */
+}
+
+static void dump_trans(struct lockspace *ls, struct trans *tr)
+{
+ struct dlm_lkb *lkb;
+ struct trans *wf;
+ int i;
+
+ log_group(ls, "trans xid %llx waitfor_count %d others_waiting_on_us %d",
+ (unsigned long long)tr->xid, tr->waitfor_count,
+ tr->others_waiting_on_us);
+
+ log_group(ls, "locks:");
+
+ list_for_each_entry(lkb, &tr->locks, trans_list) {
+ log_group(ls, " %s: id %08x gr %s rq %s pid %u:%u \"%s\"",
+ status_str(lkb->lock.status),
+ lkb->lock.id,
+ dlm_mode_str(lkb->lock.grmode),
+ dlm_mode_str(lkb->lock.rqmode),
+ lkb->home,
+ lkb->lock.ownpid,
+ lkb->rsb->name);
+ }
+
+ if (!tr->waitfor_count)
+ return;
+
+ log_group(ls, "waitfor:");
+
+ for (i = 0; i < tr->waitfor_alloc; i++) {
+ if (!tr->waitfor[i])
+ continue;
+ wf = tr->waitfor[i];
+ log_group(ls, " xid %llx", (unsigned long long)wf->xid);
+ }
+}
+
+static void dump_all_trans(struct lockspace *ls)
+{
+ struct trans *tr;
+
+ log_group(ls, "Transaction dump:");
+
+ list_for_each_entry(tr, &ls->transactions, list)
+ dump_trans(ls, tr);
+}
+
+static void find_deadlock(struct lockspace *ls)
+{
+ if (list_empty(&ls->resources)) {
+ log_group(ls, "no deadlock: no resources");
+ goto out;
+ }
+
+ if (!list_empty(&ls->transactions)) {
+ log_group(ls, "transactions list should be empty");
+ goto out;
+ }
+
+ dump_resources(ls);
+ create_trans_list(ls);
+ create_waitfor_graph(ls);
+ dump_all_trans(ls);
+ reduce_waitfor_graph_loop(ls);
+
+ if (list_empty(&ls->transactions)) {
+ log_group(ls, "no deadlock: all transactions reduced");
+ goto out;
+ }
+
+ log_group(ls, "found deadlock");
+ dump_all_trans(ls);
+
+ cancel_trans(ls);
+ reduce_waitfor_graph_loop(ls);
+
+ if (list_empty(&ls->transactions)) {
+ log_group(ls, "resolved deadlock with cancel");
+ goto out;
+ }
+
+ log_error("deadlock resolution failed");
+ dump_all_trans(ls);
+ out:
+ send_cycle_end(ls);
+}
+
diff --git a/dlm_controld/dlm_controld.8 b/dlm_controld/dlm_controld.8
new file mode 100644
index 0000000..7100f0e
--- /dev/null
+++ b/dlm_controld/dlm_controld.8
@@ -0,0 +1,313 @@
+.TH DLM_CONTROLD 8 2009-01-18 cluster cluster
+
+.SH NAME
+dlm_controld \- daemon that configures dlm according to cluster events
+
+.SH SYNOPSIS
+.B dlm_controld
+[OPTIONS]
+
+.SH DESCRIPTION
+The dlm lives in the kernel, and the cluster infrastructure (corosync
+membership and group management) lives in user space. The dlm in the
+kernel needs to adjust/recover for certain cluster events. It's the job
+of dlm_controld to receive these events and reconfigure the kernel dlm as
+needed. dlm_controld controls and configures the dlm through sysfs and
+configfs files that are considered dlm-internal interfaces.
+
+The cman init script usually starts the dlm_controld daemon.
+
+.SH OPTIONS
+Command line options override a corresponding setting in cluster.conf.
+
+.TP
+.B \-D
+Enable debugging to stderr and don't fork.
+.br
+See also
+.B dlm_tool dump
+in
+.BR dlm_tool (8).
+
+.TP
+.B \-L
+Enable debugging to log file.
+.br
+See also
+.B logging
+in
+.BR cluster.conf (5).
+
+.TP
+.B \-K
+Enable kernel dlm debugging messages.
+.br
+See also
+.B log_debug
+below.
+
+.TP
+.BI \-r " num"
+dlm kernel lowcomms protocol, 0 tcp, 1 sctp, 2 detect.
+2 selects tcp if corosync rrp_mode is "none", otherwise sctp.
+.br
+Default 2.
+
+.TP
+.BI \-g " num"
+groupd compatibility mode, 0 off, 1 on.
+.br
+Default 0.
+
+.TP
+.BI \-f " num"
+Enable (1) or disable (0) fencing recovery dependency.
+.br
+Default 1.
+
+.TP
+.BI \-q " num"
+Enable (1) or disable (0) quorum recovery dependency.
+.br
+Default 0.
+
+.TP
+.BI \-d " num"
+Enable (1) or disable (0) deadlock detection code.
+.br
+Default 0.
+
+.TP
+.BI \-p " num"
+Enable (1) or disable (0) plock code for cluster fs.
+.br
+Default 1.
+
+.TP
+.BI \-l " num"
+Limit the rate of plock operations, 0 for no limit.
+.br
+Default 0.
+
+.TP
+.BI \-o " num"
+Enable (1) or disable (0) plock ownership.
+.br
+Default 1.
+
+.TP
+.BI \-t " ms"
+Plock ownership drop resources time (milliseconds).
+.br
+Default 10000.
+
+.TP
+.BI \-c " num"
+Plock ownership drop resources count.
+.br
+Default 10.
+
+.TP
+.BI \-a " ms"
+Plock ownership drop resources age (milliseconds).
+.br
+Default 10000.
+
+.TP
+.B \-P
+Enable plock debugging messages (can produce excessive output).
+
+.TP
+.B \-h
+Print a help message describing available options, then exit.
+
+.TP
+.B \-V
+Print program version information, then exit.
+
+
+.SH FILES
+.BR cluster.conf (5)
+is usually located at /etc/cluster/cluster.conf. It is not read directly.
+Other cluster components load the contents into memory, and the values are
+accessed through the libccs library.
+
+Configuration options for dlm (kernel) and dlm_controld are added to the
+<dlm /> section of cluster.conf, within the top level <cluster> section.
+
+.SS Kernel options
+
+.TP
+.B protocol
+The network
+.B protocol
+can be set to tcp, sctp or detect which selects tcp or sctp based on
+the corosync rrp_mode configuration (redundant ring protocol).
+The rrp_mode "none" results in tcp. Default detect.
+
+<dlm protocol="detect"/>
+
+.TP
+.B timewarn
+After waiting
+.B timewarn
+centiseconds, the dlm will emit a warning via netlink. This only applies
+to lockspaces created with the DLM_LSFL_TIMEWARN flag, and is used for
+deadlock detection. Default 500 (5 seconds).
+
+<dlm timewarn="500"/>
+
+.TP
+.B log_debug
+DLM kernel debug messages can be enabled by setting
+.B log_debug
+to 1. Default 0.
+
+<dlm log_debug="0"/>
+
+.TP
+.B clusternode/weight
+The lock directory
+.B weight
+can be specified one the clusternode lines. Weights would usually be
+used in the lock server configurations shown below instead.
+
+<clusternode name="node01" nodeid="1" weight="1"/>
+
+.SS Daemon options
+
+.TP
+.B enable_fencing
+See command line description.
+
+<dlm enable_fencing="1"/>
+
+.TP
+.B enable_quorum
+See command line description.
+
+<dlm enable_quorum="0"/>
+
+.TP
+.B enable_deadlk
+See command line description.
+
+<dlm enable_deadlk="0"/>
+
+.TP
+.B enable_plock
+See command line description.
+
+<dlm enable_plock="1"/>
+
+.TP
+.B plock_rate_limit
+See command line description.
+
+<dlm plock_rate_limit="0"/>
+
+.TP
+.B plock_ownership
+See command line description.
+
+<dlm plock_ownership="1"/>
+
+.TP
+.B drop_resources_time
+See command line description.
+
+<dlm drop_resources_time="10000"/>
+
+.TP
+.B drop_resources_count
+See command line description.
+
+<dlm drop_resources_count="10"/>
+
+.TP
+.B drop_resources_age
+See command line description.
+
+<dlm drop_resources_age="10000"/>
+
+.TP
+.B plock_debug
+Enable (1) or disable (0) plock debugging messages (can produce excessive
+output). Default 0.
+
+<dlm plock_debug="0"/>
+
+
+.SS Disabling resource directory
+
+Lockspaces usually use a resource directory to keep track of which node is
+the master of each resource. The dlm can operate without the resource
+directory, though, by statically assigning the master of a resource using
+a hash of the resource name. To enable, set the per-lockspace
+.B nodir
+option to 1.
+
+.nf
+<dlm>
+ <lockspace name="foo" nodir="1">
+</dlm>
+.fi
+
+.SS Lock-server configuration
+
+The nodir setting can be combined with node weights to create a
+configuration where select node(s) are the master of all resources/locks.
+These
+.B master
+nodes can be viewed as "lock servers" for the other nodes.
+
+.nf
+<dlm>
+ <lockspace name="foo" nodir="1">
+ <master name="node01"/>
+ </lockspace>
+</dlm>
+
+or,
+
+<dlm>
+ <lockspace name="foo" nodir="1">
+ <master name="node01"/>
+ <master name="node02"/>
+ </lockspace>
+</dlm>
+.fi
+
+Lock management will be partitioned among the available masters. There
+can be any number of masters defined. The designated master nodes will
+master all resources/locks (according to the resource name hash). When no
+masters are members of the lockspace, then the nodes revert to the common
+fully-distributed configuration. Recovery is faster, with little
+disruption, when a non-master node joins/leaves.
+
+There is no special mode in the dlm for this lock server configuration,
+it's just a natural consequence of combining the "nodir" option with node
+weights. When a lockspace has master nodes defined, the master has a
+default weight of 1 and all non-master nodes have weight of 0. An explicit
+non-zero
+.B weight
+can also be assigned to master nodes, e.g.
+
+.nf
+<dlm>
+ <lockspace name="foo" nodir="1">
+ <master name="node01" weight="2"/>
+ <master name="node02" weight="1"/>
+ </lockspace>
+</dlm>
+.fi
+
+In which case node01 will master 2/3 of the total resources and node2 will
+master the other 1/3.
+
+.SH SEE ALSO
+.BR dlm_tool (8),
+.BR fenced (8),
+.BR cman (5),
+.BR cluster.conf (5)
+
diff --git a/dlm_controld/dlm_controld.h b/dlm_controld/dlm_controld.h
new file mode 100644
index 0000000..73e4ecc
--- /dev/null
+++ b/dlm_controld/dlm_controld.h
@@ -0,0 +1,38 @@
+#ifndef __DLM_CONTROLD_DOT_H__
+#define __DLM_CONTROLD_DOT_H__
+
+/* This defines the interface between dlm_controld and libdlmcontrol, and
+ should only be used by libdlmcontrol. */
+
+#define DLMC_SOCK_PATH "dlmc_sock"
+#define DLMC_QUERY_SOCK_PATH "dlmc_query_sock"
+
+#define DLMC_MAGIC 0xD13CD13C
+#define DLMC_VERSION 0x00010001
+
+#define DLMC_CMD_DUMP_DEBUG 1
+#define DLMC_CMD_DUMP_PLOCKS 2
+#define DLMC_CMD_LOCKSPACE_INFO 3
+#define DLMC_CMD_NODE_INFO 4
+#define DLMC_CMD_LOCKSPACES 5
+#define DLMC_CMD_LOCKSPACE_NODES 6
+#define DLMC_CMD_FS_REGISTER 7
+#define DLMC_CMD_FS_UNREGISTER 8
+#define DLMC_CMD_FS_NOTIFIED 9
+#define DLMC_CMD_DEADLOCK_CHECK 10
+#define DLMC_CMD_DUMP_LOG_PLOCK 11
+
+struct dlmc_header {
+ unsigned int magic;
+ unsigned int version;
+ unsigned int command;
+ unsigned int option;
+ unsigned int len;
+ int data; /* embedded command-specific data, for convenience */
+ int unused1;
+ int unsued2;
+ char name[DLM_LOCKSPACE_LEN]; /* no terminating null space */
+};
+
+#endif
+
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
new file mode 100644
index 0000000..0c89f67
--- /dev/null
+++ b/dlm_controld/dlm_daemon.h
@@ -0,0 +1,326 @@
+#ifndef __DLM_DAEMON_DOT_H__
+#define __DLM_DAEMON_DOT_H__
+
+#include <sys/types.h>
+#include <asm/types.h>
+#include <sys/uio.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/utsname.h>
+#include <sys/poll.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <net/if.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <fcntl.h>
+#include <netdb.h>
+#include <limits.h>
+#include <unistd.h>
+#include <time.h>
+#include <syslog.h>
+#include <sched.h>
+#include <signal.h>
+#include <sys/time.h>
+#include <dirent.h>
+
+#include <corosync/cpg.h>
+#include <liblogthread.h>
+
+#include <linux/dlmconstants.h>
+#include "libdlmcontrol.h"
+#include "dlm_controld.h"
+#include "config.h"
+#include "list.h"
+#include "rbtree.h"
+#include "linux_endian.h"
+
+/* TODO: cleanup */
+#define CLUSTERVARLIB "/var/lib/cluster"
+#define CLUSTERVARRUN "/var/run/cluster"
+#define LOGDIR "/var/log/cluster"
+#define VERSION "master"
+#define SYSLOGFACILITY LOG_LOCAL4
+#define SYSLOGLEVEL LOG_INFO
+#define REDHAT_COPYRIGHT "Copyright (C) Red Hat, Inc. 2004-2011 All rights
reserved."
+#define LOCKFILE_NAME CLUSTERVARRUN "/dlm_controld.pid"
+#define DAEMON_NAME "dlm_controld"
+#define DEFAULT_LOG_MODE LOG_MODE_OUTPUT_FILE|LOG_MODE_OUTPUT_SYSLOG
+#define DEFAULT_SYSLOG_FACILITY SYSLOGFACILITY
+#define DEFAULT_SYSLOG_PRIORITY SYSLOGLEVEL
+#define DEFAULT_LOGFILE_PRIORITY LOG_INFO /* ? */
+#define DEFAULT_LOGFILE LOGDIR "/" DAEMON_NAME ".log"
+#define DLM_CONFIG_FILE "/etc/dlm.conf"
+
+
+/* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h.
+ Copied in libdlm.h so apps don't need to include the kernel header.
+ The libcpg limit is larger at CPG_MAX_NAME_LENGTH 128. Our cpg name includes
+ a "dlm:" prefix before the lockspace name. */
+
+/* Maximum members of a ls, should match CPG_MEMBERS_MAX in corosync/cpg.h.
+ There are no max defines in dlm-kernel for lockspace members. */
+
+#define MAX_NODES 128
+
+/* Maximum number of IP addresses per node, when using SCTP and multi-ring in
+ corosync In dlm-kernel this is DLM_MAX_ADDR_COUNT, currently 3. */
+
+#define MAX_NODE_ADDRESSES 4
+
+/* Max string length printed on a line, for debugging/dump output. */
+
+#define MAXLINE 256
+
+/* cfgk_protocol */
+
+#define PROTO_TCP 0
+#define PROTO_SCTP 1
+#define PROTO_DETECT 2
+
+extern int daemon_debug_opt;
+extern int daemon_quit;
+extern int cluster_down;
+extern int poll_fencing;
+extern int poll_quorum;
+extern int poll_fs;
+extern int poll_ignore_plock;
+extern int poll_drop_plock;
+extern int plock_fd;
+extern int plock_ci;
+extern struct list_head lockspaces;
+extern int cluster_quorate;
+extern uint32_t cluster_ringid_seq;
+extern int our_nodeid;
+extern uint32_t control_minor;
+extern uint32_t monitor_minor;
+extern uint32_t plock_minor;
+extern uint32_t old_plock_minor;
+
+#define LOG_DUMP_SIZE DLMC_DUMP_SIZE
+
+#define LOG_PLOCK 0x00010000
+
+void log_level(char *name_in, uint32_t level_in, const char *fmt, ...);
+
+#define log_error(fmt, args...) log_level(NULL, LOG_ERR, fmt, ##args)
+#define log_debug(fmt, args...) log_level(NULL, LOG_DEBUG, fmt, ##args)
+#define log_group(ls, fmt, args...) log_level((ls)->name, LOG_DEBUG, fmt, ##args)
+
+#define log_plock(ls, fmt, args...) log_level((ls)->name, LOG_PLOCK, fmt, ##args)
+#define log_dlock(ls, fmt, args...) log_level((ls)->name, LOG_PLOCK|LOG_DEBUG, fmt,
##args)
+#define log_elock(ls, fmt, args...) log_level((ls)->name, LOG_PLOCK|LOG_ERR, fmt,
##args)
+
+/* dlm_header types */
+enum {
+ DLM_MSG_PROTOCOL = 1,
+ DLM_MSG_START,
+ DLM_MSG_PLOCK,
+ DLM_MSG_PLOCK_OWN,
+ DLM_MSG_PLOCK_DROP,
+ DLM_MSG_PLOCK_SYNC_LOCK,
+ DLM_MSG_PLOCK_SYNC_WAITER,
+ DLM_MSG_PLOCKS_DONE,
+ DLM_MSG_PLOCKS_DATA,
+ DLM_MSG_DEADLK_CYCLE_START,
+ DLM_MSG_DEADLK_CYCLE_END,
+ DLM_MSG_DEADLK_CHECKPOINT_READY,
+ DLM_MSG_DEADLK_CANCEL_LOCK
+};
+
+/* dlm_header flags */
+#define DLM_MFLG_JOINING 1 /* accompanies start, we are joining */
+#define DLM_MFLG_HAVEPLOCK 2 /* accompanies start, we have plock state */
+#define DLM_MFLG_NACK 4 /* accompanies start, prevent wrong match when
+ two outstanding changes are the same */
+#define DLM_MFLG_PLOCK_SIG 8 /* msgdata2 is a plock signature */
+
+struct dlm_header {
+ uint16_t version[3];
+ uint16_t type; /* DLM_MSG_ */
+ uint32_t nodeid; /* sender */
+ uint32_t to_nodeid; /* recipient, 0 for all */
+ uint32_t global_id; /* global unique id for this lockspace */
+ uint32_t flags; /* DLM_MFLG_ */
+ uint32_t msgdata; /* in-header payload depends on MSG type; lkid
+ for deadlock, seq for lockspace membership */
+ uint32_t msgdata2; /* second MSG-specific data */
+ uint64_t pad;
+};
+
+struct lockspace {
+ struct list_head list;
+ char name[DLM_LOCKSPACE_LEN+1];
+ uint32_t global_id;
+
+ /* lockspace membership stuff */
+
+ cpg_handle_t cpg_handle;
+ int cpg_client;
+ int cpg_fd;
+ int joining;
+ int leaving;
+ int kernel_stopped;
+ int fs_registered;
+ uint32_t change_seq;
+ uint32_t started_count;
+ struct change *started_change;
+ struct list_head changes;
+ struct list_head node_history;
+
+ /* plock stuff */
+
+ int plock_data_node;
+ int need_plocks;
+ int save_plocks;
+ int disable_plock;
+ uint32_t recv_plocks_data_count;
+ uint32_t associated_mg_id;
+ struct list_head saved_messages;
+ struct list_head plock_resources;
+ struct rb_root plock_resources_root;
+ time_t last_checkpoint_time;
+ time_t last_plock_time;
+ struct timeval drop_resources_last;
+
+#if 0
+ /* deadlock stuff */
+
+ int deadlk_low_nodeid;
+ struct list_head deadlk_nodes;
+ uint64_t deadlk_ckpt_handle;
+ int deadlk_confchg_init;
+ struct list_head transactions;
+ struct list_head resources;
+ struct timeval cycle_start_time;
+ struct timeval cycle_end_time;
+ struct timeval last_send_cycle_start;
+ int cycle_running;
+ int all_checkpoints_ready;
+#endif
+};
+
+/* action.c */
+void set_associated_id(uint32_t mg_id);
+int set_sysfs_control(char *name, int val);
+int set_sysfs_event_done(char *name, int val);
+int set_sysfs_id(char *name, uint32_t id);
+int set_configfs_members(char *name, int new_count, int *new_members,
+ int renew_count, int *renew_members);
+int add_configfs_node(int nodeid, char *addr, int addrlen, int local);
+void del_configfs_node(int nodeid);
+void clear_configfs(void);
+int setup_configfs(void);
+int check_uncontrolled_lockspaces(void);
+int setup_misc_devices(void);
+int path_exists(const char *path);
+
+/* config.c */
+void setup_config(int update);
+int get_weight(int nodeid, char *lockspace);
+
+/* cpg.c */
+int setup_cpg_daemon(void);
+void close_cpg_daemon(void);
+void process_cpg_daemon(int ci);
+int set_protocol(void);
+void process_lockspace_changes(void);
+void dlm_send_message(struct lockspace *ls, char *buf, int len);
+int dlm_join_lockspace(struct lockspace *ls);
+int dlm_leave_lockspace(struct lockspace *ls);
+const char *msg_name(int type);
+void update_flow_control_status(void);
+void node_history_cluster_add(int nodeid);
+void node_history_cluster_remove(int nodeid);
+int set_node_info(struct lockspace *ls, int nodeid, struct dlmc_node *node);
+int set_lockspace_info(struct lockspace *ls, struct dlmc_lockspace *lockspace);
+int set_lockspaces(int *count, struct dlmc_lockspace **lss_out);
+int set_lockspace_nodes(struct lockspace *ls, int option, int *node_count,
+ struct dlmc_node **nodes_out);
+int set_fs_notified(struct lockspace *ls, int nodeid);
+
+/* deadlock.c */
+void setup_deadlock(void);
+void send_cycle_start(struct lockspace *ls);
+void receive_checkpoint_ready(struct lockspace *ls, struct dlm_header *hd,
+ int len);
+void receive_cycle_start(struct lockspace *ls, struct dlm_header *hd, int len);
+void receive_cycle_end(struct lockspace *ls, struct dlm_header *hd, int len);
+void receive_cancel_lock(struct lockspace *ls, struct dlm_header *hd, int len);
+void deadlk_confchg(struct lockspace *ls,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries);
+
+/* main.c */
+int do_read(int fd, void *buf, size_t count);
+int do_write(int fd, void *buf, size_t count);
+void client_dead(int ci);
+int client_add(int fd, void (*workfn)(int ci), void (*deadfn)(int ci));
+int client_fd(int ci);
+void client_ignore(int ci, int fd);
+void client_back(int ci, int fd);
+struct lockspace *find_ls(char *name);
+struct lockspace *find_ls_id(uint32_t id);
+const char *dlm_mode_str(int mode);
+void cluster_dead(int ci);
+
+/* member_cman.c */
+int setup_cluster(void);
+void close_cluster(void);
+void process_cluster(int ci);
+void update_cluster(void);
+int is_cluster_member(uint32_t nodeid);
+int setup_cluster_cfg(void);
+void close_cluster_cfg(void);
+void process_cluster_cfg(int ci);
+void kick_node_from_cluster(int nodeid);
+int fence_node_time(int nodeid, uint64_t *last_fenced_time);
+int fence_in_progress(int *count);
+
+/* netlink.c */
+int setup_netlink(void);
+void process_netlink(int ci);
+
+/* plock.c */
+int setup_plocks(void);
+void close_plocks(void);
+void process_plocks(int ci);
+void drop_resources_all(void);
+int limit_plocks(void);
+void receive_plock(struct lockspace *ls, struct dlm_header *hd, int len);
+void receive_own(struct lockspace *ls, struct dlm_header *hd, int len);
+void receive_sync(struct lockspace *ls, struct dlm_header *hd, int len);
+void receive_drop(struct lockspace *ls, struct dlm_header *hd, int len);
+void process_saved_plocks(struct lockspace *ls);
+void purge_plocks(struct lockspace *ls, int nodeid, int unmount);
+int copy_plock_state(struct lockspace *ls, char *buf, int *len_out);
+
+void send_all_plocks_data(struct lockspace *ls, uint32_t seq, uint32_t *plocks_data);
+void receive_plocks_data(struct lockspace *ls, struct dlm_header *hd, int len);
+void clear_plocks_data(struct lockspace *ls);
+
+/* logging.c */
+
+void init_logging(void);
+void setup_logging(void);
+void close_logging(void);
+void copy_log_dump(char *buf, int *len);
+void copy_log_dump_plock(char *buf, int *len);
+
+/* crc.c */
+uint32_t cpgname_to_crc(const char *data, int len);
+
+#endif
+
diff --git a/dlm_controld/endian.h b/dlm_controld/endian.h
new file mode 100644
index 0000000..43089d2
--- /dev/null
+++ b/dlm_controld/endian.h
@@ -0,0 +1,68 @@
+#ifndef __LINUX_ENDIAN_DOT_H__
+#define __LINUX_ENDIAN_DOT_H__
+
+
+#include <endian.h>
+#include <byteswap.h>
+
+
+/* I'm not sure which versions of alpha glibc/gcc are broken,
+ so fix all of them. */
+#ifdef __alpha__
+#undef bswap_64
+static __inline__ unsigned long bswap_64(unsigned long x)
+{
+ unsigned int h = x >> 32;
+ unsigned int l = x;
+
+ h = bswap_32(h);
+ l = bswap_32(l);
+
+ return ((unsigned long)l << 32) | h;
+}
+#endif /* __alpha__ */
+
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+
+#define be16_to_cpu(x) (x)
+#define be32_to_cpu(x) (x)
+#define be64_to_cpu(x) (x)
+
+#define cpu_to_be16(x) (x)
+#define cpu_to_be32(x) (x)
+#define cpu_to_be64(x) (x)
+
+#define le16_to_cpu(x) (bswap_16((x)))
+#define le32_to_cpu(x) (bswap_32((x)))
+#define le64_to_cpu(x) (bswap_64((x)))
+
+#define cpu_to_le16(x) (bswap_16((x)))
+#define cpu_to_le32(x) (bswap_32((x)))
+#define cpu_to_le64(x) (bswap_64((x)))
+
+#endif /* __BYTE_ORDER == __BIG_ENDIAN */
+
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+
+#define be16_to_cpu(x) (bswap_16((x)))
+#define be32_to_cpu(x) (bswap_32((x)))
+#define be64_to_cpu(x) (bswap_64((x)))
+
+#define cpu_to_be16(x) (bswap_16((x)))
+#define cpu_to_be32(x) (bswap_32((x)))
+#define cpu_to_be64(x) (bswap_64((x)))
+
+#define le16_to_cpu(x) (x)
+#define le32_to_cpu(x) (x)
+#define le64_to_cpu(x) (x)
+
+#define cpu_to_le16(x) (x)
+#define cpu_to_le32(x) (x)
+#define cpu_to_le64(x) (x)
+
+#endif /* __BYTE_ORDER == __LITTLE_ENDIAN */
+
+
+#endif /* __LINUX_ENDIAN_DOT_H__ */
diff --git a/dlm_controld/list.h b/dlm_controld/list.h
new file mode 100644
index 0000000..8100cbc
--- /dev/null
+++ b/dlm_controld/list.h
@@ -0,0 +1,336 @@
+/* Copied from include/linux/list.h */
+
+#ifndef _LINUX_LIST_H
+#define _LINUX_LIST_H
+
+/**
+ * container_of - cast a member of a structure out to the containing structure
+ *
+ * @ptr: the pointer to the member.
+ * @type: the type of the container struct this is embedded in.
+ * @member: the name of the member within the struct.
+ *
+ */
+#define container_of(ptr, type, member) ({ \
+ const typeof( ((type *)0)->member ) *__mptr = (ptr); \
+ (type *)( (char *)__mptr - offsetof(type,member) );})
+
+
+/*
+ * These are non-NULL pointers that will result in page faults
+ * under normal circumstances, used to verify that nobody uses
+ * non-initialized list entries.
+ */
+#define LIST_POISON1 ((void *) 0x00100100)
+#define LIST_POISON2 ((void *) 0x00200200)
+
+/*
+ * Simple doubly linked list implementation.
+ *
+ * Some of the internal functions ("__xxx") are useful when
+ * manipulating whole lists rather than single entries, as
+ * sometimes we already know the next/prev entries and we can
+ * generate better code by using them directly rather than
+ * using the generic single-entry routines.
+ */
+
+struct list_head {
+ struct list_head *next, *prev;
+};
+
+#define LIST_HEAD_INIT(name) { &(name), &(name) }
+
+#define LIST_HEAD(name) \
+ struct list_head name = LIST_HEAD_INIT(name)
+
+#define INIT_LIST_HEAD(ptr) do { \
+ (ptr)->next = (ptr); (ptr)->prev = (ptr); \
+} while (0)
+
+/*
+ * Insert a new entry between two known consecutive entries.
+ *
+ * This is only for internal list manipulation where we know
+ * the prev/next entries already!
+ */
+static inline void __list_add(struct list_head *new,
+ struct list_head *prev,
+ struct list_head *next)
+{
+ next->prev = new;
+ new->next = next;
+ new->prev = prev;
+ prev->next = new;
+}
+
+/**
+ * list_add - add a new entry
+ * @new: new entry to be added
+ * @head: list head to add it after
+ *
+ * Insert a new entry after the specified head.
+ * This is good for implementing stacks.
+ */
+static inline void list_add(struct list_head *new, struct list_head *head)
+{
+ __list_add(new, head, head->next);
+}
+
+/**
+ * list_add_tail - add a new entry
+ * @new: new entry to be added
+ * @head: list head to add it before
+ *
+ * Insert a new entry before the specified head.
+ * This is useful for implementing queues.
+ */
+static inline void list_add_tail(struct list_head *new, struct list_head *head)
+{
+ __list_add(new, head->prev, head);
+}
+
+/*
+ * Delete a list entry by making the prev/next entries
+ * point to each other.
+ *
+ * This is only for internal list manipulation where we know
+ * the prev/next entries already!
+ */
+static inline void __list_del(struct list_head * prev, struct list_head * next)
+{
+ next->prev = prev;
+ prev->next = next;
+}
+
+/**
+ * list_del - deletes entry from list.
+ * @entry: the element to delete from the list.
+ * Note: list_empty on entry does not return true after this, the entry is
+ * in an undefined state.
+ */
+static inline void list_del(struct list_head *entry)
+{
+ __list_del(entry->prev, entry->next);
+ entry->next = LIST_POISON1;
+ entry->prev = LIST_POISON2;
+}
+
+/**
+ * list_del_init - deletes entry from list and reinitialize it.
+ * @entry: the element to delete from the list.
+ */
+static inline void list_del_init(struct list_head *entry)
+{
+ __list_del(entry->prev, entry->next);
+ INIT_LIST_HEAD(entry);
+}
+
+/**
+ * list_move - delete from one list and add as another's head
+ * @list: the entry to move
+ * @head: the head that will precede our entry
+ */
+static inline void list_move(struct list_head *list, struct list_head *head)
+{
+ __list_del(list->prev, list->next);
+ list_add(list, head);
+}
+
+/**
+ * list_move_tail - delete from one list and add as another's tail
+ * @list: the entry to move
+ * @head: the head that will follow our entry
+ */
+static inline void list_move_tail(struct list_head *list,
+ struct list_head *head)
+{
+ __list_del(list->prev, list->next);
+ list_add_tail(list, head);
+}
+
+/**
+ * list_empty - tests whether a list is empty
+ * @head: the list to test.
+ */
+static inline int list_empty(const struct list_head *head)
+{
+ return head->next == head;
+}
+
+/**
+ * list_empty_careful - tests whether a list is
+ * empty _and_ checks that no other CPU might be
+ * in the process of still modifying either member
+ *
+ * NOTE: using list_empty_careful() without synchronization
+ * can only be safe if the only activity that can happen
+ * to the list entry is list_del_init(). Eg. it cannot be used
+ * if another CPU could re-list_add() it.
+ *
+ * @head: the list to test.
+ */
+static inline int list_empty_careful(const struct list_head *head)
+{
+ struct list_head *next = head->next;
+ return (next == head) && (next == head->prev);
+}
+
+static inline void __list_splice(struct list_head *list,
+ struct list_head *head)
+{
+ struct list_head *first = list->next;
+ struct list_head *last = list->prev;
+ struct list_head *at = head->next;
+
+ first->prev = head;
+ head->next = first;
+
+ last->next = at;
+ at->prev = last;
+}
+
+/**
+ * list_splice - join two lists
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ */
+static inline void list_splice(struct list_head *list, struct list_head *head)
+{
+ if (!list_empty(list))
+ __list_splice(list, head);
+}
+
+/**
+ * list_splice_init - join two lists and reinitialise the emptied list.
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ *
+ * The list at @list is reinitialised
+ */
+static inline void list_splice_init(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list)) {
+ __list_splice(list, head);
+ INIT_LIST_HEAD(list);
+ }
+}
+
+/**
+ * list_entry - get the struct for this entry
+ * @ptr: the &struct list_head pointer.
+ * @type: the type of the struct this is embedded in.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_entry(ptr, type, member) \
+ container_of(ptr, type, member)
+
+/**
+ * list_first_entry - get the first element from a list
+ * @ptr: the list head to take the element from.
+ * @type: the type of the struct this is embedded in.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Note, that list is expected to be not empty.
+ */
+#define list_first_entry(ptr, type, member) \
+ list_entry((ptr)->next, type, member)
+
+/**
+ * list_for_each - iterate over a list
+ * @pos: the &struct list_head to use as a loop counter.
+ * @head: the head for your list.
+ */
+#define list_for_each(pos, head) \
+ for (pos = (head)->next; pos != (head); pos = pos->next)
+
+/**
+ * __list_for_each - iterate over a list
+ * @pos: the &struct list_head to use as a loop counter.
+ * @head: the head for your list.
+ *
+ * This variant differs from list_for_each() in that it's the
+ * simplest possible list iteration code, no prefetching is done.
+ * Use this for code that knows the list to be very short (empty
+ * or 1 entry) most of the time.
+ */
+#define __list_for_each(pos, head) \
+ for (pos = (head)->next; pos != (head); pos = pos->next)
+
+/**
+ * list_for_each_prev - iterate over a list backwards
+ * @pos: the &struct list_head to use as a loop counter.
+ * @head: the head for your list.
+ */
+#define list_for_each_prev(pos, head) \
+ for (pos = (head)->prev; pos != (head); pos = pos->prev)
+
+/**
+ * list_for_each_safe - iterate over a list safe against removal of list entry
+ * @pos: the &struct list_head to use as a loop counter.
+ * @n: another &struct list_head to use as temporary storage
+ * @head: the head for your list.
+ */
+#define list_for_each_safe(pos, n, head) \
+ for (pos = (head)->next, n = pos->next; pos != (head); \
+ pos = n, n = pos->next)
+
+/**
+ * list_for_each_entry - iterate over list of given type
+ * @pos: the type * to use as a loop counter.
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_for_each_entry(pos, head, member) \
+ for (pos = list_entry((head)->next, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = list_entry(pos->member.next, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_reverse - iterate backwards over list of given type.
+ * @pos: the type * to use as a loop counter.
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_for_each_entry_reverse(pos, head, member) \
+ for (pos = list_entry((head)->prev, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = list_entry(pos->member.prev, typeof(*pos), member))
+
+/**
+ * list_prepare_entry - prepare a pos entry for use as a start point in
+ * list_for_each_entry_continue
+ * @pos: the type * to use as a start point
+ * @head: the head of the list
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_prepare_entry(pos, head, member) \
+ ((pos) ? : list_entry(head, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_continue - iterate over list of given type
+ * continuing after existing point
+ * @pos: the type * to use as a loop counter.
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_for_each_entry_continue(pos, head, member) \
+ for (pos = list_entry(pos->member.next, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = list_entry(pos->member.next, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_safe - iterate over list of given type safe against removal of
list entry
+ * @pos: the type * to use as a loop counter.
+ * @n: another type * to use as temporary storage
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_for_each_entry_safe(pos, n, head, member) \
+ for (pos = list_entry((head)->next, typeof(*pos), member), \
+ n = list_entry(pos->member.next, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+
+#endif
diff --git a/dlm_controld/logging.c b/dlm_controld/logging.c
new file mode 100644
index 0000000..e6e82b7
--- /dev/null
+++ b/dlm_controld/logging.c
@@ -0,0 +1,173 @@
+#include "dlm_daemon.h"
+
+static int log_mode;
+static int syslog_facility;
+static int syslog_priority;
+static int logfile_priority;
+static char logfile[PATH_MAX];
+
+void init_logging(void)
+{
+ log_mode = DEFAULT_LOG_MODE;
+ syslog_facility = DEFAULT_SYSLOG_FACILITY;
+ syslog_priority = DEFAULT_SYSLOG_PRIORITY;
+ logfile_priority = DEFAULT_LOGFILE_PRIORITY;
+ strcpy(logfile, DEFAULT_LOGFILE);
+
+ /* logfile_priority is the only one of these options that
+ can be controlled from command line or environment variable */
+
+ if (cfgd_debug_logfile)
+ logfile_priority = LOG_DEBUG;
+
+ log_debug("logging mode %d syslog f %d p %d logfile p %d %s",
+ log_mode, syslog_facility, syslog_priority,
+ logfile_priority, logfile);
+
+ logt_init(DAEMON_NAME, log_mode, syslog_facility, syslog_priority,
+ logfile_priority, logfile);
+}
+
+void setup_logging(void)
+{
+ /* TODO */
+ /*
+ ccs_read_logging(ccs_handle, DAEMON_NAME,
+ &cfgd_debug_logfile, &log_mode,
+ &syslog_facility, &syslog_priority,
+ &logfile_priority, logfile);
+ */
+
+ log_debug("logging mode %d syslog f %d p %d logfile p %d %s",
+ log_mode, syslog_facility, syslog_priority,
+ logfile_priority, logfile);
+
+ logt_conf(DAEMON_NAME, log_mode, syslog_facility, syslog_priority,
+ logfile_priority, logfile);
+}
+
+void close_logging(void)
+{
+ logt_exit();
+}
+
+#define NAME_ID_SIZE 32
+#define LOG_STR_LEN 512
+static char log_str[LOG_STR_LEN];
+
+static char log_dump[LOG_DUMP_SIZE];
+static unsigned int log_point;
+static unsigned int log_wrap;
+
+static char log_dump_plock[LOG_DUMP_SIZE];
+static unsigned int log_point_plock;
+static unsigned int log_wrap_plock;
+
+static void log_copy(char *buf, int *len, char *log_buf,
+ unsigned int *point, unsigned int *wrap)
+{
+ unsigned int p = *point;
+ unsigned int w = *wrap;
+ int tail_len;
+
+ if (!w && !p) {
+ *len = 0;
+ } else if (*wrap) {
+ tail_len = LOG_DUMP_SIZE - p;
+ memcpy(buf, log_buf + p, tail_len);
+ if (p)
+ memcpy(buf+tail_len, log_buf, p);
+ *len = LOG_DUMP_SIZE;
+ } else {
+ memcpy(buf, log_buf, p-1);
+ *len = p-1;
+ }
+}
+
+void copy_log_dump(char *buf, int *len)
+{
+ log_copy(buf, len, log_dump, &log_point, &log_wrap);
+}
+
+void copy_log_dump_plock(char *buf, int *len)
+{
+ log_copy(buf, len, log_dump_plock, &log_point_plock, &log_wrap_plock);
+}
+
+static void log_save_str(int level, int len, char *log_buf,
+ unsigned int *point, unsigned int *wrap)
+{
+ unsigned int p = *point;
+ unsigned int w = *wrap;
+ int i;
+
+ if (len < LOG_DUMP_SIZE - p) {
+ memcpy(log_buf + p, log_str, len);
+ p += len;
+
+ if (p == LOG_DUMP_SIZE) {
+ p = 0;
+ w = 1;
+ }
+ goto out;
+ }
+
+ for (i = 0; i < len; i++) {
+ log_buf[p++] = log_str[i];
+
+ if (p == LOG_DUMP_SIZE) {
+ p = 0;
+ w = 1;
+ }
+ }
+ out:
+ *point = p;
+ *wrap = w;
+}
+
+void log_level(char *name_in, uint32_t level_in, const char *fmt, ...)
+{
+ va_list ap;
+ char name[NAME_ID_SIZE + 1];
+ uint32_t level = level_in & 0x0000FFFF;
+ uint32_t extra = level_in & 0xFFFF0000;
+ int ret, pos = 0;
+ int len = LOG_STR_LEN - 2;
+ int plock = extra & LOG_PLOCK;
+
+ memset(name, 0, sizeof(name));
+
+ if (name_in)
+ snprintf(name, NAME_ID_SIZE, "%s ", name_in);
+
+ ret = snprintf(log_str + pos, len - pos, "%llu %s",
+ (unsigned long long)time(NULL), name);
+
+ pos += ret;
+
+ va_start(ap, fmt);
+ ret = vsnprintf(log_str + pos, len - pos, fmt, ap);
+ va_end(ap);
+
+ if (ret >= len - pos)
+ pos = len - 1;
+ else
+ pos += ret;
+
+ log_str[pos++] = '\n';
+ log_str[pos++] = '\0';
+
+ if (level)
+ log_save_str(level, pos - 1, log_dump, &log_point, &log_wrap);
+ if (plock)
+ log_save_str(level, pos - 1, log_dump_plock, &log_point_plock,
&log_wrap_plock);
+ if (level)
+ logt_print(level, "%s", log_str);
+
+ if (!daemon_debug_opt)
+ return;
+
+ if (level || (plock && cfgd_plock_debug))
+ fprintf(stderr, "%s", log_str);
+}
+
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
new file mode 100644
index 0000000..b0d9b03
--- /dev/null
+++ b/dlm_controld/main.c
@@ -0,0 +1,1274 @@
+#include "dlm_daemon.h"
+#include <pthread.h>
+#include <linux/netlink.h>
+#include <linux/genetlink.h>
+#include <linux/dlm_netlink.h>
+
+#define CLIENT_NALLOC 32
+static int client_maxi;
+static int client_size = 0;
+static struct client *client = NULL;
+static struct pollfd *pollfd = NULL;
+static pthread_t query_thread;
+static pthread_mutex_t query_mutex;
+static struct list_head fs_register_list;
+static int kernel_monitor_fd;
+
+struct client {
+ int fd;
+ void *workfn;
+ void *deadfn;
+ struct lockspace *ls;
+};
+
+int do_read(int fd, void *buf, size_t count)
+{
+ int rv, off = 0;
+
+ while (off < count) {
+ rv = read(fd, (char *)buf + off, count - off);
+ if (rv == 0)
+ return -1;
+ if (rv == -1 && errno == EINTR)
+ continue;
+ if (rv == -1)
+ return -1;
+ off += rv;
+ }
+ return 0;
+}
+
+int do_write(int fd, void *buf, size_t count)
+{
+ int rv, off = 0;
+
+ retry:
+ rv = write(fd, (char *)buf + off, count);
+ if (rv == -1 && errno == EINTR)
+ goto retry;
+ if (rv < 0) {
+ log_error("write errno %d", errno);
+ return rv;
+ }
+
+ if (rv != count) {
+ count -= rv;
+ off += rv;
+ goto retry;
+ }
+ return 0;
+}
+
+static void client_alloc(void)
+{
+ int i;
+
+ if (!client) {
+ client = malloc(CLIENT_NALLOC * sizeof(struct client));
+ pollfd = malloc(CLIENT_NALLOC * sizeof(struct pollfd));
+ } else {
+ client = realloc(client, (client_size + CLIENT_NALLOC) *
+ sizeof(struct client));
+ pollfd = realloc(pollfd, (client_size + CLIENT_NALLOC) *
+ sizeof(struct pollfd));
+ if (!pollfd)
+ log_error("can't alloc for pollfd");
+ }
+ if (!client || !pollfd)
+ log_error("can't alloc for client array");
+
+ for (i = client_size; i < client_size + CLIENT_NALLOC; i++) {
+ client[i].workfn = NULL;
+ client[i].deadfn = NULL;
+ client[i].fd = -1;
+ pollfd[i].fd = -1;
+ pollfd[i].revents = 0;
+ }
+ client_size += CLIENT_NALLOC;
+}
+
+void client_dead(int ci)
+{
+ close(client[ci].fd);
+ client[ci].workfn = NULL;
+ client[ci].fd = -1;
+ pollfd[ci].fd = -1;
+}
+
+int client_add(int fd, void (*workfn)(int ci), void (*deadfn)(int ci))
+{
+ int i;
+
+ if (!client)
+ client_alloc();
+ again:
+ for (i = 0; i < client_size; i++) {
+ if (client[i].fd == -1) {
+ client[i].workfn = workfn;
+ if (deadfn)
+ client[i].deadfn = deadfn;
+ else
+ client[i].deadfn = client_dead;
+ client[i].fd = fd;
+ pollfd[i].fd = fd;
+ pollfd[i].events = POLLIN;
+ if (i > client_maxi)
+ client_maxi = i;
+ return i;
+ }
+ }
+
+ client_alloc();
+ goto again;
+}
+
+int client_fd(int ci)
+{
+ return client[ci].fd;
+}
+
+void client_ignore(int ci, int fd)
+{
+ pollfd[ci].fd = -1;
+ pollfd[ci].events = 0;
+}
+
+void client_back(int ci, int fd)
+{
+ pollfd[ci].fd = fd;
+ pollfd[ci].events = POLLIN;
+}
+
+static void sigterm_handler(int sig)
+{
+ daemon_quit = 1;
+}
+
+static struct lockspace *create_ls(char *name)
+{
+ struct lockspace *ls;
+
+ ls = malloc(sizeof(*ls));
+ if (!ls)
+ goto out;
+ memset(ls, 0, sizeof(struct lockspace));
+ strncpy(ls->name, name, DLM_LOCKSPACE_LEN);
+
+ INIT_LIST_HEAD(&ls->changes);
+ INIT_LIST_HEAD(&ls->node_history);
+ INIT_LIST_HEAD(&ls->saved_messages);
+ INIT_LIST_HEAD(&ls->plock_resources);
+ ls->plock_resources_root = RB_ROOT;
+#if 0
+ INIT_LIST_HEAD(&ls->deadlk_nodes);
+ INIT_LIST_HEAD(&ls->transactions);
+ INIT_LIST_HEAD(&ls->resources);
+#endif
+ out:
+ return ls;
+}
+
+struct lockspace *find_ls(char *name)
+{
+ struct lockspace *ls;
+
+ list_for_each_entry(ls, &lockspaces, list) {
+ if ((strlen(ls->name) == strlen(name)) &&
+ !strncmp(ls->name, name, strlen(name)))
+ return ls;
+ }
+ return NULL;
+}
+
+struct lockspace *find_ls_id(uint32_t id)
+{
+ struct lockspace *ls;
+
+ list_for_each_entry(ls, &lockspaces, list) {
+ if (ls->global_id == id)
+ return ls;
+ }
+ return NULL;
+}
+
+struct fs_reg {
+ struct list_head list;
+ char name[DLM_LOCKSPACE_LEN+1];
+};
+
+static int fs_register_check(char *name)
+{
+ struct fs_reg *fs;
+ list_for_each_entry(fs, &fs_register_list, list) {
+ if (!strcmp(name, fs->name))
+ return 1;
+ }
+ return 0;
+}
+
+static int fs_register_add(char *name)
+{
+ struct fs_reg *fs;
+
+ if (fs_register_check(name))
+ return -EALREADY;
+
+ fs = malloc(sizeof(struct fs_reg));
+ if (!fs)
+ return -ENOMEM;
+ strncpy(fs->name, name, DLM_LOCKSPACE_LEN);
+ list_add(&fs->list, &fs_register_list);
+ return 0;
+}
+
+static void fs_register_del(char *name)
+{
+ struct fs_reg *fs;
+ list_for_each_entry(fs, &fs_register_list, list) {
+ if (!strcmp(name, fs->name)) {
+ list_del(&fs->list);
+ free(fs);
+ return;
+ }
+ }
+}
+
+#define MAXARGS 8
+
+static char *get_args(char *buf, int *argc, char **argv, char sep, int want)
+{
+ char *p = buf, *rp = NULL;
+ int i;
+
+ argv[0] = p;
+
+ for (i = 1; i < MAXARGS; i++) {
+ p = strchr(buf, sep);
+ if (!p)
+ break;
+ *p = '\0';
+
+ if (want == i) {
+ rp = p + 1;
+ break;
+ }
+
+ argv[i] = p + 1;
+ buf = p + 1;
+ }
+ *argc = i;
+
+ /* we ended by hitting \0, return the point following that */
+ if (!rp)
+ rp = strchr(buf, '\0') + 1;
+
+ return rp;
+}
+
+const char *dlm_mode_str(int mode)
+{
+ switch (mode) {
+ case DLM_LOCK_IV:
+ return "IV";
+ case DLM_LOCK_NL:
+ return "NL";
+ case DLM_LOCK_CR:
+ return "CR";
+ case DLM_LOCK_CW:
+ return "CW";
+ case DLM_LOCK_PR:
+ return "PR";
+ case DLM_LOCK_PW:
+ return "PW";
+ case DLM_LOCK_EX:
+ return "EX";
+ }
+ return "??";
+}
+
+/* recv "online" (join) and "offline" (leave) messages from dlm via
uevents */
+
+static void process_uevent(int ci)
+{
+ struct lockspace *ls;
+ char buf[MAXLINE];
+ char *argv[MAXARGS], *act, *sys;
+ int rv, argc = 0;
+
+ memset(buf, 0, sizeof(buf));
+ memset(argv, 0, sizeof(char *) * MAXARGS);
+
+ retry_recv:
+ rv = recv(client[ci].fd, &buf, sizeof(buf), 0);
+ if (rv < 0) {
+ if (errno == EINTR)
+ goto retry_recv;
+ if (errno != EAGAIN)
+ log_error("uevent recv error %d errno %d", rv, errno);
+ return;
+ }
+
+ if (!strstr(buf, "dlm"))
+ return;
+
+ log_debug("uevent: %s", buf);
+
+ get_args(buf, &argc, argv, '/', 4);
+ if (argc != 4)
+ log_error("uevent message has %d args", argc);
+ act = argv[0];
+ sys = argv[2];
+
+ if ((strlen(sys) != strlen("dlm")) || strcmp(sys, "dlm"))
+ return;
+
+ log_debug("kernel: %s %s", act, argv[3]);
+
+ rv = 0;
+
+ if (!strcmp(act, "online@")) {
+ ls = find_ls(argv[3]);
+ if (ls) {
+ rv = -EEXIST;
+ goto out;
+ }
+
+ ls = create_ls(argv[3]);
+ if (!ls) {
+ rv = -ENOMEM;
+ goto out;
+ }
+
+ if (fs_register_check(ls->name))
+ ls->fs_registered = 1;
+
+ rv = dlm_join_lockspace(ls);
+ if (rv) {
+ /* ls already freed */
+ goto out;
+ }
+
+ } else if (!strcmp(act, "offline@")) {
+ ls = find_ls(argv[3]);
+ if (!ls) {
+ rv = -ENOENT;
+ goto out;
+ }
+
+ dlm_leave_lockspace(ls);
+ }
+ out:
+ if (rv < 0)
+ log_error("process_uevent %s error %d errno %d",
+ act, rv, errno);
+}
+
+static int setup_uevent(void)
+{
+ struct sockaddr_nl snl;
+ int s, rv;
+
+ s = socket(AF_NETLINK, SOCK_DGRAM, NETLINK_KOBJECT_UEVENT);
+ if (s < 0) {
+ log_error("uevent netlink socket");
+ return s;
+ }
+
+ memset(&snl, 0, sizeof(snl));
+ snl.nl_family = AF_NETLINK;
+ snl.nl_pid = getpid();
+ snl.nl_groups = 1;
+
+ rv = bind(s, (struct sockaddr *) &snl, sizeof(snl));
+ if (rv < 0) {
+ log_error("uevent bind error %d errno %d", rv, errno);
+ close(s);
+ return rv;
+ }
+
+ return s;
+}
+
+static void init_header(struct dlmc_header *h, int cmd, char *name, int result,
+ int extra_len)
+{
+ memset(h, 0, sizeof(struct dlmc_header));
+
+ h->magic = DLMC_MAGIC;
+ h->version = DLMC_VERSION;
+ h->len = sizeof(struct dlmc_header) + extra_len;
+ h->command = cmd;
+ h->data = result;
+
+ if (name)
+ strncpy(h->name, name, DLM_LOCKSPACE_LEN);
+}
+
+static char copy_buf[LOG_DUMP_SIZE];
+
+static void query_dump_debug(int fd)
+{
+ struct dlmc_header h;
+ int len = 0;
+
+ copy_log_dump(copy_buf, &len);
+
+ init_header(&h, DLMC_CMD_DUMP_DEBUG, NULL, 0, len);
+ send(fd, &h, sizeof(h), MSG_NOSIGNAL);
+
+ if (len)
+ send(fd, copy_buf, len, MSG_NOSIGNAL);
+}
+
+static void query_dump_log_plock(int fd)
+{
+ struct dlmc_header h;
+ int len = 0;
+
+ copy_log_dump_plock(copy_buf, &len);
+
+ init_header(&h, DLMC_CMD_DUMP_DEBUG, NULL, 0, len);
+ send(fd, &h, sizeof(h), MSG_NOSIGNAL);
+
+ if (len)
+ send(fd, copy_buf, len, MSG_NOSIGNAL);
+}
+
+static void query_dump_plocks(int fd, char *name)
+{
+ struct lockspace *ls;
+ struct dlmc_header h;
+ int len = 0;
+ int rv;
+
+ ls = find_ls(name);
+ if (!ls) {
+ rv = -ENOENT;
+ goto out;
+ }
+
+ rv = copy_plock_state(ls, copy_buf, &len);
+ out:
+ init_header(&h, DLMC_CMD_DUMP_PLOCKS, name, rv, len);
+ send(fd, &h, sizeof(h), MSG_NOSIGNAL);
+
+ if (len)
+ send(fd, copy_buf, len, MSG_NOSIGNAL);
+}
+
+/* combines a header and the data and sends it back to the client in
+ a single do_write() call */
+
+static void do_reply(int fd, int cmd, char *name, int result, int option,
+ char *buf, int buflen)
+{
+ struct dlmc_header *h;
+ char *reply;
+ int reply_len;
+
+ reply_len = sizeof(struct dlmc_header) + buflen;
+ reply = malloc(reply_len);
+ if (!reply)
+ return;
+ memset(reply, 0, reply_len);
+ h = (struct dlmc_header *)reply;
+
+ init_header(h, cmd, name, result, buflen);
+ h->option = option;
+
+ if (buf && buflen)
+ memcpy(reply + sizeof(struct dlmc_header), buf, buflen);
+
+ do_write(fd, reply, reply_len);
+
+ free(reply);
+}
+
+static void query_lockspace_info(int fd, char *name)
+{
+ struct lockspace *ls;
+ struct dlmc_lockspace lockspace;
+ int rv;
+
+ ls = find_ls(name);
+ if (!ls) {
+ rv = -ENOENT;
+ goto out;
+ }
+
+ memset(&lockspace, 0, sizeof(lockspace));
+
+ rv = set_lockspace_info(ls, &lockspace);
+ out:
+ do_reply(fd, DLMC_CMD_LOCKSPACE_INFO, name, rv, 0,
+ (char *)&lockspace, sizeof(lockspace));
+}
+
+static void query_node_info(int fd, char *name, int nodeid)
+{
+ struct lockspace *ls;
+ struct dlmc_node node;
+ int rv;
+
+ ls = find_ls(name);
+ if (!ls) {
+ rv = -ENOENT;
+ goto out;
+ }
+
+ memset(&node, 0, sizeof(node));
+
+ rv = set_node_info(ls, nodeid, &node);
+ out:
+ do_reply(fd, DLMC_CMD_NODE_INFO, name, rv, 0,
+ (char *)&node, sizeof(node));
+}
+
+static void query_lockspaces(int fd, int max)
+{
+ int ls_count = 0;
+ struct dlmc_lockspace *lss = NULL;
+ int rv, result;
+
+ rv = set_lockspaces(&ls_count, &lss);
+ if (rv < 0) {
+ result = rv;
+ ls_count = 0;
+ goto out;
+ }
+
+ if (ls_count > max) {
+ result = -E2BIG;
+ ls_count = max;
+ } else {
+ result = ls_count;
+ }
+ out:
+ do_reply(fd, DLMC_CMD_LOCKSPACES, NULL, result, 0,
+ (char *)lss, ls_count * sizeof(struct dlmc_lockspace));
+
+ if (lss)
+ free(lss);
+}
+
+static void query_lockspace_nodes(int fd, char *name, int option, int max)
+{
+ struct lockspace *ls;
+ int node_count = 0;
+ struct dlmc_node *nodes = NULL;
+ int rv, result;
+
+ ls = find_ls(name);
+ if (!ls) {
+ result = -ENOENT;
+ node_count = 0;
+ goto out;
+ }
+
+ rv = set_lockspace_nodes(ls, option, &node_count, &nodes);
+ if (rv < 0) {
+ result = rv;
+ node_count = 0;
+ goto out;
+ }
+
+ /* node_count is the number of structs copied/returned; the caller's
+ max may be less than that, in which case we copy as many as they
+ asked for and return -E2BIG */
+
+ if (node_count > max) {
+ result = -E2BIG;
+ node_count = max;
+ } else {
+ result = node_count;
+ }
+ out:
+ do_reply(fd, DLMC_CMD_LOCKSPACE_NODES, name, result, 0,
+ (char *)nodes, node_count * sizeof(struct dlmc_node));
+
+ if (nodes)
+ free(nodes);
+}
+
+static void process_connection(int ci)
+{
+ struct dlmc_header h;
+ char *extra = NULL;
+ int rv, extra_len;
+ struct lockspace *ls;
+
+ rv = do_read(client[ci].fd, &h, sizeof(h));
+ if (rv < 0) {
+ log_debug("connection %d read error %d", ci, rv);
+ goto out;
+ }
+
+ if (h.magic != DLMC_MAGIC) {
+ log_debug("connection %d magic error %x", ci, h.magic);
+ goto out;
+ }
+
+ if ((h.version & 0xFFFF0000) != (DLMC_VERSION & 0xFFFF0000)) {
+ log_debug("connection %d version error %x", ci, h.version);
+ goto out;
+ }
+
+ if (h.len > sizeof(h)) {
+ extra_len = h.len - sizeof(h);
+ extra = malloc(extra_len);
+ if (!extra) {
+ log_error("process_connection no mem %d", extra_len);
+ goto out;
+ }
+ memset(extra, 0, extra_len);
+
+ rv = do_read(client[ci].fd, extra, extra_len);
+ if (rv < 0) {
+ log_debug("connection %d extra read error %d", ci, rv);
+ goto out;
+ }
+ }
+
+ switch (h.command) {
+ case DLMC_CMD_FS_REGISTER:
+ rv = fs_register_add(h.name);
+ ls = find_ls(h.name);
+ if (ls)
+ ls->fs_registered = 1;
+ do_reply(client[ci].fd, DLMC_CMD_FS_REGISTER, h.name, rv, 0,
+ NULL, 0);
+ break;
+
+ case DLMC_CMD_FS_UNREGISTER:
+ fs_register_del(h.name);
+ ls = find_ls(h.name);
+ if (ls)
+ ls->fs_registered = 0;
+ break;
+
+ case DLMC_CMD_FS_NOTIFIED:
+ ls = find_ls(h.name);
+ if (ls)
+ rv = set_fs_notified(ls, h.data);
+ else
+ rv = -ENOENT;
+ /* pass back the nodeid provided by caller in option field */
+ do_reply(client[ci].fd, DLMC_CMD_FS_NOTIFIED, h.name, rv,
+ h.data, NULL, 0);
+ break;
+
+#if 0
+ case DLMC_CMD_DEADLOCK_CHECK:
+ ls = find_ls(h.name);
+ if (ls)
+ send_cycle_start(ls);
+ client_dead(ci);
+ break;
+#endif
+ default:
+ log_error("process_connection %d unknown command %d",
+ ci, h.command);
+ }
+ out:
+ if (extra)
+ free(extra);
+}
+
+static void process_listener(int ci)
+{
+ int fd, i;
+
+ fd = accept(client[ci].fd, NULL, NULL);
+ if (fd < 0) {
+ log_error("process_listener: accept error %d %d", fd, errno);
+ return;
+ }
+
+ i = client_add(fd, process_connection, NULL);
+
+ log_debug("client connection %d fd %d", i, fd);
+}
+
+static int setup_listener(const char *sock_path)
+{
+ struct sockaddr_un addr;
+ socklen_t addrlen;
+ int rv, s;
+
+ /* we listen for new client connections on socket s */
+
+ s = socket(AF_LOCAL, SOCK_STREAM, 0);
+ if (s < 0) {
+ log_error("socket error %d %d", s, errno);
+ return s;
+ }
+
+ memset(&addr, 0, sizeof(addr));
+ addr.sun_family = AF_LOCAL;
+ strcpy(&addr.sun_path[1], sock_path);
+ addrlen = sizeof(sa_family_t) + strlen(addr.sun_path+1) + 1;
+
+ rv = bind(s, (struct sockaddr *) &addr, addrlen);
+ if (rv < 0) {
+ log_error("bind error %d %d", rv, errno);
+ close(s);
+ return rv;
+ }
+
+ rv = listen(s, 5);
+ if (rv < 0) {
+ log_error("listen error %d %d", rv, errno);
+ close(s);
+ return rv;
+ }
+ return s;
+}
+
+static void query_lock(void)
+{
+ pthread_mutex_lock(&query_mutex);
+}
+
+static void query_unlock(void)
+{
+ pthread_mutex_unlock(&query_mutex);
+}
+
+/* This is a thread, so we have to be careful, don't call log_ functions.
+ We need a thread to process queries because the main thread may block
+ for long periods when writing to sysfs to stop dlm-kernel (any maybe
+ other places). */
+
+static void *process_queries(void *arg)
+{
+ struct dlmc_header h;
+ int s, f, rv;
+
+ rv = setup_listener(DLMC_QUERY_SOCK_PATH);
+ if (rv < 0)
+ return NULL;
+
+ s = rv;
+
+ for (;;) {
+ f = accept(s, NULL, NULL);
+ if (f < 0)
+ return NULL;
+
+ rv = do_read(f, &h, sizeof(h));
+ if (rv < 0) {
+ goto out;
+ }
+
+ if (h.magic != DLMC_MAGIC) {
+ goto out;
+ }
+
+ if ((h.version & 0xFFFF0000) != (DLMC_VERSION & 0xFFFF0000)) {
+ goto out;
+ }
+
+ query_lock();
+
+ switch (h.command) {
+ case DLMC_CMD_DUMP_DEBUG:
+ query_dump_debug(f);
+ break;
+ case DLMC_CMD_DUMP_LOG_PLOCK:
+ query_dump_log_plock(f);
+ break;
+ case DLMC_CMD_DUMP_PLOCKS:
+ query_dump_plocks(f, h.name);
+ break;
+ case DLMC_CMD_LOCKSPACE_INFO:
+ query_lockspace_info(f, h.name);
+ break;
+ case DLMC_CMD_NODE_INFO:
+ query_node_info(f, h.name, h.data);
+ break;
+ case DLMC_CMD_LOCKSPACES:
+ query_lockspaces(f, h.data);
+ break;
+ case DLMC_CMD_LOCKSPACE_NODES:
+ query_lockspace_nodes(f, h.name, h.option, h.data);
+ break;
+ default:
+ break;
+ }
+ query_unlock();
+
+ out:
+ close(f);
+ }
+}
+
+static int setup_queries(void)
+{
+ int rv;
+
+ pthread_mutex_init(&query_mutex, NULL);
+
+ rv = pthread_create(&query_thread, NULL, process_queries, NULL);
+ if (rv < 0) {
+ log_error("can't create query thread");
+ return rv;
+ }
+ return 0;
+}
+
+/* The dlm in kernels before 2.6.28 do not have the monitor device. We
+ keep this fd open as long as we're running. If we exit/terminate while
+ lockspaces exist in the kernel, the kernel will detect a close on this
+ fd and stop the lockspaces. */
+
+static void setup_monitor(void)
+{
+ if (!monitor_minor)
+ return;
+
+ kernel_monitor_fd = open("/dev/misc/dlm-monitor", O_RDONLY);
+ log_debug("/dev/misc/dlm-monitor fd %d", kernel_monitor_fd);
+}
+
+void cluster_dead(int ci)
+{
+ if (!cluster_down)
+ log_error("cluster is down, exiting");
+ daemon_quit = 1;
+ cluster_down = 1;
+}
+
+static void loop(void)
+{
+ struct lockspace *ls;
+ int poll_timeout = -1;
+ int rv, i;
+ void (*workfn) (int ci);
+ void (*deadfn) (int ci);
+
+ rv = setup_queries();
+ if (rv < 0)
+ goto out;
+
+ rv = setup_listener(DLMC_SOCK_PATH);
+ if (rv < 0)
+ goto out;
+ client_add(rv, process_listener, NULL);
+
+ rv = setup_cluster_cfg();
+ if (rv < 0)
+ goto out;
+ if (rv > 0)
+ client_add(rv, process_cluster_cfg, cluster_dead);
+
+ rv = setup_cluster();
+ if (rv < 0)
+ goto out;
+ client_add(rv, process_cluster, cluster_dead);
+
+ setup_config(0);
+
+ setup_logging();
+
+ rv = check_uncontrolled_lockspaces();
+ if (rv < 0)
+ goto out;
+
+ rv = setup_misc_devices();
+ if (rv < 0)
+ goto out;
+
+ setup_monitor();
+
+ rv = setup_configfs(); /* calls update_cluster() */
+ if (rv < 0)
+ goto out;
+
+ rv = setup_uevent();
+ if (rv < 0)
+ goto out;
+ client_add(rv, process_uevent, NULL);
+
+ rv = setup_cpg_daemon();
+ if (rv < 0)
+ goto out;
+ client_add(rv, process_cpg_daemon, cluster_dead);
+
+ rv = set_protocol();
+ if (rv < 0)
+ goto out;
+
+#if 0
+ if (cfgd_enable_deadlk) {
+ rv = setup_netlink();
+ if (rv < 0)
+ goto out;
+ client_add(rv, process_netlink, NULL);
+
+ setup_deadlock();
+ }
+#endif
+
+ rv = setup_plocks();
+ if (rv < 0)
+ goto out;
+ plock_fd = rv;
+ plock_ci = client_add(rv, process_plocks, NULL);
+
+ for (;;) {
+ rv = poll(pollfd, client_maxi + 1, poll_timeout);
+ if (rv == -1 && errno == EINTR) {
+ if (daemon_quit && list_empty(&lockspaces))
+ goto out;
+ log_error("shutdown ignored, active lockspaces");
+ daemon_quit = 0;
+ continue;
+ }
+ if (rv < 0) {
+ log_error("poll errno %d", errno);
+ goto out;
+ }
+
+ query_lock();
+
+ for (i = 0; i <= client_maxi; i++) {
+ if (client[i].fd < 0)
+ continue;
+ if (pollfd[i].revents & POLLIN) {
+ workfn = client[i].workfn;
+ workfn(i);
+ }
+ if (pollfd[i].revents & (POLLERR | POLLHUP | POLLNVAL)) {
+ deadfn = client[i].deadfn;
+ deadfn(i);
+ }
+ }
+ query_unlock();
+
+ if (daemon_quit)
+ break;
+
+ query_lock();
+
+ poll_timeout = -1;
+
+ if (poll_fencing || poll_quorum || poll_fs) {
+ process_lockspace_changes();
+ poll_timeout = 1000;
+ }
+
+ if (poll_ignore_plock) {
+ if (!limit_plocks()) {
+ poll_ignore_plock = 0;
+ client_back(plock_ci, plock_fd);
+ }
+ poll_timeout = 1000;
+ }
+
+ if (poll_drop_plock) {
+ drop_resources_all();
+ if (poll_drop_plock)
+ poll_timeout = 1000;
+ }
+
+ query_unlock();
+ }
+ out:
+ log_debug("shutdown");
+ close_plocks();
+ close_cpg_daemon();
+ clear_configfs();
+ close_logging();
+ close_cluster();
+ close_cluster_cfg();
+
+ list_for_each_entry(ls, &lockspaces, list)
+ log_error("abandoned lockspace %s", ls->name);
+}
+
+static void lockfile(void)
+{
+ int fd, error;
+ struct flock lock;
+ char buf[33];
+
+ memset(buf, 0, 33);
+
+ fd = open(LOCKFILE_NAME, O_CREAT|O_WRONLY,
+ S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
+ if (fd < 0) {
+ fprintf(stderr, "cannot open/create lock file %s\n",
+ LOCKFILE_NAME);
+ exit(EXIT_FAILURE);
+ }
+
+ lock.l_type = F_WRLCK;
+ lock.l_start = 0;
+ lock.l_whence = SEEK_SET;
+ lock.l_len = 0;
+
+ error = fcntl(fd, F_SETLK, &lock);
+ if (error) {
+ fprintf(stderr, "dlm_controld is already running\n");
+ exit(EXIT_FAILURE);
+ }
+
+ error = ftruncate(fd, 0);
+ if (error) {
+ fprintf(stderr, "cannot clear lock file %s\n", LOCKFILE_NAME);
+ exit(EXIT_FAILURE);
+ }
+
+ sprintf(buf, "%d\n", getpid());
+
+ error = write(fd, buf, strlen(buf));
+ if (error <= 0) {
+ fprintf(stderr, "cannot write lock file %s\n", LOCKFILE_NAME);
+ exit(EXIT_FAILURE);
+ }
+}
+
+static void print_usage(void)
+{
+ printf("Usage:\n");
+ printf("\n");
+ printf("dlm_controld [options]\n");
+ printf("\n");
+ printf("Options:\n");
+ printf("\n");
+ printf(" -D Enable debugging to stderr and don't fork\n");
+ printf(" -L Enable debugging to log file\n");
+ printf(" -K Enable kernel dlm debugging messages\n");
+ printf(" -r <num> dlm kernel lowcomms protocol, 0 tcp, 1 sctp, 2
detect\n");
+ printf(" 2 selects tcp if corosync rrp_mode is \"none\",
otherwise sctp\n");
+ printf(" Default is 2\n");
+ printf(" -f <num> Enable (1) or disable (0) fencing recovery
dependency\n");
+ printf(" Default is %d\n", DEFAULT_ENABLE_FENCING);
+ printf(" -q <num> Enable (1) or disable (0) quorum recovery
dependency\n");
+ printf(" Default is %d\n", DEFAULT_ENABLE_QUORUM);
+#if 0
+ printf(" -d <num> Enable (1) or disable (0) deadlock detection
code\n");
+ printf(" Default is %d\n", DEFAULT_ENABLE_DEADLK);
+#endif
+ printf(" -p <num> Enable (1) or disable (0) plock code for cluster
fs\n");
+ printf(" Default is %d\n", DEFAULT_ENABLE_PLOCK);
+ printf(" -P Enable plock debugging\n");
+ printf(" -l <limit> Limit the rate of plock operations\n");
+ printf(" Default is %d, set to 0 for no limit\n", DEFAULT_PLOCK_RATE_LIMIT);
+ printf(" -o <n> Enable (1) or disable (0) plock ownership\n");
+ printf(" Default is %d\n", DEFAULT_PLOCK_OWNERSHIP);
+ printf(" -t <ms> plock ownership drop resources time
(milliseconds)\n");
+ printf(" Default is %u\n", DEFAULT_DROP_RESOURCES_TIME);
+ printf(" -c <num> plock ownership drop resources count\n");
+ printf(" Default is %u\n", DEFAULT_DROP_RESOURCES_COUNT);
+ printf(" -a <ms> plock ownership drop resources age
(milliseconds)\n");
+ printf(" Default is %u\n", DEFAULT_DROP_RESOURCES_AGE);
+ printf(" -h Print this help, then exit\n");
+ printf(" -V Print program version information, then exit\n");
+}
+
+#define OPTION_STRING "LDKf:q:p:Pl:o:t:c:a:hVr:"
+
+static void read_arguments(int argc, char **argv)
+{
+ int cont = 1;
+ int optchar;
+
+ while (cont) {
+ optchar = getopt(argc, argv, OPTION_STRING);
+
+ switch (optchar) {
+ case 'D':
+ daemon_debug_opt = 1;
+ break;
+
+ case 'L':
+ optd_debug_logfile = 1;
+ cfgd_debug_logfile = 1;
+ break;
+
+ case 'K':
+ optk_debug = 1;
+ cfgk_debug = 1;
+ break;
+
+ case 'r':
+ optk_protocol = 1;
+ cfgk_protocol = atoi(optarg);
+ break;
+
+ case 'f':
+ optd_enable_fencing = 1;
+ cfgd_enable_fencing = atoi(optarg);
+ break;
+
+ case 'q':
+ optd_enable_quorum = 1;
+ cfgd_enable_quorum = atoi(optarg);
+ break;
+
+ case 'p':
+ optd_enable_plock = 1;
+ cfgd_enable_plock = atoi(optarg);
+ break;
+
+ case 'P':
+ optd_plock_debug = 1;
+ cfgd_plock_debug = 1;
+ break;
+
+ case 'l':
+ optd_plock_rate_limit = 1;
+ cfgd_plock_rate_limit = atoi(optarg);
+ break;
+
+ case 'o':
+ optd_plock_ownership = 1;
+ cfgd_plock_ownership = atoi(optarg);
+ break;
+
+ case 't':
+ optd_drop_resources_time = 1;
+ cfgd_drop_resources_time = atoi(optarg);
+ break;
+
+ case 'c':
+ optd_drop_resources_count = 1;
+ cfgd_drop_resources_count = atoi(optarg);
+ break;
+
+ case 'a':
+ optd_drop_resources_age = 1;
+ cfgd_drop_resources_age = atoi(optarg);
+ break;
+
+ case 'h':
+ print_usage();
+ exit(EXIT_SUCCESS);
+ break;
+
+ case 'V':
+ printf("dlm_controld %s (built %s %s)\n",
+ VERSION, __DATE__, __TIME__);
+ printf("%s\n", REDHAT_COPYRIGHT);
+ exit(EXIT_SUCCESS);
+ break;
+
+ case ':':
+ case '?':
+ fprintf(stderr, "Please use '-h' for usage.\n");
+ exit(EXIT_FAILURE);
+ break;
+
+ case EOF:
+ cont = 0;
+ break;
+
+ default:
+ fprintf(stderr, "unknown option: %c\n", optchar);
+ exit(EXIT_FAILURE);
+ break;
+ };
+ }
+
+ if (getenv("DLM_CONTROLD_DEBUG")) {
+ optd_debug_logfile = 1;
+ cfgd_debug_logfile = 1;
+ }
+}
+
+static void set_scheduler(void)
+{
+ struct sched_param sched_param;
+ int rv;
+
+ rv = sched_get_priority_max(SCHED_RR);
+ if (rv != -1) {
+ sched_param.sched_priority = rv;
+ rv = sched_setscheduler(0, SCHED_RR, &sched_param);
+ if (rv == -1)
+ log_error("could not set SCHED_RR priority %d err %d",
+ sched_param.sched_priority, errno);
+ } else {
+ log_error("could not get maximum scheduler priority err %d",
+ errno);
+ }
+}
+
+int main(int argc, char **argv)
+{
+ INIT_LIST_HEAD(&lockspaces);
+ INIT_LIST_HEAD(&fs_register_list);
+
+ read_arguments(argc, argv);
+
+ if (!daemon_debug_opt) {
+ if (daemon(0, 0) < 0) {
+ perror("daemon error");
+ exit(EXIT_FAILURE);
+ }
+ }
+ lockfile();
+ init_logging();
+ log_level(NULL, LOG_INFO, "dlm_controld %s started", VERSION);
+ signal(SIGTERM, sigterm_handler);
+ set_scheduler();
+
+ loop();
+
+ return 0;
+}
+
+int daemon_debug_opt;
+int daemon_quit;
+int cluster_down;
+int poll_fencing;
+int poll_quorum;
+int poll_fs;
+int poll_ignore_plock;
+int poll_drop_plock;
+int plock_fd;
+int plock_ci;
+struct list_head lockspaces;
+int cluster_quorate;
+int our_nodeid;
+uint32_t cluster_ringid_seq;
+uint32_t control_minor;
+uint32_t monitor_minor;
+uint32_t plock_minor;
+uint32_t old_plock_minor;
+
+/* was a config value set on command line?, 0 or 1.
+ optk is a kernel option, optd is a daemon option */
+
+int optk_debug;
+int optk_timewarn;
+int optk_protocol;
+int optd_debug_logfile;
+int optd_enable_fencing;
+int optd_enable_quorum;
+int optd_enable_plock;
+int optd_plock_debug;
+int optd_plock_rate_limit;
+int optd_plock_ownership;
+int optd_drop_resources_time;
+int optd_drop_resources_count;
+int optd_drop_resources_age;
+
+/* actual config value from command line, cluster.conf, or default.
+ cfgk is a kernel config value, cfgd is a daemon config value */
+
+int cfgk_debug = -1;
+int cfgk_timewarn = -1;
+int cfgk_protocol = PROTO_DETECT;
+int cfgd_debug_logfile = DEFAULT_DEBUG_LOGFILE;
+int cfgd_enable_fencing = DEFAULT_ENABLE_FENCING;
+int cfgd_enable_quorum = DEFAULT_ENABLE_QUORUM;
+int cfgd_enable_plock = DEFAULT_ENABLE_PLOCK;
+int cfgd_plock_debug = DEFAULT_PLOCK_DEBUG;
+int cfgd_plock_rate_limit = DEFAULT_PLOCK_RATE_LIMIT;
+int cfgd_plock_ownership = DEFAULT_PLOCK_OWNERSHIP;
+int cfgd_drop_resources_time = DEFAULT_DROP_RESOURCES_TIME;
+int cfgd_drop_resources_count = DEFAULT_DROP_RESOURCES_COUNT;
+int cfgd_drop_resources_age = DEFAULT_DROP_RESOURCES_AGE;
+
diff --git a/dlm_controld/member_cman.c b/dlm_controld/member_cman.c
new file mode 100644
index 0000000..909c249
--- /dev/null
+++ b/dlm_controld/member_cman.c
@@ -0,0 +1,267 @@
+#include "dlm_daemon.h"
+#include <corosync/corotypes.h>
+#include <corosync/cfg.h>
+#include <corosync/quorum.h>
+#include "libfenced.h"
+
+static corosync_cfg_handle_t ch;
+static quorum_handle_t qh;
+static uint32_t old_nodes[MAX_NODES];
+static int old_node_count;
+static uint32_t quorum_nodes[MAX_NODES];
+static int quorum_node_count;
+
+static int is_member(uint32_t *node_list, int count, uint32_t nodeid)
+{
+ int i;
+
+ for (i = 0; i < count; i++) {
+ if (node_list[i] == nodeid)
+ return 1;
+ }
+ return 0;
+}
+
+static int is_old_member(uint32_t nodeid)
+{
+ return is_member(old_nodes, old_node_count, nodeid);
+}
+
+int is_cluster_member(uint32_t nodeid)
+{
+ return is_member(quorum_nodes, quorum_node_count, nodeid);
+}
+
+static void quorum_callback(quorum_handle_t h, uint32_t quorate,
+ uint64_t ring_seq, uint32_t node_list_entries,
+ uint32_t *node_list)
+{
+ corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES];
+ corosync_cfg_node_address_t *addrptr = addrs;
+ cs_error_t err;
+ int i, j, num_addrs;
+
+
+ cluster_quorate = quorate;
+ cluster_ringid_seq = (uint32_t)ring_seq;
+
+ log_debug("cluster quorum %u seq %u nodes %u",
+ cluster_quorate, cluster_ringid_seq, node_list_entries);
+
+ old_node_count = quorum_node_count;
+ memcpy(&old_nodes, &quorum_nodes, sizeof(old_nodes));
+
+ quorum_node_count = 0;
+ memset(&quorum_nodes, 0, sizeof(quorum_nodes));
+
+ for (i = 0; i < node_list_entries; i++)
+ quorum_nodes[quorum_node_count++] = node_list[i];
+
+ for (i = 0; i < old_node_count; i++) {
+ if (!is_cluster_member(old_nodes[i])) {
+ log_debug("cluster node %u removed seq %u",
+ old_nodes[i], cluster_ringid_seq);
+ node_history_cluster_remove(old_nodes[i]);
+ del_configfs_node(old_nodes[i]);
+ }
+ }
+
+ for (i = 0; i < quorum_node_count; i++) {
+ if (!is_old_member(quorum_nodes[i])) {
+ log_debug("cluster node %u added seq %u",
+ quorum_nodes[i], cluster_ringid_seq);
+ node_history_cluster_add(quorum_nodes[i]);
+
+ err = corosync_cfg_get_node_addrs(ch, quorum_nodes[i],
+ MAX_NODE_ADDRESSES,
+ &num_addrs, addrs);
+ if (err != CS_OK) {
+ log_error("corosync_cfg_get_node_addrs failed "
+ "nodeid %u", quorum_nodes[i]);
+ continue;
+ }
+
+ for (j = 0; j < num_addrs; j++) {
+ add_configfs_node(quorum_nodes[i],
+ addrptr[j].address,
+ addrptr[j].address_length,
+ (quorum_nodes[i] ==
+ our_nodeid));
+ }
+ }
+ }
+}
+
+static quorum_callbacks_t quorum_callbacks =
+{
+ .quorum_notify_fn = quorum_callback,
+};
+
+void process_cluster(int ci)
+{
+ cs_error_t err;
+
+ err = quorum_dispatch(qh, CS_DISPATCH_ALL);
+ if (err != CS_OK)
+ cluster_dead(0);
+}
+
+/* Force re-read of quorum nodes */
+void update_cluster(void)
+{
+ cs_error_t err;
+
+ err = quorum_dispatch(qh, CS_DISPATCH_ONE);
+ if (err != CS_OK)
+ cluster_dead(0);
+}
+
+int setup_cluster(void)
+{
+ cs_error_t err;
+ int fd;
+
+ err = quorum_initialize(&qh, &quorum_callbacks);
+ if (err != CS_OK) {
+ log_error("quorum init error %d", err);
+ return -1;
+ }
+
+ err = quorum_fd_get(qh, &fd);
+ if (err != CS_OK) {
+ log_error("quorum fd_get error %d", err);
+ goto fail;
+ }
+
+ err = quorum_trackstart(qh, CS_TRACK_CHANGES);
+ if (err != CS_OK) {
+ log_error("quorum trackstart error %d", err);
+ goto fail;
+ }
+
+ old_node_count = 0;
+ memset(&old_nodes, 0, sizeof(old_nodes));
+ quorum_node_count = 0;
+ memset(&quorum_nodes, 0, sizeof(quorum_nodes));
+
+ return fd;
+ fail:
+ quorum_finalize(qh);
+ return -1;
+}
+
+void close_cluster(void)
+{
+ quorum_trackstop(qh);
+ quorum_finalize(qh);
+}
+
+void kick_node_from_cluster(int nodeid)
+{
+ if (!nodeid) {
+ log_error("telling corosync to shut down cluster locally");
+ corosync_cfg_try_shutdown(ch,
+ COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
+ } else {
+ log_error("telling corosync to remove nodeid %d from cluster",
+ nodeid);
+ corosync_cfg_kill_node(ch, nodeid, "dlm_controld");
+ }
+}
+
+static void shutdown_callback(corosync_cfg_handle_t h,
+ corosync_cfg_shutdown_flags_t flags)
+{
+ if (flags & COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) {
+ if (list_empty(&lockspaces))
+ corosync_cfg_replyto_shutdown(ch,
+ COROSYNC_CFG_SHUTDOWN_FLAG_YES);
+ else {
+ log_debug("no to corosync shutdown");
+ corosync_cfg_replyto_shutdown(ch,
+ COROSYNC_CFG_SHUTDOWN_FLAG_NO);
+ }
+ }
+}
+
+static corosync_cfg_callbacks_t cfg_callbacks =
+{
+ .corosync_cfg_shutdown_callback = shutdown_callback,
+ .corosync_cfg_state_track_callback = NULL,
+};
+
+void process_cluster_cfg(int ci)
+{
+ cs_error_t err;
+
+ err = corosync_cfg_dispatch(ch, CS_DISPATCH_ALL);
+ if (err != CS_OK)
+ cluster_dead(0);
+}
+
+int setup_cluster_cfg(void)
+{
+ cs_error_t err;
+ unsigned int nodeid;
+ int fd;
+
+ err = corosync_cfg_initialize(&ch, &cfg_callbacks);
+ if (err != CS_OK) {
+ log_error("corosync cfg init error %d", err);
+ return -1;
+ }
+
+ err = corosync_cfg_fd_get(ch, &fd);
+ if (err != CS_OK) {
+ log_error("corosync cfg fd_get error %d", err);
+ corosync_cfg_finalize(ch);
+ return -1;
+ }
+
+ err = corosync_cfg_local_get(ch, &nodeid);
+ if (err != CS_OK) {
+ log_error("corosync cfg local_get error %d", err);
+ corosync_cfg_finalize(ch);
+ return -1;
+ }
+ our_nodeid = nodeid;
+ log_debug("our_nodeid %d", our_nodeid);
+
+ return fd;
+}
+
+void close_cluster_cfg(void)
+{
+ corosync_cfg_finalize(ch);
+}
+
+int fence_node_time(int nodeid, uint64_t *last_fenced_time)
+{
+ struct fenced_node nodeinfo;
+ int rv;
+
+ memset(&nodeinfo, 0, sizeof(nodeinfo));
+
+ rv = fenced_node_info(nodeid, &nodeinfo);
+ if (rv < 0)
+ return rv;
+
+ *last_fenced_time = nodeinfo.last_fenced_time;
+ return 0;
+}
+
+int fence_in_progress(int *count)
+{
+ struct fenced_domain domain;
+ int rv;
+
+ memset(&domain, 0, sizeof(domain));
+
+ rv = fenced_domain_info(&domain);
+ if (rv < 0)
+ return rv;
+
+ *count = domain.victim_count;
+ return 0;
+}
+
diff --git a/dlm_controld/netlink.c b/dlm_controld/netlink.c
new file mode 100644
index 0000000..63122f7
--- /dev/null
+++ b/dlm_controld/netlink.c
@@ -0,0 +1,225 @@
+#include "dlm_daemon.h"
+#include "config.h"
+#include <linux/dlm.h>
+#include <linux/netlink.h>
+#include <linux/genetlink.h>
+#include <linux/dlm_netlink.h>
+
+#define DEADLOCK_CHECK_SECS 10
+
+/* FIXME: look into using libnl/libnetlink */
+
+#define GENLMSG_DATA(glh) ((void *)((char *)NLMSG_DATA(glh) + GENL_HDRLEN))
+#define GENLMSG_PAYLOAD(glh) (NLMSG_PAYLOAD(glh, 0) - GENL_HDRLEN)
+#define NLA_DATA(na) ((void *)((char*)(na) + NLA_HDRLEN))
+#define NLA_PAYLOAD(len) (len - NLA_HDRLEN)
+
+/* Maximum size of response requested or message sent */
+#define MAX_MSG_SIZE 1024
+
+struct msgtemplate {
+ struct nlmsghdr n;
+ struct genlmsghdr g;
+ char buf[MAX_MSG_SIZE];
+};
+
+static int send_genetlink_cmd(int sd, uint16_t nlmsg_type, uint32_t nlmsg_pid,
+ uint8_t genl_cmd, uint16_t nla_type,
+ void *nla_data, int nla_len)
+{
+ struct nlattr *na;
+ struct sockaddr_nl nladdr;
+ int r, buflen;
+ char *buf;
+
+ struct msgtemplate msg;
+
+ msg.n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
+ msg.n.nlmsg_type = nlmsg_type;
+ msg.n.nlmsg_flags = NLM_F_REQUEST;
+ msg.n.nlmsg_seq = 0;
+ msg.n.nlmsg_pid = nlmsg_pid;
+ msg.g.cmd = genl_cmd;
+ msg.g.version = 0x1;
+ na = (struct nlattr *) GENLMSG_DATA(&msg);
+ na->nla_type = nla_type;
+ na->nla_len = nla_len + 1 + NLA_HDRLEN;
+ if (nla_data)
+ memcpy(NLA_DATA(na), nla_data, nla_len);
+ msg.n.nlmsg_len += NLMSG_ALIGN(na->nla_len);
+
+ buf = (char *) &msg;
+ buflen = msg.n.nlmsg_len ;
+ memset(&nladdr, 0, sizeof(nladdr));
+ nladdr.nl_family = AF_NETLINK;
+ while ((r = sendto(sd, buf, buflen, 0, (struct sockaddr *) &nladdr,
+ sizeof(nladdr))) < buflen) {
+ if (r > 0) {
+ buf += r;
+ buflen -= r;
+ } else if (errno != EAGAIN)
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * Probe the controller in genetlink to find the family id
+ * for the DLM family
+ */
+static int get_family_id(int sd)
+{
+ char genl_name[100];
+ struct {
+ struct nlmsghdr n;
+ struct genlmsghdr g;
+ char buf[256];
+ } ans;
+
+ int id = 0, rc;
+ struct nlattr *na;
+ int rep_len;
+
+ strcpy(genl_name, DLM_GENL_NAME);
+ rc = send_genetlink_cmd(sd, GENL_ID_CTRL, getpid(), CTRL_CMD_GETFAMILY,
+ CTRL_ATTR_FAMILY_NAME, (void *)genl_name,
+ strlen(DLM_GENL_NAME)+1);
+
+ rep_len = recv(sd, &ans, sizeof(ans), 0);
+ if (ans.n.nlmsg_type == NLMSG_ERROR ||
+ (rep_len < 0) || !NLMSG_OK((&ans.n), rep_len))
+ return 0;
+
+ na = (struct nlattr *) GENLMSG_DATA(&ans);
+ na = (struct nlattr *) ((char *) na + NLA_ALIGN(na->nla_len));
+ if (na->nla_type == CTRL_ATTR_FAMILY_ID) {
+ id = *(uint16_t *) NLA_DATA(na);
+ }
+ return id;
+}
+
+/* genetlink messages are timewarnings used as part of deadlock detection */
+
+int setup_netlink(void)
+{
+ struct sockaddr_nl snl;
+ int s, rv;
+ uint16_t id;
+
+ s = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
+ if (s < 0) {
+ log_error("generic netlink socket");
+ return s;
+ }
+
+ memset(&snl, 0, sizeof(snl));
+ snl.nl_family = AF_NETLINK;
+
+ rv = bind(s, (struct sockaddr *) &snl, sizeof(snl));
+ if (rv < 0) {
+ log_error("gen netlink bind error %d errno %d", rv, errno);
+ close(s);
+ return rv;
+ }
+
+ id = get_family_id(s);
+ if (!id) {
+ log_error("Error getting family id, errno %d", errno);
+ close(s);
+ return -1;
+ }
+
+ rv = send_genetlink_cmd(s, id, getpid(), DLM_CMD_HELLO, 0, NULL, 0);
+ if (rv < 0) {
+ log_error("error sending hello cmd, errno %d", errno);
+ close(s);
+ return -1;
+ }
+
+ return s;
+}
+
+static void process_timewarn(struct dlm_lock_data *data)
+{
+ struct lockspace *ls;
+ struct timeval now;
+ unsigned int sec;
+
+ ls = find_ls_id(data->lockspace_id);
+ if (!ls)
+ return;
+
+ data->resource_name[data->resource_namelen] = '\0';
+
+ log_group(ls, "timewarn: lkid %x pid %d name %s",
+ data->id, data->ownpid, data->resource_name);
+
+ /* Problem: we don't want to get a timewarn, assume it's resolved
+ by the current cycle, but in fact it's from a deadlock that
+ formed after the checkpoints for the current cycle. Then we'd
+ have to hope for another warning (that may not come) to trigger
+ a new cycle to catch the deadlock. If our last cycle ckpt
+ was say N (~5?) sec before we receive the timewarn, then we
+ can be confident that the cycle included the lock in question.
+ Otherwise, we're not sure if the warning is for a new deadlock
+ that's formed since our last cycle ckpt (unless it's a long
+ enough time since the last cycle that we're confident it *is*
+ a new deadlock). When there is a deadlock, I suspect it will
+ be common to receive warnings before, during, and possibly
+ after the cycle that resolves it. Wonder if we should record
+ timewarns and match them with deadlock cycles so we can tell
+ which timewarns are addressed by a given cycle and which aren't. */
+
+
+ gettimeofday(&now, NULL);
+
+ /* don't send a new start until at least SECS after the last
+ we sent, and at least SECS after the last completed cycle */
+
+ sec = now.tv_sec - ls->last_send_cycle_start.tv_sec;
+
+ if (sec < DEADLOCK_CHECK_SECS) {
+ log_group(ls, "skip send: recent send cycle %d sec", sec);
+ return;
+ }
+
+ sec = now.tv_sec - ls->cycle_end_time.tv_sec;
+
+ if (sec < DEADLOCK_CHECK_SECS) {
+ log_group(ls, "skip send: recent cycle end %d sec", sec);
+ return;
+ }
+
+ gettimeofday(&ls->last_send_cycle_start, NULL);
+
+ if (cfgd_enable_deadlk)
+ send_cycle_start(ls);
+}
+
+void process_netlink(int ci)
+{
+ struct msgtemplate msg;
+ struct nlattr *na;
+ int len;
+ int fd;
+
+ fd = client_fd(ci);
+
+ len = recv(fd, &msg, sizeof(msg), 0);
+
+ if (len < 0) {
+ log_error("nonfatal netlink error: errno %d", errno);
+ return;
+ }
+
+ if (msg.n.nlmsg_type == NLMSG_ERROR || !NLMSG_OK((&msg.n), len)) {
+ struct nlmsgerr *err = NLMSG_DATA(&msg);
+ log_error("fatal netlink error: errno %d", err->error);
+ return;
+ }
+
+ na = (struct nlattr *) GENLMSG_DATA(&msg);
+
+ process_timewarn((struct dlm_lock_data *) NLA_DATA(na));
+}
+
diff --git a/dlm_controld/plock.c b/dlm_controld/plock.c
new file mode 100644
index 0000000..c38b998
--- /dev/null
+++ b/dlm_controld/plock.c
@@ -0,0 +1,2197 @@
+#include "dlm_daemon.h"
+#include <linux/dlm_plock.h>
+
+/* FIXME: remove this once everyone is using the version of
+ * dlm_plock.h which defines it */
+
+#ifndef DLM_PLOCK_FL_CLOSE
+#warning DLM_PLOCK_FL_CLOSE undefined. Enabling build workaround.
+#define DLM_PLOCK_FL_CLOSE 1
+#define DLM_PLOCK_BUILD_WORKAROUND 1
+#endif
+
+static uint32_t plock_read_count;
+static uint32_t plock_recv_count;
+static uint32_t plock_rate_delays;
+static struct timeval plock_read_time;
+static struct timeval plock_recv_time;
+static struct timeval plock_rate_last;
+
+static int plock_device_fd = -1;
+static int need_fsid_translation = 0;
+
+extern int message_flow_control_on;
+
+#define RD_CONTINUE 0x00000001
+
+struct resource_data {
+ uint64_t number;
+ int owner;
+ uint32_t lock_count;
+ uint32_t flags;
+ uint32_t pad;
+};
+
+struct plock_data {
+ uint64_t start;
+ uint64_t end;
+ uint64_t owner;
+ uint32_t pid;
+ uint32_t nodeid;
+ uint8_t ex;
+ uint8_t waiter;
+ uint16_t pad1;
+ uint32_t pad;
+};
+
+#define R_GOT_UNOWN 0x00000001 /* have received owner=0 message */
+#define R_SEND_UNOWN 0x00000002 /* have sent owner=0 message */
+#define R_SEND_OWN 0x00000004 /* have sent owner=our_nodeid message */
+#define R_PURGE_UNOWN 0x00000008 /* set owner=0 in purge */
+#define R_SEND_DROP 0x00000010
+
+struct resource {
+ struct list_head list; /* list of resources */
+ uint64_t number;
+ int owner; /* nodeid or 0 for unowned */
+ uint32_t flags;
+ struct timeval last_access;
+ struct list_head locks; /* one lock for each range */
+ struct list_head waiters;
+ struct list_head pending; /* discovering r owner */
+ struct rb_node rb_node;
+};
+
+#define P_SYNCING 0x00000001 /* plock has been sent as part of sync but not
+ yet received */
+
+struct posix_lock {
+ struct list_head list; /* resource locks or waiters list */
+ uint32_t pid;
+ uint64_t owner;
+ uint64_t start;
+ uint64_t end;
+ int ex;
+ int nodeid;
+ uint32_t flags;
+};
+
+struct lock_waiter {
+ struct list_head list;
+ uint32_t flags;
+ struct dlm_plock_info info;
+};
+
+struct save_msg {
+ struct list_head list;
+ int nodeid;
+ int len;
+ int type;
+ char buf[0];
+};
+
+
+static void send_own(struct lockspace *ls, struct resource *r, int owner);
+static void save_pending_plock(struct lockspace *ls, struct resource *r,
+ struct dlm_plock_info *in);
+
+
+static int got_unown(struct resource *r)
+{
+ return !!(r->flags & R_GOT_UNOWN);
+}
+
+static void info_bswap_out(struct dlm_plock_info *i)
+{
+ i->version[0] = cpu_to_le32(i->version[0]);
+ i->version[1] = cpu_to_le32(i->version[1]);
+ i->version[2] = cpu_to_le32(i->version[2]);
+ i->pid = cpu_to_le32(i->pid);
+ i->nodeid = cpu_to_le32(i->nodeid);
+ i->rv = cpu_to_le32(i->rv);
+ i->fsid = cpu_to_le32(i->fsid);
+ i->number = cpu_to_le64(i->number);
+ i->start = cpu_to_le64(i->start);
+ i->end = cpu_to_le64(i->end);
+ i->owner = cpu_to_le64(i->owner);
+}
+
+static void info_bswap_in(struct dlm_plock_info *i)
+{
+ i->version[0] = le32_to_cpu(i->version[0]);
+ i->version[1] = le32_to_cpu(i->version[1]);
+ i->version[2] = le32_to_cpu(i->version[2]);
+ i->pid = le32_to_cpu(i->pid);
+ i->nodeid = le32_to_cpu(i->nodeid);
+ i->rv = le32_to_cpu(i->rv);
+ i->fsid = le32_to_cpu(i->fsid);
+ i->number = le64_to_cpu(i->number);
+ i->start = le64_to_cpu(i->start);
+ i->end = le64_to_cpu(i->end);
+ i->owner = le64_to_cpu(i->owner);
+}
+
+static const char *op_str(int optype)
+{
+ switch (optype) {
+ case DLM_PLOCK_OP_LOCK:
+ return "LK";
+ case DLM_PLOCK_OP_UNLOCK:
+ return "UN";
+ case DLM_PLOCK_OP_GET:
+ return "GET";
+ default:
+ return "??";
+ }
+}
+
+static const char *ex_str(int optype, int ex)
+{
+ if (optype == DLM_PLOCK_OP_UNLOCK || optype == DLM_PLOCK_OP_GET)
+ return "-";
+ if (ex)
+ return "WR";
+ else
+ return "RD";
+}
+
+/*
+ * In kernels before 2.6.26, plocks came from gfs2's lock_dlm module.
+ * Reading plocks from there as well should allow us to use cluster3
+ * on old (RHEL5) kernels. In this case, the fsid we read in plock_info
+ * structs is the mountgroup id, which we need to translate to the ls id.
+ */
+
+int setup_plocks(void)
+{
+ plock_read_count = 0;
+ plock_recv_count = 0;
+ plock_rate_delays = 0;
+ gettimeofday(&plock_read_time, NULL);
+ gettimeofday(&plock_recv_time, NULL);
+ gettimeofday(&plock_rate_last, NULL);
+
+ if (plock_minor) {
+ plock_device_fd = open("/dev/misc/dlm_plock", O_RDWR);
+ } else if (old_plock_minor) {
+ log_debug("setup_plocks using old lock_dlm interface");
+ need_fsid_translation = 1;
+ plock_device_fd = open("/dev/misc/lock_dlm_plock", O_RDWR);
+ }
+
+ if (plock_device_fd < 0) {
+ log_error("Failure to open plock device: %s", strerror(errno));
+ return -1;
+ }
+
+ log_debug("plocks %d", plock_device_fd);
+ log_debug("plock cpg message size: %u bytes",
+ (unsigned int) (sizeof(struct dlm_header) +
+ sizeof(struct dlm_plock_info)));
+
+ return plock_device_fd;
+}
+
+void close_plocks(void)
+{
+ if (plock_device_fd > 0)
+ close(plock_device_fd);
+}
+
+static uint32_t mg_to_ls_id(uint32_t fsid)
+{
+ struct lockspace *ls;
+ int do_set = 1;
+
+ retry:
+ list_for_each_entry(ls, &lockspaces, list) {
+ if (ls->associated_mg_id == fsid)
+ return ls->global_id;
+ }
+
+ if (do_set) {
+ do_set = 0;
+ set_associated_id(fsid);
+ goto retry;
+ }
+
+ return fsid;
+}
+
+/* FIXME: unify these two */
+
+static unsigned long time_diff_ms(struct timeval *begin, struct timeval *end)
+{
+ struct timeval result;
+ timersub(end, begin, &result);
+ return (result.tv_sec * 1000) + (result.tv_usec / 1000);
+}
+
+static uint64_t dt_usec(struct timeval *start, struct timeval *stop)
+{
+ uint64_t dt;
+
+ dt = stop->tv_sec - start->tv_sec;
+ dt *= 1000000;
+ dt += stop->tv_usec - start->tv_usec;
+ return dt;
+}
+
+static struct resource * rb_search_plock_resource(struct lockspace *ls, uint64_t number)
+{
+ struct rb_node *n = ls->plock_resources_root.rb_node;
+ struct resource *r;
+
+ while (n) {
+ r = rb_entry(n, struct resource, rb_node);
+ if (number < r->number)
+ n = n->rb_left;
+ else if (number > r->number)
+ n = n->rb_right;
+ else
+ return r;
+ }
+ return NULL;
+}
+
+static void rb_insert_plock_resource(struct lockspace *ls, struct resource *r)
+{
+ struct resource *entry;
+ struct rb_node **p;
+ struct rb_node *parent = NULL;
+
+ p = &ls->plock_resources_root.rb_node;
+ while (*p) {
+ parent = *p;
+ entry = rb_entry(parent, struct resource, rb_node);
+ if (r->number < entry->number)
+ p = &parent->rb_left;
+ else if (r->number > entry->number)
+ p = &parent->rb_right;
+ else
+ return;
+ }
+ rb_link_node(&r->rb_node, parent, p);
+ rb_insert_color(&r->rb_node, &ls->plock_resources_root);
+}
+
+static void rb_del_plock_resource(struct lockspace *ls, struct resource *r)
+{
+ if (!RB_EMPTY_NODE(&r->rb_node)) {
+ rb_erase(&r->rb_node, &ls->plock_resources_root);
+ RB_CLEAR_NODE(&r->rb_node);
+ }
+}
+
+static struct resource *search_resource(struct lockspace *ls, uint64_t number)
+{
+ struct resource *r;
+
+ list_for_each_entry(r, &ls->plock_resources, list) {
+ if (r->number == number)
+ return r;
+ }
+ return NULL;
+}
+
+static int find_resource(struct lockspace *ls, uint64_t number, int create,
+ struct resource **r_out)
+{
+ struct resource *r = NULL;
+ int rv = 0;
+
+ r = rb_search_plock_resource(ls, number);
+ if (r)
+ goto out;
+
+ if (create == 0) {
+ rv = -ENOENT;
+ goto out;
+ }
+
+ r = malloc(sizeof(struct resource));
+ if (!r) {
+ log_elock(ls, "find_resource no memory %d", errno);
+ rv = -ENOMEM;
+ goto out;
+ }
+
+ memset(r, 0, sizeof(struct resource));
+ r->number = number;
+ INIT_LIST_HEAD(&r->locks);
+ INIT_LIST_HEAD(&r->waiters);
+ INIT_LIST_HEAD(&r->pending);
+
+ if (cfgd_plock_ownership)
+ r->owner = -1;
+ else
+ r->owner = 0;
+
+ list_add_tail(&r->list, &ls->plock_resources);
+ rb_insert_plock_resource(ls, r);
+ out:
+ if (r)
+ gettimeofday(&r->last_access, NULL);
+ *r_out = r;
+ return rv;
+}
+
+static void put_resource(struct lockspace *ls, struct resource *r)
+{
+ /* with ownership, resources are only freed via drop messages */
+ if (cfgd_plock_ownership)
+ return;
+
+ if (list_empty(&r->locks) && list_empty(&r->waiters)) {
+ rb_del_plock_resource(ls, r);
+ list_del(&r->list);
+ free(r);
+ }
+}
+
+static inline int ranges_overlap(uint64_t start1, uint64_t end1,
+ uint64_t start2, uint64_t end2)
+{
+ if (end1 < start2 || start1 > end2)
+ return 0;
+ return 1;
+}
+
+/**
+ * overlap_type - returns a value based on the type of overlap
+ * @s1 - start of new lock range
+ * @e1 - end of new lock range
+ * @s2 - start of existing lock range
+ * @e2 - end of existing lock range
+ *
+ */
+
+static int overlap_type(uint64_t s1, uint64_t e1, uint64_t s2, uint64_t e2)
+{
+ int ret;
+
+ /*
+ * ---r1---
+ * ---r2---
+ */
+
+ if (s1 == s2 && e1 == e2)
+ ret = 0;
+
+ /*
+ * --r1--
+ * ---r2---
+ */
+
+ else if (s1 == s2 && e1 < e2)
+ ret = 1;
+
+ /*
+ * --r1--
+ * ---r2---
+ */
+
+ else if (s1 > s2 && e1 == e2)
+ ret = 1;
+
+ /*
+ * --r1--
+ * ---r2---
+ */
+
+ else if (s1 > s2 && e1 < e2)
+ ret = 2;
+
+ /*
+ * ---r1--- or ---r1--- or ---r1---
+ * --r2-- --r2-- --r2--
+ */
+
+ else if (s1 <= s2 && e1 >= e2)
+ ret = 3;
+
+ /*
+ * ---r1---
+ * ---r2---
+ */
+
+ else if (s1 > s2 && e1 > e2)
+ ret = 4;
+
+ /*
+ * ---r1---
+ * ---r2---
+ */
+
+ else if (s1 < s2 && e1 < e2)
+ ret = 4;
+
+ else
+ ret = -1;
+
+ return ret;
+}
+
+/* shrink the range start2:end2 by the partially overlapping start:end */
+
+static int shrink_range2(uint64_t *start2, uint64_t *end2,
+ uint64_t start, uint64_t end)
+{
+ int error = 0;
+
+ if (*start2 < start)
+ *end2 = start - 1;
+ else if (*end2 > end)
+ *start2 = end + 1;
+ else
+ error = -1;
+ return error;
+}
+
+static int shrink_range(struct posix_lock *po, uint64_t start, uint64_t end)
+{
+ return shrink_range2(&po->start, &po->end, start, end);
+}
+
+static int is_conflict(struct resource *r, struct dlm_plock_info *in, int get)
+{
+ struct posix_lock *po;
+
+ list_for_each_entry(po, &r->locks, list) {
+ if (po->nodeid == in->nodeid && po->owner == in->owner)
+ continue;
+ if (!ranges_overlap(po->start, po->end, in->start, in->end))
+ continue;
+
+ if (in->ex || po->ex) {
+ if (get) {
+ in->ex = po->ex;
+ in->pid = po->pid;
+ in->start = po->start;
+ in->end = po->end;
+ }
+ return 1;
+ }
+ }
+ return 0;
+}
+
+static int add_lock(struct resource *r, uint32_t nodeid, uint64_t owner,
+ uint32_t pid, int ex, uint64_t start, uint64_t end)
+{
+ struct posix_lock *po;
+
+ po = malloc(sizeof(struct posix_lock));
+ if (!po)
+ return -ENOMEM;
+ memset(po, 0, sizeof(struct posix_lock));
+
+ po->start = start;
+ po->end = end;
+ po->nodeid = nodeid;
+ po->owner = owner;
+ po->pid = pid;
+ po->ex = ex;
+ list_add_tail(&po->list, &r->locks);
+
+ return 0;
+}
+
+/* RN within RE (and starts or ends on RE boundary)
+ 1. add new lock for non-overlap area of RE, orig mode
+ 2. convert RE to RN range and mode */
+
+static int lock_case1(struct posix_lock *po, struct resource *r,
+ struct dlm_plock_info *in)
+{
+ uint64_t start2, end2;
+ int rv;
+
+ /* non-overlapping area start2:end2 */
+ start2 = po->start;
+ end2 = po->end;
+ rv = shrink_range2(&start2, &end2, in->start, in->end);
+ if (rv)
+ goto out;
+
+ po->start = in->start;
+ po->end = in->end;
+ po->ex = in->ex;
+
+ rv = add_lock(r, in->nodeid, in->owner, in->pid, !in->ex, start2, end2);
+ out:
+ return rv;
+}
+
+/* RN within RE (RE overlaps RN on both sides)
+ 1. add new lock for front fragment, orig mode
+ 2. add new lock for back fragment, orig mode
+ 3. convert RE to RN range and mode */
+
+static int lock_case2(struct posix_lock *po, struct resource *r,
+ struct dlm_plock_info *in)
+
+{
+ int rv;
+
+ rv = add_lock(r, in->nodeid, in->owner, in->pid,
+ !in->ex, po->start, in->start - 1);
+ if (rv)
+ goto out;
+
+ rv = add_lock(r, in->nodeid, in->owner, in->pid,
+ !in->ex, in->end + 1, po->end);
+ if (rv)
+ goto out;
+
+ po->start = in->start;
+ po->end = in->end;
+ po->ex = in->ex;
+ out:
+ return rv;
+}
+
+static int lock_internal(struct lockspace *ls, struct resource *r,
+ struct dlm_plock_info *in)
+{
+ struct posix_lock *po, *safe;
+ int rv = 0;
+
+ list_for_each_entry_safe(po, safe, &r->locks, list) {
+ if (po->nodeid != in->nodeid || po->owner != in->owner)
+ continue;
+ if (!ranges_overlap(po->start, po->end, in->start, in->end))
+ continue;
+
+ /* existing range (RE) overlaps new range (RN) */
+
+ switch(overlap_type(in->start, in->end, po->start, po->end)) {
+
+ case 0:
+ if (po->ex == in->ex)
+ goto out;
+
+ /* ranges the same - just update the existing lock */
+ po->ex = in->ex;
+ goto out;
+
+ case 1:
+ if (po->ex == in->ex)
+ goto out;
+
+ rv = lock_case1(po, r, in);
+ goto out;
+
+ case 2:
+ if (po->ex == in->ex)
+ goto out;
+
+ rv = lock_case2(po, r, in);
+ goto out;
+
+ case 3:
+ list_del(&po->list);
+ free(po);
+ break;
+
+ case 4:
+ if (po->start < in->start)
+ po->end = in->start - 1;
+ else
+ po->start = in->end + 1;
+ break;
+
+ default:
+ rv = -1;
+ goto out;
+ }
+ }
+
+ rv = add_lock(r, in->nodeid, in->owner, in->pid,
+ in->ex, in->start, in->end);
+ out:
+ return rv;
+
+}
+
+static int unlock_internal(struct lockspace *ls, struct resource *r,
+ struct dlm_plock_info *in)
+{
+ struct posix_lock *po, *safe;
+ int rv = 0;
+
+ list_for_each_entry_safe(po, safe, &r->locks, list) {
+ if (po->nodeid != in->nodeid || po->owner != in->owner)
+ continue;
+ if (!ranges_overlap(po->start, po->end, in->start, in->end))
+ continue;
+
+ /* existing range (RE) overlaps new range (RN) */
+
+ switch (overlap_type(in->start, in->end, po->start, po->end)) {
+
+ case 0:
+ /* ranges the same - just remove the existing lock */
+
+ list_del(&po->list);
+ free(po);
+ goto out;
+
+ case 1:
+ /* RN within RE and starts or ends on RE boundary -
+ * shrink and update RE */
+
+ rv = shrink_range(po, in->start, in->end);
+ goto out;
+
+ case 2:
+ /* RN within RE - shrink and update RE to be front
+ * fragment, and add a new lock for back fragment */
+
+ rv = add_lock(r, in->nodeid, in->owner, in->pid,
+ po->ex, in->end + 1, po->end);
+ po->end = in->start - 1;
+ goto out;
+
+ case 3:
+ /* RE within RN - remove RE, then continue checking
+ * because RN could cover other locks */
+
+ list_del(&po->list);
+ free(po);
+ continue;
+
+ case 4:
+ /* front of RE in RN, or end of RE in RN - shrink and
+ * update RE, then continue because RN could cover
+ * other locks */
+
+ rv = shrink_range(po, in->start, in->end);
+ continue;
+
+ default:
+ rv = -1;
+ goto out;
+ }
+ }
+ out:
+ return rv;
+}
+
+static void clear_waiters(struct lockspace *ls, struct resource *r,
+ struct dlm_plock_info *in)
+{
+ struct lock_waiter *w, *safe;
+
+ list_for_each_entry_safe(w, safe, &r->waiters, list) {
+ if (w->info.nodeid != in->nodeid || w->info.owner != in->owner)
+ continue;
+
+ list_del(&w->list);
+
+ log_elock(ls, "clear waiter %llx %llx-%llx %d/%u/%llx",
+ (unsigned long long)in->number,
+ (unsigned long long)in->start,
+ (unsigned long long)in->end,
+ in->nodeid, in->pid,
+ (unsigned long long)in->owner);
+ free(w);
+ }
+}
+
+static int add_waiter(struct lockspace *ls, struct resource *r,
+ struct dlm_plock_info *in)
+
+{
+ struct lock_waiter *w;
+
+ w = malloc(sizeof(struct lock_waiter));
+ if (!w)
+ return -ENOMEM;
+ memcpy(&w->info, in, sizeof(struct dlm_plock_info));
+ list_add_tail(&w->list, &r->waiters);
+ return 0;
+}
+
+static void write_result(struct lockspace *ls, struct dlm_plock_info *in,
+ int rv)
+{
+ if (need_fsid_translation)
+ in->fsid = ls->associated_mg_id;
+
+ in->rv = rv;
+ write(plock_device_fd, in, sizeof(struct dlm_plock_info));
+}
+
+static void do_waiters(struct lockspace *ls, struct resource *r)
+{
+ struct lock_waiter *w, *safe;
+ struct dlm_plock_info *in;
+ int rv;
+
+ list_for_each_entry_safe(w, safe, &r->waiters, list) {
+ in = &w->info;
+
+ if (is_conflict(r, in, 0))
+ continue;
+
+ list_del(&w->list);
+
+ /*
+ log_group(ls, "take waiter %llx %llx-%llx %d/%u/%llx",
+ in->number, in->start, in->end,
+ in->nodeid, in->pid, in->owner);
+ */
+
+ rv = lock_internal(ls, r, in);
+
+ if (in->nodeid == our_nodeid)
+ write_result(ls, in, rv);
+
+ free(w);
+ }
+}
+
+static void do_lock(struct lockspace *ls, struct dlm_plock_info *in,
+ struct resource *r)
+{
+ int rv;
+
+ if (is_conflict(r, in, 0)) {
+ if (!in->wait)
+ rv = -EAGAIN;
+ else {
+ rv = add_waiter(ls, r, in);
+ if (rv)
+ goto out;
+ rv = -EINPROGRESS;
+ }
+ } else
+ rv = lock_internal(ls, r, in);
+
+ out:
+ if (in->nodeid == our_nodeid && rv != -EINPROGRESS)
+ write_result(ls, in, rv);
+
+ do_waiters(ls, r);
+ put_resource(ls, r);
+}
+
+static void do_unlock(struct lockspace *ls, struct dlm_plock_info *in,
+ struct resource *r)
+{
+ int rv;
+
+ rv = unlock_internal(ls, r, in);
+
+#ifdef DLM_PLOCK_BUILD_WORKAROUND
+ if (in->pad & DLM_PLOCK_FL_CLOSE) {
+#else
+ if (in->flags & DLM_PLOCK_FL_CLOSE) {
+#endif
+ clear_waiters(ls, r, in);
+ /* no replies for unlock-close ops */
+ goto skip_result;
+ }
+
+ if (in->nodeid == our_nodeid)
+ write_result(ls, in, rv);
+
+ skip_result:
+ do_waiters(ls, r);
+ put_resource(ls, r);
+}
+
+/* we don't even get to this function if the getlk isn't from us */
+
+static void do_get(struct lockspace *ls, struct dlm_plock_info *in,
+ struct resource *r)
+{
+ int rv;
+
+ if (is_conflict(r, in, 1))
+ rv = 1;
+ else
+ rv = 0;
+
+ write_result(ls, in, rv);
+ put_resource(ls, r);
+}
+
+static void save_message(struct lockspace *ls, struct dlm_header *hd, int len,
+ int from, int type)
+{
+ struct save_msg *sm;
+
+ sm = malloc(sizeof(struct save_msg) + len);
+ if (!sm)
+ return;
+ memset(sm, 0, sizeof(struct save_msg) + len);
+
+ memcpy(&sm->buf, hd, len);
+ sm->type = type;
+ sm->len = len;
+ sm->nodeid = from;
+
+ log_plock(ls, "save %s from %d len %d", msg_name(type), from, len);
+
+ list_add_tail(&sm->list, &ls->saved_messages);
+}
+
+static void __receive_plock(struct lockspace *ls, struct dlm_plock_info *in,
+ int from, struct resource *r)
+{
+ switch (in->optype) {
+ case DLM_PLOCK_OP_LOCK:
+ ls->last_plock_time = time(NULL);
+ do_lock(ls, in, r);
+ break;
+ case DLM_PLOCK_OP_UNLOCK:
+ ls->last_plock_time = time(NULL);
+ do_unlock(ls, in, r);
+ break;
+ case DLM_PLOCK_OP_GET:
+ do_get(ls, in, r);
+ break;
+ default:
+ log_elock(ls, "receive_plock error from %d optype %d",
+ from, in->optype);
+ if (from == our_nodeid)
+ write_result(ls, in, -EINVAL);
+ }
+}
+
+/* When ls members receive our options message (for our mount), one of them
+ saves all plock state received to that point in a checkpoint and then sends
+ us our journals message. We know to retrieve the plock state from the
+ checkpoint when we receive our journals message. Any plocks messages that
+ arrive between seeing our options message and our journals message needs to
+ be saved and processed after we synchronize our plock state from the
+ checkpoint. Any plock message received while we're mounting but before we
+ set save_plocks (when we see our options message) can be ignored because it
+ should be reflected in the checkpointed state. */
+
+static void _receive_plock(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ struct dlm_plock_info info;
+ struct resource *r = NULL;
+ struct timeval now;
+ uint64_t usec;
+ int from = hd->nodeid;
+ int rv, create;
+
+ memcpy(&info, (char *)hd + sizeof(struct dlm_header), sizeof(info));
+ info_bswap_in(&info);
+
+ log_plock(ls, "receive plock %llx %s %s %llx-%llx %d/%u/%llx w %d",
+ (unsigned long long)info.number,
+ op_str(info.optype),
+ ex_str(info.optype, info.ex),
+ (unsigned long long)info.start, (unsigned long long)info.end,
+ info.nodeid, info.pid, (unsigned long long)info.owner,
+ info.wait);
+
+ plock_recv_count++;
+ if (!(plock_recv_count % 1000)) {
+ gettimeofday(&now, NULL);
+ usec = dt_usec(&plock_recv_time, &now);
+ log_plock(ls, "plock_recv_count %u time %.3f s",
+ plock_recv_count, usec * 1.e-6);
+ plock_recv_time = now;
+ }
+
+ if (info.optype == DLM_PLOCK_OP_GET && from != our_nodeid)
+ return;
+
+ if (from != hd->nodeid || from != info.nodeid) {
+ log_elock(ls, "receive_plock error from %d header %d info %d",
+ from, hd->nodeid, info.nodeid);
+ return;
+ }
+
+ create = !cfgd_plock_ownership;
+
+ rv = find_resource(ls, info.number, create, &r);
+
+ if (rv && cfgd_plock_ownership) {
+ /* There must have been a race with a drop, so we need to
+ ignore this plock op which will be resent. If we're the one
+ who sent the plock, we need to send_own() and put it on the
+ pending list to resend once the owner is established. */
+
+ log_plock(ls, "receive_plock from %d no r %llx", from,
+ (unsigned long long)info.number);
+
+ if (from != our_nodeid)
+ return;
+
+ rv = find_resource(ls, info.number, 1, &r);
+ if (rv)
+ return;
+ send_own(ls, r, our_nodeid);
+ save_pending_plock(ls, r, &info);
+ return;
+ }
+ if (rv) {
+ /* r not found, rv is -ENOENT, this shouldn't happen because
+ process_plocks() creates a resource for every op */
+
+ log_elock(ls, "receive_plock error from %d no r %llx %d",
+ from, (unsigned long long)info.number, rv);
+ return;
+ }
+
+ /* The owner should almost always be 0 here, but other owners may
+ be possible given odd combinations of races with drop. Odd races to
+ worry about (some seem pretty improbable):
+
+ - A sends drop, B sends plock, receive drop, receive plock.
+ This is addressed above.
+
+ - A sends drop, B sends plock, receive drop, B reads plock
+ and sends own, receive plock, on B we find owner of -1.
+
+ - A sends drop, B sends two plocks, receive drop, receive plocks.
+ Receiving the first plock is the previous case, receiving the
+ second plock will find r with owner of -1.
+
+ - A sends drop, B sends two plocks, receive drop, C sends own,
+ receive plock, B sends own, receive own (C), receive plock,
+ receive own (B).
+
+ Haven't tried to cook up a scenario that would lead to the
+ last case below; receiving a plock from ourself and finding
+ we're the owner of r. */
+
+ if (!r->owner) {
+ __receive_plock(ls, &info, from, r);
+
+ } else if (r->owner == -1) {
+ log_plock(ls, "receive_plock from %d r %llx owner %d", from,
+ (unsigned long long)info.number, r->owner);
+
+ if (from == our_nodeid)
+ save_pending_plock(ls, r, &info);
+
+ } else if (r->owner != our_nodeid) {
+ log_plock(ls, "receive_plock from %d r %llx owner %d", from,
+ (unsigned long long)info.number, r->owner);
+
+ if (from == our_nodeid)
+ save_pending_plock(ls, r, &info);
+
+ } else if (r->owner == our_nodeid) {
+ log_plock(ls, "receive_plock from %d r %llx owner %d", from,
+ (unsigned long long)info.number, r->owner);
+
+ if (from == our_nodeid)
+ __receive_plock(ls, &info, from, r);
+ }
+}
+
+void receive_plock(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ if (ls->save_plocks) {
+ save_message(ls, hd, len, hd->nodeid, DLM_MSG_PLOCK);
+ return;
+ }
+
+ _receive_plock(ls, hd, len);
+}
+
+static int send_struct_info(struct lockspace *ls, struct dlm_plock_info *in,
+ int msg_type)
+{
+ struct dlm_header *hd;
+ int rv = 0, len;
+ char *buf;
+
+ len = sizeof(struct dlm_header) + sizeof(struct dlm_plock_info);
+ buf = malloc(len);
+ if (!buf) {
+ rv = -ENOMEM;
+ goto out;
+ }
+ memset(buf, 0, len);
+
+ info_bswap_out(in);
+
+ hd = (struct dlm_header *)buf;
+ hd->type = msg_type;
+
+ memcpy(buf + sizeof(struct dlm_header), in, sizeof(*in));
+
+ dlm_send_message(ls, buf, len);
+
+ free(buf);
+ out:
+ if (rv)
+ log_elock(ls, "send_struct_info error %d", rv);
+ return rv;
+}
+
+static void send_plock(struct lockspace *ls, struct resource *r,
+ struct dlm_plock_info *in)
+{
+ send_struct_info(ls, in, DLM_MSG_PLOCK);
+}
+
+static void send_own(struct lockspace *ls, struct resource *r, int owner)
+{
+ struct dlm_plock_info info;
+
+ /* if we've already sent an own message for this resource,
+ (pending list is not empty), then we shouldn't send another */
+
+ if (!list_empty(&r->pending)) {
+ log_plock(ls, "send_own %llx already pending",
+ (unsigned long long)r->number);
+ return;
+ }
+
+ if (!owner)
+ r->flags |= R_SEND_UNOWN;
+ else
+ r->flags |= R_SEND_OWN;
+
+ memset(&info, 0, sizeof(info));
+ info.number = r->number;
+ info.nodeid = owner;
+
+ send_struct_info(ls, &info, DLM_MSG_PLOCK_OWN);
+}
+
+static void send_syncs(struct lockspace *ls, struct resource *r)
+{
+ struct dlm_plock_info info;
+ struct posix_lock *po;
+ struct lock_waiter *w;
+ int rv;
+
+ list_for_each_entry(po, &r->locks, list) {
+ memset(&info, 0, sizeof(info));
+ info.number = r->number;
+ info.start = po->start;
+ info.end = po->end;
+ info.nodeid = po->nodeid;
+ info.owner = po->owner;
+ info.pid = po->pid;
+ info.ex = po->ex;
+
+ rv = send_struct_info(ls, &info, DLM_MSG_PLOCK_SYNC_LOCK);
+ if (rv)
+ goto out;
+
+ po->flags |= P_SYNCING;
+ }
+
+ list_for_each_entry(w, &r->waiters, list) {
+ memcpy(&info, &w->info, sizeof(info));
+
+ rv = send_struct_info(ls, &info, DLM_MSG_PLOCK_SYNC_WAITER);
+ if (rv)
+ goto out;
+
+ w->flags |= P_SYNCING;
+ }
+ out:
+ return;
+}
+
+static void send_drop(struct lockspace *ls, struct resource *r)
+{
+ struct dlm_plock_info info;
+
+ memset(&info, 0, sizeof(info));
+ info.number = r->number;
+ r->flags |= R_SEND_DROP;
+
+ send_struct_info(ls, &info, DLM_MSG_PLOCK_DROP);
+}
+
+/* plock op can't be handled until we know the owner value of the resource,
+ so the op is saved on the pending list until the r owner is established */
+
+static void save_pending_plock(struct lockspace *ls, struct resource *r,
+ struct dlm_plock_info *in)
+{
+ struct lock_waiter *w;
+
+ w = malloc(sizeof(struct lock_waiter));
+ if (!w) {
+ log_elock(ls, "save_pending_plock no mem");
+ return;
+ }
+ memcpy(&w->info, in, sizeof(struct dlm_plock_info));
+ list_add_tail(&w->list, &r->pending);
+}
+
+/* plock ops are on pending list waiting for ownership to be established.
+ owner has now become us, so add these plocks to r */
+
+static void add_pending_plocks(struct lockspace *ls, struct resource *r)
+{
+ struct lock_waiter *w, *safe;
+
+ list_for_each_entry_safe(w, safe, &r->pending, list) {
+ __receive_plock(ls, &w->info, our_nodeid, r);
+ list_del(&w->list);
+ free(w);
+ }
+}
+
+/* plock ops are on pending list waiting for ownership to be established.
+ owner has now become 0, so send these plocks to everyone */
+
+static void send_pending_plocks(struct lockspace *ls, struct resource *r)
+{
+ struct lock_waiter *w, *safe;
+
+ list_for_each_entry_safe(w, safe, &r->pending, list) {
+ send_plock(ls, r, &w->info);
+ list_del(&w->list);
+ free(w);
+ }
+}
+
+static void _receive_own(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ struct dlm_plock_info info;
+ struct resource *r;
+ int should_not_happen = 0;
+ int from = hd->nodeid;
+ int rv;
+
+ ls->last_plock_time = time(NULL);
+
+ memcpy(&info, (char *)hd + sizeof(struct dlm_header), sizeof(info));
+ info_bswap_in(&info);
+
+ log_plock(ls, "receive_own %llx from %u owner %u",
+ (unsigned long long)info.number, hd->nodeid, info.nodeid);
+
+ rv = find_resource(ls, info.number, 1, &r);
+ if (rv)
+ return;
+
+ if (from == our_nodeid) {
+ /*
+ * received our own own message
+ */
+
+ if (info.nodeid == 0) {
+ /* we are setting owner to 0 */
+
+ if (r->owner == our_nodeid) {
+ /* we set owner to 0 when we relinquish
+ ownership */
+ should_not_happen = 1;
+ } else if (r->owner == 0) {
+ /* this happens when we relinquish ownership */
+ r->flags |= R_GOT_UNOWN;
+ } else {
+ should_not_happen = 1;
+ }
+
+ } else if (info.nodeid == our_nodeid) {
+ /* we are setting owner to ourself */
+
+ if (r->owner == -1) {
+ /* we have gained ownership */
+ r->owner = our_nodeid;
+ add_pending_plocks(ls, r);
+ } else if (r->owner == our_nodeid) {
+ should_not_happen = 1;
+ } else if (r->owner == 0) {
+ send_pending_plocks(ls, r);
+ } else {
+ /* resource is owned by other node;
+ they should set owner to 0 shortly */
+ }
+
+ } else {
+ /* we should only ever set owner to 0 or ourself */
+ should_not_happen = 1;
+ }
+ } else {
+ /*
+ * received own message from another node
+ */
+
+ if (info.nodeid == 0) {
+ /* other node is setting owner to 0 */
+
+ if (r->owner == -1) {
+ /* we should have a record of the owner before
+ it relinquishes */
+ should_not_happen = 1;
+ } else if (r->owner == our_nodeid) {
+ /* only the owner should relinquish */
+ should_not_happen = 1;
+ } else if (r->owner == 0) {
+ should_not_happen = 1;
+ } else {
+ r->owner = 0;
+ r->flags |= R_GOT_UNOWN;
+ send_pending_plocks(ls, r);
+ }
+
+ } else if (info.nodeid == from) {
+ /* other node is setting owner to itself */
+
+ if (r->owner == -1) {
+ /* normal path for a node becoming owner */
+ r->owner = from;
+ } else if (r->owner == our_nodeid) {
+ /* we relinquish our ownership: sync our local
+ plocks to everyone, then set owner to 0 */
+ send_syncs(ls, r);
+ send_own(ls, r, 0);
+ /* we need to set owner to 0 here because
+ local ops may arrive before we receive
+ our send_own message and can't be added
+ locally */
+ r->owner = 0;
+ } else if (r->owner == 0) {
+ /* can happen because we set owner to 0 before
+ we receive our send_own sent just above */
+ } else {
+ /* do nothing, current owner should be
+ relinquishing its ownership */
+ }
+
+ } else if (info.nodeid == our_nodeid) {
+ /* no one else should try to set the owner to us */
+ should_not_happen = 1;
+ } else {
+ /* a node should only ever set owner to 0 or itself */
+ should_not_happen = 1;
+ }
+ }
+
+ if (should_not_happen) {
+ log_elock(ls, "receive_own error from %u %llx "
+ "info nodeid %d r owner %d",
+ from, (unsigned long long)r->number,
+ info.nodeid, r->owner);
+ }
+}
+
+void receive_own(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ if (ls->save_plocks) {
+ save_message(ls, hd, len, hd->nodeid, DLM_MSG_PLOCK_OWN);
+ return;
+ }
+
+ _receive_own(ls, hd, len);
+}
+
+static void clear_syncing_flag(struct lockspace *ls, struct resource *r,
+ struct dlm_plock_info *in)
+{
+ struct posix_lock *po;
+ struct lock_waiter *w;
+
+ list_for_each_entry(po, &r->locks, list) {
+ if ((po->flags & P_SYNCING) &&
+ in->start == po->start &&
+ in->end == po->end &&
+ in->nodeid == po->nodeid &&
+ in->owner == po->owner &&
+ in->pid == po->pid &&
+ in->ex == po->ex) {
+ po->flags &= ~P_SYNCING;
+ return;
+ }
+ }
+
+ list_for_each_entry(w, &r->waiters, list) {
+ if ((w->flags & P_SYNCING) &&
+ in->start == w->info.start &&
+ in->end == w->info.end &&
+ in->nodeid == w->info.nodeid &&
+ in->owner == w->info.owner &&
+ in->pid == w->info.pid &&
+ in->ex == w->info.ex) {
+ w->flags &= ~P_SYNCING;
+ return;
+ }
+ }
+
+ log_elock(ls, "clear_syncing error %llx no match %s %llx-%llx %d/%u/%llx",
+ (unsigned long long)r->number,
+ in->ex ? "WR" : "RD",
+ (unsigned long long)in->start,
+ (unsigned long long)in->end,
+ in->nodeid, in->pid,
+ (unsigned long long)in->owner);
+}
+
+static void _receive_sync(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ struct dlm_plock_info info;
+ struct resource *r;
+ int from = hd->nodeid;
+ int rv;
+
+ ls->last_plock_time = time(NULL);
+
+ memcpy(&info, (char *)hd + sizeof(struct dlm_header), sizeof(info));
+ info_bswap_in(&info);
+
+ log_plock(ls, "receive sync %llx from %u %s %llx-%llx %d/%u/%llx",
+ (unsigned long long)info.number, from, info.ex ? "WR" : "RD",
+ (unsigned long long)info.start, (unsigned long long)info.end,
+ info.nodeid, info.pid, (unsigned long long)info.owner);
+
+ rv = find_resource(ls, info.number, 0, &r);
+ if (rv) {
+ log_elock(ls, "receive_sync error no r %llx from %d",
+ info.number, from);
+ return;
+ }
+
+ if (from == our_nodeid) {
+ /* this plock now in sync on all nodes */
+ clear_syncing_flag(ls, r, &info);
+ return;
+ }
+
+ if (hd->type == DLM_MSG_PLOCK_SYNC_LOCK)
+ add_lock(r, info.nodeid, info.owner, info.pid, info.ex,
+ info.start, info.end);
+ else if (hd->type == DLM_MSG_PLOCK_SYNC_WAITER)
+ add_waiter(ls, r, &info);
+}
+
+void receive_sync(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ if (ls->save_plocks) {
+ save_message(ls, hd, len, hd->nodeid, hd->type);
+ return;
+ }
+
+ _receive_sync(ls, hd, len);
+}
+
+static void _receive_drop(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ struct dlm_plock_info info;
+ struct resource *r;
+ int from = hd->nodeid;
+ int rv;
+
+ ls->last_plock_time = time(NULL);
+
+ memcpy(&info, (char *)hd + sizeof(struct dlm_header), sizeof(info));
+ info_bswap_in(&info);
+
+ log_plock(ls, "receive_drop %llx from %u",
+ (unsigned long long)info.number, from);
+
+ rv = find_resource(ls, info.number, 0, &r);
+ if (rv) {
+ /* we'll find no r if two nodes sent drop at once */
+ log_plock(ls, "receive_drop from %d no r %llx", from,
+ (unsigned long long)info.number);
+ return;
+ }
+
+ if (r->owner != 0) {
+ /* - A sent drop, B sent drop, receive drop A, C sent own,
+ receive drop B (this warning on C, owner -1)
+ - A sent drop, B sent drop, receive drop A, A sent own,
+ receive own A, receive drop B (this warning on all,
+ owner A) */
+ log_plock(ls, "receive_drop from %d r %llx owner %d", from,
+ (unsigned long long)r->number, r->owner);
+ return;
+ }
+
+ if (!list_empty(&r->pending)) {
+ /* shouldn't happen */
+ log_elock(ls, "receive_drop error from %d r %llx pending op",
+ from, (unsigned long long)r->number);
+ return;
+ }
+
+ /* the decision to drop or not must be based on things that are
+ guaranteed to be the same on all nodes */
+
+ if (list_empty(&r->locks) && list_empty(&r->waiters)) {
+ rb_del_plock_resource(ls, r);
+ list_del(&r->list);
+ free(r);
+ } else {
+ /* A sent drop, B sent a plock, receive plock, receive drop */
+ log_plock(ls, "receive_drop from %d r %llx in use", from,
+ (unsigned long long)r->number);
+ }
+}
+
+void receive_drop(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ if (ls->save_plocks) {
+ save_message(ls, hd, len, hd->nodeid, DLM_MSG_PLOCK_DROP);
+ return;
+ }
+
+ _receive_drop(ls, hd, len);
+}
+
+/* We only drop resources from the unowned state to simplify things.
+ If we want to drop a resource we own, we unown/relinquish it first. */
+
+/* FIXME: in the transition from owner = us, to owner = 0, to drop;
+ we want the second period to be shorter than the first */
+
+static int drop_resources(struct lockspace *ls)
+{
+ struct resource *r;
+ struct timeval now;
+ int count = 0;
+
+ if (!cfgd_plock_ownership)
+ return 0;
+
+ if (list_empty(&ls->plock_resources))
+ return 0;
+
+ gettimeofday(&now, NULL);
+
+ if (time_diff_ms(&ls->drop_resources_last, &now) <
+ cfgd_drop_resources_time)
+ return 1;
+
+ ls->drop_resources_last = now;
+
+ /* try to drop the oldest, unused resources */
+
+ list_for_each_entry_reverse(r, &ls->plock_resources, list) {
+ if (count >= cfgd_drop_resources_count)
+ break;
+ if (r->owner && r->owner != our_nodeid)
+ continue;
+ if (time_diff_ms(&r->last_access, &now) <
+ cfgd_drop_resources_age)
+ continue;
+
+ if (list_empty(&r->locks) && list_empty(&r->waiters)) {
+ if (r->owner == our_nodeid) {
+ send_own(ls, r, 0);
+ r->owner = 0;
+ } else if (r->owner == 0 && got_unown(r)) {
+ send_drop(ls, r);
+ }
+
+ count++;
+ }
+ }
+
+ return 1;
+}
+
+void drop_resources_all(void)
+{
+ struct lockspace *ls;
+ int rv = 0;
+
+ poll_drop_plock = 0;
+
+ list_for_each_entry(ls, &lockspaces, list) {
+ rv = drop_resources(ls);
+ if (rv)
+ poll_drop_plock = 1;
+ }
+}
+
+int limit_plocks(void)
+{
+ struct timeval now;
+
+ /* Don't send more messages while the cpg message queue is backed up */
+
+ if (message_flow_control_on) {
+ update_flow_control_status();
+ if (message_flow_control_on)
+ return 1;
+ }
+
+ if (!cfgd_plock_rate_limit || !plock_read_count)
+ return 0;
+
+ gettimeofday(&now, NULL);
+
+ /* Every time a plock op is read from the kernel, we increment
+ plock_read_count. After every cfgd_plock_rate_limit (N) reads,
+ we check the time it's taken to do those N; if the time is less than
+ a second, then we delay reading any more until a second is up.
+ This way we read a max of N ops from the kernel every second. */
+
+ if (!(plock_read_count % cfgd_plock_rate_limit)) {
+ if (time_diff_ms(&plock_rate_last, &now) < 1000) {
+ plock_rate_delays++;
+ return 2;
+ }
+ plock_rate_last = now;
+ plock_read_count++;
+ }
+ return 0;
+}
+
+void process_plocks(int ci)
+{
+ struct lockspace *ls;
+ struct resource *r;
+ struct dlm_plock_info info;
+ struct timeval now;
+ uint64_t usec;
+ int create, rv;
+
+ if (limit_plocks()) {
+ poll_ignore_plock = 1;
+ client_ignore(plock_ci, plock_fd);
+ return;
+ }
+
+ gettimeofday(&now, NULL);
+
+ memset(&info, 0, sizeof(info));
+
+ rv = do_read(plock_device_fd, &info, sizeof(info));
+ if (rv < 0) {
+ log_debug("process_plocks: read error %d fd %d\n",
+ errno, plock_device_fd);
+ return;
+ }
+
+ /* kernel doesn't set the nodeid field */
+ info.nodeid = our_nodeid;
+
+ if (!cfgd_enable_plock) {
+ rv = -ENOSYS;
+ goto fail;
+ }
+
+ if (need_fsid_translation)
+ info.fsid = mg_to_ls_id(info.fsid);
+
+ ls = find_ls_id(info.fsid);
+ if (!ls) {
+ log_plock(ls, "process_plocks: no ls id %x", info.fsid);
+ rv = -EEXIST;
+ goto fail;
+ }
+
+ if (ls->disable_plock) {
+ rv = -ENOSYS;
+ goto fail;
+ }
+
+ log_plock(ls, "read plock %llx %s %s %llx-%llx %d/%u/%llx w %d",
+ (unsigned long long)info.number,
+ op_str(info.optype),
+ ex_str(info.optype, info.ex),
+ (unsigned long long)info.start, (unsigned long long)info.end,
+ info.nodeid, info.pid, (unsigned long long)info.owner,
+ info.wait);
+
+ /* report plock rate and any delays since the last report */
+ plock_read_count++;
+ if (!(plock_read_count % 1000)) {
+ usec = dt_usec(&plock_read_time, &now) ;
+ log_plock(ls, "plock_read_count %u time %.3f s delays %u",
+ plock_read_count, usec * 1.e-6, plock_rate_delays);
+ plock_read_time = now;
+ plock_rate_delays = 0;
+ }
+
+ create = (info.optype == DLM_PLOCK_OP_UNLOCK) ? 0 : 1;
+
+ rv = find_resource(ls, info.number, create, &r);
+ if (rv)
+ goto fail;
+
+ if (r->owner == 0) {
+ /* plock state replicated on all nodes */
+ send_plock(ls, r, &info);
+
+ } else if (r->owner == our_nodeid) {
+ /* we are the owner of r, so our plocks are local */
+ __receive_plock(ls, &info, our_nodeid, r);
+
+ } else {
+ /* r owner is -1: r is new, try to become the owner;
+ r owner > 0: tell other owner to give up ownership;
+ both done with a message trying to set owner to ourself */
+ send_own(ls, r, our_nodeid);
+ save_pending_plock(ls, r, &info);
+ }
+
+ if (cfgd_plock_ownership && !list_empty(&ls->plock_resources))
+ poll_drop_plock = 1;
+ return;
+
+ fail:
+#ifdef DLM_PLOCK_BUILD_WORKAROUND
+ if (!(info.pad & DLM_PLOCK_FL_CLOSE)) {
+#else
+ if (!(info.flags & DLM_PLOCK_FL_CLOSE)) {
+#endif
+ info.rv = rv;
+ rv = write(plock_device_fd, &info, sizeof(info));
+ }
+}
+
+void process_saved_plocks(struct lockspace *ls)
+{
+ struct save_msg *sm, *sm2;
+ struct dlm_header *hd;
+ int count = 0;
+
+ log_dlock(ls, "process_saved_plocks begin");
+
+ if (list_empty(&ls->saved_messages))
+ goto out;
+
+ list_for_each_entry_safe(sm, sm2, &ls->saved_messages, list) {
+ hd = (struct dlm_header *)sm->buf;
+
+ switch (sm->type) {
+ case DLM_MSG_PLOCK:
+ _receive_plock(ls, hd, sm->len);
+ break;
+ case DLM_MSG_PLOCK_OWN:
+ _receive_own(ls, hd, sm->len);
+ break;
+ case DLM_MSG_PLOCK_DROP:
+ _receive_drop(ls, hd, sm->len);
+ break;
+ case DLM_MSG_PLOCK_SYNC_LOCK:
+ case DLM_MSG_PLOCK_SYNC_WAITER:
+ _receive_sync(ls, hd, sm->len);
+ break;
+ default:
+ continue;
+ }
+
+ list_del(&sm->list);
+ free(sm);
+ count++;
+ }
+ out:
+ log_dlock(ls, "process_saved_plocks %d done", count);
+}
+
+/* locks still marked SYNCING should not go into the ckpt; the new node
+ will get those locks by receiving PLOCK_SYNC messages */
+
+#define MAX_SEND_SIZE 1024 /* 1024 holds 24 plock_data */
+
+static char send_buf[MAX_SEND_SIZE];
+
+static int pack_send_buf(struct lockspace *ls, struct resource *r, int owner,
+ int full, int *count_out, void **last)
+{
+ struct resource_data *rd;
+ struct plock_data *pp;
+ struct posix_lock *po;
+ struct lock_waiter *w;
+ int count = 0;
+ int find = 0;
+ int len;
+
+ /* N.B. owner not always equal to r->owner */
+ rd = (struct resource_data *)(send_buf + sizeof(struct dlm_header));
+ rd->number = cpu_to_le64(r->number);
+ rd->owner = cpu_to_le32(owner);
+
+ if (full) {
+ rd->flags = RD_CONTINUE;
+ find = 1;
+ }
+
+ /* plocks not replicated for owned resources */
+ if (cfgd_plock_ownership && (owner == our_nodeid))
+ goto done;
+
+ len = sizeof(struct dlm_header) + sizeof(struct resource_data);
+
+ pp = (struct plock_data *)(send_buf + sizeof(struct dlm_header) + sizeof(struct
resource_data));
+
+ list_for_each_entry(po, &r->locks, list) {
+ if (find && *last != po)
+ continue;
+ find = 0;
+
+ if (po->flags & P_SYNCING)
+ continue;
+
+ if (len + sizeof(struct plock_data) > sizeof(send_buf)) {
+ *last = po;
+ goto full;
+ }
+ len += sizeof(struct plock_data);
+
+ pp->start = cpu_to_le64(po->start);
+ pp->end = cpu_to_le64(po->end);
+ pp->owner = cpu_to_le64(po->owner);
+ pp->pid = cpu_to_le32(po->pid);
+ pp->nodeid = cpu_to_le32(po->nodeid);
+ pp->ex = po->ex;
+ pp->waiter = 0;
+ pp++;
+ count++;
+ }
+
+ list_for_each_entry(w, &r->waiters, list) {
+ if (find && *last != w)
+ continue;
+ find = 0;
+
+ if (w->flags & P_SYNCING)
+ continue;
+
+ if (len + sizeof(struct plock_data) > sizeof(send_buf)) {
+ *last = w;
+ goto full;
+ }
+ len += sizeof(struct plock_data);
+
+ pp->start = cpu_to_le64(w->info.start);
+ pp->end = cpu_to_le64(w->info.end);
+ pp->owner = cpu_to_le64(w->info.owner);
+ pp->pid = cpu_to_le32(w->info.pid);
+ pp->nodeid = cpu_to_le32(w->info.nodeid);
+ pp->ex = w->info.ex;
+ pp->waiter = 1;
+ pp++;
+ count++;
+ }
+ done:
+ rd->lock_count = cpu_to_le32(count);
+ *count_out = count;
+ *last = NULL;
+ return 0;
+
+ full:
+ rd->lock_count = cpu_to_le32(count);
+ *count_out = count;
+ return 1;
+}
+
+/* Copy all plock state into a checkpoint so new node can retrieve it. The
+ node creating the ckpt for the mounter needs to be the same node that's
+ sending the mounter its journals message (i.e. the low nodeid). The new
+ mounter knows the ckpt is ready to read only after it gets its journals
+ message.
+
+ If the mounter is becoming the new low nodeid in the group, the node doing
+ the store closes the ckpt and the new node unlinks the ckpt after reading
+ it. The ckpt should then disappear and the new node can create a new ckpt
+ for the next mounter. */
+
+static int send_plocks_data(struct lockspace *ls, uint32_t seq, char *buf, int len)
+{
+ struct dlm_header *hd;
+
+ hd = (struct dlm_header *)buf;
+ hd->type = DLM_MSG_PLOCKS_DATA;
+ hd->msgdata = seq;
+
+ dlm_send_message(ls, buf, len);
+
+ return 0;
+}
+
+void send_all_plocks_data(struct lockspace *ls, uint32_t seq, uint32_t *plocks_data)
+{
+ struct resource *r;
+ void *last;
+ int owner, count, len, full;
+ uint32_t send_count = 0;
+
+ if (!cfgd_enable_plock || ls->disable_plock)
+ return;
+
+ log_dlock(ls, "send_all_plocks_data %d:%u", our_nodeid, seq);
+
+ /* - If r owner is -1, ckpt nothing.
+ - If r owner is us, ckpt owner of us and no plocks.
+ - If r owner is other, ckpt that owner and any plocks we have on r
+ (they've just been synced but owner=0 msg not recved yet).
+ - If r owner is 0 and !got_unown, then we've just unowned r;
+ ckpt owner of us and any plocks that don't have SYNCING set
+ (plocks with SYNCING will be handled by our sync messages).
+ - If r owner is 0 and got_unown, then ckpt owner 0 and all plocks;
+ (there should be no SYNCING plocks) */
+
+ list_for_each_entry(r, &ls->plock_resources, list) {
+ if (!cfgd_plock_ownership)
+ owner = 0;
+ else if (r->owner == -1)
+ continue;
+ else if (r->owner == our_nodeid)
+ owner = our_nodeid;
+ else if (r->owner)
+ owner = r->owner;
+ else if (!r->owner && !got_unown(r))
+ owner = our_nodeid;
+ else if (!r->owner)
+ owner = 0;
+ else {
+ log_elock(ls, "send_all_plocks_data error owner %d r %llx",
+ r->owner, (unsigned long long)r->number);
+ continue;
+ }
+
+ memset(&send_buf, 0, sizeof(send_buf));
+ count = 0;
+ full = 0;
+ last = NULL;
+
+ do {
+ full = pack_send_buf(ls, r, owner, full, &count, &last);
+
+ len = sizeof(struct dlm_header) +
+ sizeof(struct resource_data) +
+ sizeof(struct plock_data) * count;
+
+ log_plock(ls, "send_plocks_data %d:%u n %llu o %d locks %d len %d",
+ our_nodeid, seq, (unsigned long long)r->number, r->owner,
+ count, len);
+
+ send_plocks_data(ls, seq, send_buf, len);
+
+ send_count++;
+
+ } while (full);
+ }
+
+ *plocks_data = send_count;
+
+ log_dlock(ls, "send_all_plocks_data %d:%u %u done",
+ our_nodeid, seq, send_count);
+}
+
+static void free_r_lists(struct resource *r)
+{
+ struct posix_lock *po, *po2;
+ struct lock_waiter *w, *w2;
+
+ list_for_each_entry_safe(po, po2, &r->locks, list) {
+ list_del(&po->list);
+ free(po);
+ }
+
+ list_for_each_entry_safe(w, w2, &r->waiters, list) {
+ list_del(&w->list);
+ free(w);
+ }
+}
+
+void receive_plocks_data(struct lockspace *ls, struct dlm_header *hd, int len)
+{
+ struct resource_data *rd;
+ struct plock_data *pp;
+ struct posix_lock *po;
+ struct lock_waiter *w;
+ struct resource *r;
+ uint64_t num;
+ uint32_t count;
+ uint32_t flags;
+ int owner;
+ int i;
+
+ if (!cfgd_enable_plock || ls->disable_plock)
+ return;
+
+ if (!ls->need_plocks)
+ return;
+
+ if (!ls->save_plocks)
+ return;
+
+ ls->recv_plocks_data_count++;
+
+ if (len < sizeof(struct dlm_header) + sizeof(struct resource_data)) {
+ log_elock(ls, "recv_plocks_data %d:%u bad len %d",
+ hd->nodeid, hd->msgdata, len);
+ return;
+ }
+
+ rd = (struct resource_data *)((char *)hd + sizeof(struct dlm_header));
+ num = le64_to_cpu(rd->number);
+ owner = le32_to_cpu(rd->owner);
+ count = le32_to_cpu(rd->lock_count);
+ flags = le32_to_cpu(rd->flags);
+
+ if (flags & RD_CONTINUE) {
+ r = search_resource(ls, num);
+ if (!r) {
+ log_elock(ls, "recv_plocks_data %d:%u n %llu not found",
+ hd->nodeid, hd->msgdata, (unsigned long long)num);
+ return;
+ }
+ log_plock(ls, "recv_plocks_data %d:%u n %llu continue",
+ hd->nodeid, hd->msgdata, (unsigned long long)num);
+ goto unpack;
+ }
+
+ r = malloc(sizeof(struct resource));
+ if (!r) {
+ log_elock(ls, "recv_plocks_data %d:%u n %llu no mem",
+ hd->nodeid, hd->msgdata, (unsigned long long)num);
+ return;
+ }
+ memset(r, 0, sizeof(struct resource));
+ INIT_LIST_HEAD(&r->locks);
+ INIT_LIST_HEAD(&r->waiters);
+ INIT_LIST_HEAD(&r->pending);
+
+ if (!cfgd_plock_ownership) {
+ if (owner) {
+ log_elock(ls, "recv_plocks_data %d:%u n %llu bad owner %d",
+ hd->nodeid, hd->msgdata, (unsigned long long)num,
+ owner);
+ goto fail_free;
+ }
+ } else {
+ if (!owner)
+ r->flags |= R_GOT_UNOWN;
+
+ /* no locks should be included for owned resources */
+
+ if (owner && count) {
+ log_elock(ls, "recv_plocks_data %d:%u n %llu o %d bad count %u",
+ (unsigned long long)num, owner, count);
+ goto fail_free;
+ }
+ }
+
+ r->number = num;
+ r->owner = owner;
+
+ unpack:
+ if (len < sizeof(struct dlm_header) +
+ sizeof(struct resource_data) +
+ sizeof(struct plock_data) * count) {
+ log_elock(ls, "recv_plocks_data %d:%u count %u bad len %d",
+ hd->nodeid, hd->msgdata, count, len);
+ goto fail_free;
+ }
+
+ pp = (struct plock_data *)((char *)rd + sizeof(struct resource_data));
+
+ for (i = 0; i < count; i++) {
+ if (!pp->waiter) {
+ po = malloc(sizeof(struct posix_lock));
+ if (!po)
+ goto fail_free;
+ po->start = le64_to_cpu(pp->start);
+ po->end = le64_to_cpu(pp->end);
+ po->owner = le64_to_cpu(pp->owner);
+ po->pid = le32_to_cpu(pp->pid);
+ po->nodeid = le32_to_cpu(pp->nodeid);
+ po->ex = pp->ex;
+ list_add_tail(&po->list, &r->locks);
+ } else {
+ w = malloc(sizeof(struct lock_waiter));
+ if (!w)
+ goto fail_free;
+ w->info.start = le64_to_cpu(pp->start);
+ w->info.end = le64_to_cpu(pp->end);
+ w->info.owner = le64_to_cpu(pp->owner);
+ w->info.pid = le32_to_cpu(pp->pid);
+ w->info.nodeid = le32_to_cpu(pp->nodeid);
+ w->info.ex = pp->ex;
+ list_add_tail(&w->list, &r->waiters);
+ }
+ pp++;
+ }
+
+ log_plock(ls, "recv_plocks_data %d:%u n %llu o %d locks %d len %d",
+ hd->nodeid, hd->msgdata, (unsigned long long)r->number,
+ r->owner, count, len);
+
+ if (!(flags & RD_CONTINUE)) {
+ list_add_tail(&r->list, &ls->plock_resources);
+ rb_insert_plock_resource(ls, r);
+ }
+ return;
+
+ fail_free:
+ if (!(flags & RD_CONTINUE)) {
+ free_r_lists(r);
+ free(r);
+ }
+ return;
+}
+
+void clear_plocks_data(struct lockspace *ls)
+{
+ struct resource *r, *r2;
+ uint32_t count = 0;
+
+ if (!cfgd_enable_plock || ls->disable_plock)
+ return;
+
+ list_for_each_entry_safe(r, r2, &ls->plock_resources, list) {
+ free_r_lists(r);
+ rb_del_plock_resource(ls, r);
+ list_del(&r->list);
+ free(r);
+ count++;
+ }
+
+ log_dlock(ls, "clear_plocks_data done %u recv_plocks_data_count %u",
+ count, ls->recv_plocks_data_count);
+
+ ls->recv_plocks_data_count = 0;
+}
+
+/* Called when a node has failed, or we're unmounting. For a node failure, we
+ need to call this when the cpg confchg arrives so that we're guaranteed all
+ nodes do this in the same sequence wrt other messages. */
+
+void purge_plocks(struct lockspace *ls, int nodeid, int unmount)
+{
+ struct posix_lock *po, *po2;
+ struct lock_waiter *w, *w2;
+ struct resource *r, *r2;
+ int purged = 0;
+
+ if (!cfgd_enable_plock || ls->disable_plock)
+ return;
+
+ list_for_each_entry_safe(r, r2, &ls->plock_resources, list) {
+ list_for_each_entry_safe(po, po2, &r->locks, list) {
+ if (po->nodeid == nodeid || unmount) {
+ list_del(&po->list);
+ free(po);
+ purged++;
+ }
+ }
+
+ list_for_each_entry_safe(w, w2, &r->waiters, list) {
+ if (w->info.nodeid == nodeid || unmount) {
+ list_del(&w->list);
+ free(w);
+ purged++;
+ }
+ }
+
+ /* TODO: haven't thought carefully about how this transition
+ to owner 0 might interact with other owner messages in
+ progress. */
+
+ if (r->owner == nodeid) {
+ r->owner = 0;
+ r->flags |= R_GOT_UNOWN;
+ r->flags |= R_PURGE_UNOWN;
+ send_pending_plocks(ls, r);
+ }
+
+ if (!list_empty(&r->waiters))
+ do_waiters(ls, r);
+
+ if (!cfgd_plock_ownership &&
+ list_empty(&r->locks) && list_empty(&r->waiters)) {
+ rb_del_plock_resource(ls, r);
+ list_del(&r->list);
+ free(r);
+ }
+ }
+
+ if (purged)
+ ls->last_plock_time = time(NULL);
+
+ log_dlock(ls, "purged %d plocks for %d", purged, nodeid);
+}
+
+int copy_plock_state(struct lockspace *ls, char *buf, int *len_out)
+{
+ struct posix_lock *po;
+ struct lock_waiter *w;
+ struct resource *r;
+ struct timeval now;
+ int rv = 0;
+ int len = DLMC_DUMP_SIZE, pos = 0, ret;
+
+ gettimeofday(&now, NULL);
+
+ list_for_each_entry(r, &ls->plock_resources, list) {
+
+ if (list_empty(&r->locks) &&
+ list_empty(&r->waiters) &&
+ list_empty(&r->pending)) {
+ ret = snprintf(buf + pos, len - pos,
+ "%llu rown %d unused_ms %llu\n",
+ (unsigned long long)r->number, r->owner,
+ (unsigned long long)time_diff_ms(&r->last_access,
+ &now));
+ if (ret >= len - pos) {
+ rv = -ENOSPC;
+ goto out;
+ }
+ pos += ret;
+ continue;
+ }
+
+ list_for_each_entry(po, &r->locks, list) {
+ ret = snprintf(buf + pos, len - pos,
+ "%llu %s %llu-%llu nodeid %d pid %u owner %llx rown %d\n",
+ (unsigned long long)r->number,
+ po->ex ? "WR" : "RD",
+ (unsigned long long)po->start,
+ (unsigned long long)po->end,
+ po->nodeid, po->pid,
+ (unsigned long long)po->owner, r->owner);
+
+ if (ret >= len - pos) {
+ rv = -ENOSPC;
+ goto out;
+ }
+ pos += ret;
+ }
+
+ list_for_each_entry(w, &r->waiters, list) {
+ ret = snprintf(buf + pos, len - pos,
+ "%llu %s %llu-%llu nodeid %d pid %u owner %llx rown %d WAITING\n",
+ (unsigned long long)r->number,
+ w->info.ex ? "WR" : "RD",
+ (unsigned long long)w->info.start,
+ (unsigned long long)w->info.end,
+ w->info.nodeid, w->info.pid,
+ (unsigned long long)w->info.owner, r->owner);
+
+ if (ret >= len - pos) {
+ rv = -ENOSPC;
+ goto out;
+ }
+ pos += ret;
+ }
+
+ list_for_each_entry(w, &r->pending, list) {
+ ret = snprintf(buf + pos, len - pos,
+ "%llu %s %llu-%llu nodeid %d pid %u owner %llx rown %d PENDING\n",
+ (unsigned long long)r->number,
+ w->info.ex ? "WR" : "RD",
+ (unsigned long long)w->info.start,
+ (unsigned long long)w->info.end,
+ w->info.nodeid, w->info.pid,
+ (unsigned long long)w->info.owner, r->owner);
+
+ if (ret >= len - pos) {
+ rv = -ENOSPC;
+ goto out;
+ }
+ pos += ret;
+ }
+ }
+ out:
+ *len_out = pos;
+ return rv;
+}
+
diff --git a/dlm_controld/rbtree.c b/dlm_controld/rbtree.c
new file mode 100644
index 0000000..430ccc1
--- /dev/null
+++ b/dlm_controld/rbtree.c
@@ -0,0 +1,383 @@
+/*
+ Red Black Trees
+ (C) 1999 Andrea Arcangeli <andrea(a)suse.de>
+ (C) 2002 David Woodhouse <dwmw2(a)infradead.org>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+ linux/lib/rbtree.c
+*/
+
+#include "rbtree.h"
+
+static void __rb_rotate_left(struct rb_node *node, struct rb_root *root)
+{
+ struct rb_node *right = node->rb_right;
+ struct rb_node *parent = rb_parent(node);
+
+ if ((node->rb_right = right->rb_left))
+ rb_set_parent(right->rb_left, node);
+ right->rb_left = node;
+
+ rb_set_parent(right, parent);
+
+ if (parent)
+ {
+ if (node == parent->rb_left)
+ parent->rb_left = right;
+ else
+ parent->rb_right = right;
+ }
+ else
+ root->rb_node = right;
+ rb_set_parent(node, right);
+}
+
+static void __rb_rotate_right(struct rb_node *node, struct rb_root *root)
+{
+ struct rb_node *left = node->rb_left;
+ struct rb_node *parent = rb_parent(node);
+
+ if ((node->rb_left = left->rb_right))
+ rb_set_parent(left->rb_right, node);
+ left->rb_right = node;
+
+ rb_set_parent(left, parent);
+
+ if (parent)
+ {
+ if (node == parent->rb_right)
+ parent->rb_right = left;
+ else
+ parent->rb_left = left;
+ }
+ else
+ root->rb_node = left;
+ rb_set_parent(node, left);
+}
+
+void rb_insert_color(struct rb_node *node, struct rb_root *root)
+{
+ struct rb_node *parent, *gparent;
+
+ while ((parent = rb_parent(node)) && rb_is_red(parent))
+ {
+ gparent = rb_parent(parent);
+
+ if (parent == gparent->rb_left)
+ {
+ {
+ register struct rb_node *uncle = gparent->rb_right;
+ if (uncle && rb_is_red(uncle))
+ {
+ rb_set_black(uncle);
+ rb_set_black(parent);
+ rb_set_red(gparent);
+ node = gparent;
+ continue;
+ }
+ }
+
+ if (parent->rb_right == node)
+ {
+ register struct rb_node *tmp;
+ __rb_rotate_left(parent, root);
+ tmp = parent;
+ parent = node;
+ node = tmp;
+ }
+
+ rb_set_black(parent);
+ rb_set_red(gparent);
+ __rb_rotate_right(gparent, root);
+ } else {
+ {
+ register struct rb_node *uncle = gparent->rb_left;
+ if (uncle && rb_is_red(uncle))
+ {
+ rb_set_black(uncle);
+ rb_set_black(parent);
+ rb_set_red(gparent);
+ node = gparent;
+ continue;
+ }
+ }
+
+ if (parent->rb_left == node)
+ {
+ register struct rb_node *tmp;
+ __rb_rotate_right(parent, root);
+ tmp = parent;
+ parent = node;
+ node = tmp;
+ }
+
+ rb_set_black(parent);
+ rb_set_red(gparent);
+ __rb_rotate_left(gparent, root);
+ }
+ }
+
+ rb_set_black(root->rb_node);
+}
+
+static void __rb_erase_color(struct rb_node *node, struct rb_node *parent,
+ struct rb_root *root)
+{
+ struct rb_node *other;
+
+ while ((!node || rb_is_black(node)) && node != root->rb_node)
+ {
+ if (parent->rb_left == node)
+ {
+ other = parent->rb_right;
+ if (rb_is_red(other))
+ {
+ rb_set_black(other);
+ rb_set_red(parent);
+ __rb_rotate_left(parent, root);
+ other = parent->rb_right;
+ }
+ if ((!other->rb_left || rb_is_black(other->rb_left)) &&
+ (!other->rb_right || rb_is_black(other->rb_right)))
+ {
+ rb_set_red(other);
+ node = parent;
+ parent = rb_parent(node);
+ }
+ else
+ {
+ if (!other->rb_right || rb_is_black(other->rb_right))
+ {
+ rb_set_black(other->rb_left);
+ rb_set_red(other);
+ __rb_rotate_right(other, root);
+ other = parent->rb_right;
+ }
+ rb_set_color(other, rb_color(parent));
+ rb_set_black(parent);
+ rb_set_black(other->rb_right);
+ __rb_rotate_left(parent, root);
+ node = root->rb_node;
+ break;
+ }
+ }
+ else
+ {
+ other = parent->rb_left;
+ if (rb_is_red(other))
+ {
+ rb_set_black(other);
+ rb_set_red(parent);
+ __rb_rotate_right(parent, root);
+ other = parent->rb_left;
+ }
+ if ((!other->rb_left || rb_is_black(other->rb_left)) &&
+ (!other->rb_right || rb_is_black(other->rb_right)))
+ {
+ rb_set_red(other);
+ node = parent;
+ parent = rb_parent(node);
+ }
+ else
+ {
+ if (!other->rb_left || rb_is_black(other->rb_left))
+ {
+ rb_set_black(other->rb_right);
+ rb_set_red(other);
+ __rb_rotate_left(other, root);
+ other = parent->rb_left;
+ }
+ rb_set_color(other, rb_color(parent));
+ rb_set_black(parent);
+ rb_set_black(other->rb_left);
+ __rb_rotate_right(parent, root);
+ node = root->rb_node;
+ break;
+ }
+ }
+ }
+ if (node)
+ rb_set_black(node);
+}
+
+void rb_erase(struct rb_node *node, struct rb_root *root)
+{
+ struct rb_node *child, *parent;
+ int color;
+
+ if (!node->rb_left)
+ child = node->rb_right;
+ else if (!node->rb_right)
+ child = node->rb_left;
+ else
+ {
+ struct rb_node *old = node, *left;
+
+ node = node->rb_right;
+ while ((left = node->rb_left) != NULL)
+ node = left;
+
+ if (rb_parent(old)) {
+ if (rb_parent(old)->rb_left == old)
+ rb_parent(old)->rb_left = node;
+ else
+ rb_parent(old)->rb_right = node;
+ } else
+ root->rb_node = node;
+
+ child = node->rb_right;
+ parent = rb_parent(node);
+ color = rb_color(node);
+
+ if (parent == old) {
+ parent = node;
+ } else {
+ if (child)
+ rb_set_parent(child, parent);
+ parent->rb_left = child;
+
+ node->rb_right = old->rb_right;
+ rb_set_parent(old->rb_right, node);
+ }
+
+ node->rb_parent_color = old->rb_parent_color;
+ node->rb_left = old->rb_left;
+ rb_set_parent(old->rb_left, node);
+
+ goto color;
+ }
+
+ parent = rb_parent(node);
+ color = rb_color(node);
+
+ if (child)
+ rb_set_parent(child, parent);
+ if (parent)
+ {
+ if (parent->rb_left == node)
+ parent->rb_left = child;
+ else
+ parent->rb_right = child;
+ }
+ else
+ root->rb_node = child;
+
+ color:
+ if (color == RB_BLACK)
+ __rb_erase_color(child, parent, root);
+}
+
+/*
+ * This function returns the first node (in sort order) of the tree.
+ */
+struct rb_node *rb_first(const struct rb_root *root)
+{
+ struct rb_node *n;
+
+ n = root->rb_node;
+ if (!n)
+ return NULL;
+ while (n->rb_left)
+ n = n->rb_left;
+ return n;
+}
+
+struct rb_node *rb_last(const struct rb_root *root)
+{
+ struct rb_node *n;
+
+ n = root->rb_node;
+ if (!n)
+ return NULL;
+ while (n->rb_right)
+ n = n->rb_right;
+ return n;
+}
+
+struct rb_node *rb_next(const struct rb_node *node)
+{
+ struct rb_node *parent;
+
+ if (rb_parent(node) == node)
+ return NULL;
+
+ /* If we have a right-hand child, go down and then left as far
+ as we can. */
+ if (node->rb_right) {
+ node = node->rb_right;
+ while (node->rb_left)
+ node=node->rb_left;
+ return (struct rb_node *)node;
+ }
+
+ /* No right-hand children. Everything down and left is
+ smaller than us, so any 'next' node must be in the general
+ direction of our parent. Go up the tree; any time the
+ ancestor is a right-hand child of its parent, keep going
+ up. First time it's a left-hand child of its parent, said
+ parent is our 'next' node. */
+ while ((parent = rb_parent(node)) && node == parent->rb_right)
+ node = parent;
+
+ return parent;
+}
+
+struct rb_node *rb_prev(const struct rb_node *node)
+{
+ struct rb_node *parent;
+
+ if (rb_parent(node) == node)
+ return NULL;
+
+ /* If we have a left-hand child, go down and then right as far
+ as we can. */
+ if (node->rb_left) {
+ node = node->rb_left;
+ while (node->rb_right)
+ node=node->rb_right;
+ return (struct rb_node *)node;
+ }
+
+ /* No left-hand children. Go up till we find an ancestor which
+ is a right-hand child of its parent */
+ while ((parent = rb_parent(node)) && node == parent->rb_left)
+ node = parent;
+
+ return parent;
+}
+
+void rb_replace_node(struct rb_node *victim, struct rb_node *new,
+ struct rb_root *root)
+{
+ struct rb_node *parent = rb_parent(victim);
+
+ /* Set the surrounding nodes to point to the replacement */
+ if (parent) {
+ if (victim == parent->rb_left)
+ parent->rb_left = new;
+ else
+ parent->rb_right = new;
+ } else {
+ root->rb_node = new;
+ }
+ if (victim->rb_left)
+ rb_set_parent(victim->rb_left, new);
+ if (victim->rb_right)
+ rb_set_parent(victim->rb_right, new);
+
+ /* Copy the pointers/colour from the victim to the replacement */
+ *new = *victim;
+}
diff --git a/dlm_controld/rbtree.h b/dlm_controld/rbtree.h
new file mode 100644
index 0000000..e2b2409
--- /dev/null
+++ b/dlm_controld/rbtree.h
@@ -0,0 +1,160 @@
+/*
+ Red Black Trees
+ (C) 1999 Andrea Arcangeli <andrea(a)suse.de>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+ linux/include/linux/rbtree.h
+
+ To use rbtrees you'll have to implement your own insert and search cores.
+ This will avoid us to use callbacks and to drop drammatically performances.
+ I know it's not the cleaner way, but in C (not in C++) to get
+ performances and genericity...
+
+ Some example of insert and search follows here. The search is a plain
+ normal search over an ordered tree. The insert instead must be implemented
+ int two steps: as first thing the code must insert the element in
+ order as a red leaf in the tree, then the support library function
+ rb_insert_color() must be called. Such function will do the
+ not trivial work to rebalance the rbtree if necessary.
+
+-----------------------------------------------------------------------
+static inline struct page * rb_search_page_cache(struct inode * inode,
+ unsigned long offset)
+{
+ struct rb_node * n = inode->i_rb_page_cache.rb_node;
+ struct page * page;
+
+ while (n)
+ {
+ page = rb_entry(n, struct page, rb_page_cache);
+
+ if (offset < page->offset)
+ n = n->rb_left;
+ else if (offset > page->offset)
+ n = n->rb_right;
+ else
+ return page;
+ }
+ return NULL;
+}
+
+static inline struct page * __rb_insert_page_cache(struct inode * inode,
+ unsigned long offset,
+ struct rb_node * node)
+{
+ struct rb_node ** p = &inode->i_rb_page_cache.rb_node;
+ struct rb_node * parent = NULL;
+ struct page * page;
+
+ while (*p)
+ {
+ parent = *p;
+ page = rb_entry(parent, struct page, rb_page_cache);
+
+ if (offset < page->offset)
+ p = &(*p)->rb_left;
+ else if (offset > page->offset)
+ p = &(*p)->rb_right;
+ else
+ return page;
+ }
+
+ rb_link_node(node, parent, p);
+
+ return NULL;
+}
+
+static inline struct page * rb_insert_page_cache(struct inode * inode,
+ unsigned long offset,
+ struct rb_node * node)
+{
+ struct page * ret;
+ if ((ret = __rb_insert_page_cache(inode, offset, node)))
+ goto out;
+ rb_insert_color(node, &inode->i_rb_page_cache);
+ out:
+ return ret;
+}
+-----------------------------------------------------------------------
+*/
+
+#ifndef _LINUX_RBTREE_H
+#define _LINUX_RBTREE_H
+
+#include <linux/stddef.h>
+
+struct rb_node
+{
+ unsigned long rb_parent_color;
+#define RB_RED 0
+#define RB_BLACK 1
+ struct rb_node *rb_right;
+ struct rb_node *rb_left;
+} __attribute__((aligned(sizeof(long))));
+ /* The alignment might seem pointless, but allegedly CRIS needs it */
+
+struct rb_root
+{
+ struct rb_node *rb_node;
+};
+
+
+#define rb_parent(r) ((struct rb_node *)((r)->rb_parent_color & ~3))
+#define rb_color(r) ((r)->rb_parent_color & 1)
+#define rb_is_red(r) (!rb_color(r))
+#define rb_is_black(r) rb_color(r)
+#define rb_set_red(r) do { (r)->rb_parent_color &= ~1; } while (0)
+#define rb_set_black(r) do { (r)->rb_parent_color |= 1; } while (0)
+
+static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p)
+{
+ rb->rb_parent_color = (rb->rb_parent_color & 3) | (unsigned long)p;
+}
+static inline void rb_set_color(struct rb_node *rb, int color)
+{
+ rb->rb_parent_color = (rb->rb_parent_color & ~1) | color;
+}
+
+#define RB_ROOT (struct rb_root) { NULL, }
+#define rb_entry(ptr, type, member) container_of(ptr, type, member)
+
+#define RB_EMPTY_ROOT(root) ((root)->rb_node == NULL)
+#define RB_EMPTY_NODE(node) (rb_parent(node) == node)
+#define RB_CLEAR_NODE(node) (rb_set_parent(node, node))
+
+extern void rb_insert_color(struct rb_node *, struct rb_root *);
+extern void rb_erase(struct rb_node *, struct rb_root *);
+
+/* Find logical next and previous nodes in a tree */
+extern struct rb_node *rb_next(const struct rb_node *);
+extern struct rb_node *rb_prev(const struct rb_node *);
+extern struct rb_node *rb_first(const struct rb_root *);
+extern struct rb_node *rb_last(const struct rb_root *);
+
+/* Fast replacement of a single node without remove/rebalance/add/rebalance */
+extern void rb_replace_node(struct rb_node *victim, struct rb_node *new,
+ struct rb_root *root);
+
+static inline void rb_link_node(struct rb_node * node, struct rb_node * parent,
+ struct rb_node ** rb_link)
+{
+ node->rb_parent_color = (unsigned long )parent;
+ node->rb_left = node->rb_right = NULL;
+
+ *rb_link = node;
+}
+
+#endif /* _LINUX_RBTREE_H */
diff --git a/group/dlm_controld/Makefile b/group/dlm_controld/Makefile
deleted file mode 100644
index 63dcb9b..0000000
--- a/group/dlm_controld/Makefile
+++ /dev/null
@@ -1,50 +0,0 @@
-TARGET = dlm_controld
-
-SOURCE = \
- action.c \
- cpg.c \
- crc.c \
- main.c \
- plock.c \
- config.c \
- member_cman.c \
- logging.c \
- rbtree.c
-
-CFLAGS += -D_GNU_SOURCE -g \
- -Wall \
- -Wformat \
- -Wformat-security \
- -Wmissing-prototypes \
- -Wnested-externs \
- -Wpointer-arith \
- -Wextra -Wshadow \
- -Wcast-align \
- -Wwrite-strings \
- -Waggregate-return \
- -Wstrict-prototypes \
- -Winline \
- -Wredundant-decls \
- -Wno-sign-compare \
- -Wno-unused-parameter \
- -Wp,-D_FORTIFY_SOURCE=2 \
- -fexceptions \
- -fasynchronous-unwind-tables \
- -fdiagnostics-show-option \
-
-CFLAGS += -fPIE -DPIE
-CFLAGS += `xml2-config --cflags`
-CFLAGS += -I../include -I../../dlm/libdlm -I../../dlm/libdlmcontrol
-
-LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
-LDFLAGS += `xml2-config --libs`
-LDFLAGS += -lpthread -llogthread -lcpg -lconfdb -lcfg -lquorum -lfenced
-
-all: $(TARGET)
-
-$(TARGET): $(SOURCE)
- $(CC) $(CFLAGS) $(LDFLAGS) $(SOURCE) -o $@ -L.
-
-clean:
- rm -f *.o *.so *.so.* $(TARGET)
-
diff --git a/group/dlm_controld/action.c b/group/dlm_controld/action.c
deleted file mode 100644
index e9148a1..0000000
--- a/group/dlm_controld/action.c
+++ /dev/null
@@ -1,1084 +0,0 @@
-#include "dlm_daemon.h"
-
-#include <corosync/corotypes.h>
-#include <corosync/confdb.h>
-
-static int dir_members[MAX_NODES];
-static int dir_members_count;
-static int comms_nodes[MAX_NODES];
-static int comms_nodes_count;
-static char mg_name[DLM_LOCKSPACE_LEN+1];
-
-#define DLM_SYSFS_DIR "/sys/kernel/dlm"
-#define CLUSTER_DIR "/sys/kernel/config/dlm/cluster"
-#define SPACES_DIR "/sys/kernel/config/dlm/cluster/spaces"
-#define COMMS_DIR "/sys/kernel/config/dlm/cluster/comms"
-
-static int detect_protocol(void)
-{
- confdb_handle_t handle;
- hdb_handle_t totem_handle;
- char key_value[256];
- size_t value_len;
- int rv, proto = -1;
- confdb_callbacks_t callbacks = {
- .confdb_key_change_notify_fn = NULL,
- .confdb_object_create_change_notify_fn = NULL,
- .confdb_object_delete_change_notify_fn = NULL
- };
-
- rv = confdb_initialize(&handle, &callbacks);
- if (rv != CS_OK) {
- log_error("confdb_initialize error %d", rv);
- return -1;
- }
-
- rv = confdb_object_find_start(handle, OBJECT_PARENT_HANDLE);
- if (rv != CS_OK) {
- log_error("confdb_object_find_start error %d", rv);
- goto out;
- }
-
- rv = confdb_object_find(handle, OBJECT_PARENT_HANDLE,
- "totem", strlen("totem"), &totem_handle);
- if (rv != CS_OK) {
- log_error("confdb_object_find error %d", rv);
- goto out;
- }
-
- rv = confdb_key_get(handle, totem_handle,
- "rrp_mode", strlen("rrp_mode"),
- key_value, &value_len);
- if (rv != CS_OK) {
- log_error("confdb_key_get error %d", rv);
- goto out;
- }
-
- key_value[value_len] = '\0';
- log_debug("totem/rrp_mode = '%s'", key_value);
-
- if (!strcmp(key_value, "none"))
- proto = PROTO_TCP;
- else
- proto = PROTO_SCTP;
- out:
- confdb_finalize(handle);
- return proto;
-}
-
-/* look for an id that matches in e.g. /sys/fs/gfs/bull\:x/lock_module/id
- and then extract the "x" as the name */
-
-static int get_mountgroup_name(uint32_t mg_id)
-{
- char path[PATH_MAX];
- char *fsname;
- const char *fsdir;
- DIR *d;
- FILE *file;
- struct dirent *de;
- uint32_t id;
- int retry_gfs2 = 1;
- int rv, error;
-
- fsdir = "/sys/fs/gfs";
- retry:
- rv = -1;
-
- d = opendir(fsdir);
- if (!d) {
- log_debug("%s: opendir failed: %d", path, errno);
- goto out;
- }
-
- while ((de = readdir(d))) {
- if (de->d_name[0] == '.')
- continue;
-
- id = 0;
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s/lock_module/id",
- fsdir, de->d_name);
-
- file = fopen(path, "r");
- if (!file) {
- log_error("can't open %s %d", path, errno);
- continue;
- }
-
- error = fscanf(file, "%u", &id);
- fclose(file);
-
- if (error != 1) {
- log_error("bad read %s %d", path, errno);
- continue;
- }
- if (id != mg_id) {
- log_debug("get_mountgroup_name skip %x %s",
- id, de->d_name);
- continue;
- }
-
- /* take the fsname out of clustername:fsname */
- fsname = strstr(de->d_name, ":");
- if (!fsname) {
- log_debug("get_mountgroup_name skip2 %x %s",
- id, de->d_name);
- continue;
- }
- fsname++;
-
- log_debug("get_mountgroup_name found %x %s %s",
- id, de->d_name, fsname);
- strncpy(mg_name, fsname, sizeof(mg_name));
- rv = 0;
- break;
- }
-
- closedir(d);
-
- out:
- if (rv && retry_gfs2) {
- retry_gfs2 = 0;
- fsdir = "/sys/fs/gfs2";
- goto retry;
- }
-
- return rv;
-}
-
-/* This is for the case where dlm_controld exits/fails, abandoning dlm
- lockspaces in the kernel, and then dlm_controld is restarted. When
- dlm_controld exits and abandons lockspaces, that node needs to be
- rebooted to clear the uncontrolled lockspaces from the kernel. */
-
-int check_uncontrolled_lockspaces(void)
-{
- DIR *d;
- struct dirent *de;
- int count = 0;
-
- d = opendir(DLM_SYSFS_DIR);
- if (!d)
- return 0;
-
- while ((de = readdir(d))) {
- if (de->d_name[0] == '.')
- continue;
-
- log_error("found uncontrolled lockspace %s", de->d_name);
- count++;
- }
- closedir(d);
-
- if (count) {
- kick_node_from_cluster(our_nodeid);
- return -1;
- }
- return 0;
-}
-
-/* find the mountgroup with "mg_id" in sysfs, get it's name, then look for
- the ls with with the same name in lockspaces list, return its id */
-
-void set_associated_id(uint32_t mg_id)
-{
- struct lockspace *ls;
- int rv;
-
- log_debug("set_associated_id mg_id %x %d", mg_id, mg_id);
-
- memset(&mg_name, 0, sizeof(mg_name));
-
- rv = get_mountgroup_name(mg_id);
- if (rv) {
- log_error("no mountgroup found with id %x", mg_id);
- return;
- }
-
- ls = find_ls(mg_name);
- if (!ls) {
- log_error("no lockspace found with name %s for mg_id %x",
- mg_name, mg_id);
- return;
- }
-
- log_debug("set_associated_id mg %x is ls %x", mg_id, ls->global_id);
-
- ls->associated_mg_id = mg_id;
-}
-
-static int do_sysfs(const char *name, const char *file, char *val)
-{
- char fname[512];
- int rv, fd;
-
- sprintf(fname, "%s/%s/%s", DLM_SYSFS_DIR, name, file);
-
- fd = open(fname, O_WRONLY);
- if (fd < 0) {
- log_error("open \"%s\" error %d %d", fname, fd, errno);
- return -1;
- }
-
- log_debug("write \"%s\" to \"%s\"", val, fname);
-
- rv = do_write(fd, val, strlen(val) + 1);
- close(fd);
- return rv;
-}
-
-int set_sysfs_control(char *name, int val)
-{
- char buf[32];
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%d", val);
-
- return do_sysfs(name, "control", buf);
-}
-
-int set_sysfs_event_done(char *name, int val)
-{
- char buf[32];
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%d", val);
-
- return do_sysfs(name, "event_done", buf);
-}
-
-int set_sysfs_id(char *name, uint32_t id)
-{
- char buf[32];
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%u", id);
-
- return do_sysfs(name, "id", buf);
-}
-
-static int update_dir_members(char *name)
-{
- char path[PATH_MAX];
- DIR *d;
- struct dirent *de;
- int i = 0;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s/nodes", SPACES_DIR, name);
-
- d = opendir(path);
- if (!d) {
- log_debug("%s: opendir failed: %d", path, errno);
- return -1;
- }
-
- memset(dir_members, 0, sizeof(dir_members));
- dir_members_count = 0;
-
- /* FIXME: we should probably read the nodeid in each dir instead */
-
- while ((de = readdir(d))) {
- if (de->d_name[0] == '.')
- continue;
- dir_members[i++] = atoi(de->d_name);
- log_debug("dir_member %d", dir_members[i-1]);
- }
- closedir(d);
-
- dir_members_count = i;
- return 0;
-}
-
-static int id_exists(int id, int count, int *array)
-{
- int i;
- for (i = 0; i < count; i++) {
- if (array[i] == id)
- return 1;
- }
- return 0;
-}
-
-static int create_path(const char *path)
-{
- mode_t old_umask;
- int rv;
-
- old_umask = umask(0022);
- rv = mkdir(path, 0777);
- if (rv < 0 && errno == EEXIST)
- rv = 0;
- if (rv < 0)
- log_error("%s: mkdir failed: %d", path, errno);
- umask(old_umask);
- return rv;
-}
-
-int path_exists(const char *path)
-{
- struct stat buf;
-
- if (stat(path, &buf) < 0) {
- if (errno != ENOENT)
- log_error("%s: stat failed: %d", path, errno);
- return 0;
- }
- return 1;
-}
-
-/* The "renew" nodes are those that have left and rejoined since the last
- call to set_members(). We rmdir/mkdir for these nodes so dlm-kernel
- can notice they've left and rejoined. */
-
-int set_configfs_members(char *name, int new_count, int *new_members,
- int renew_count, int *renew_members)
-{
- char path[PATH_MAX];
- char buf[32];
- int i, w, fd, rv, id, old_count, *old_members;
- int do_renew;
-
- /*
- * create lockspace dir if it doesn't exist yet
- */
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s", SPACES_DIR, name);
-
- if (!path_exists(path)) {
- if (create_path(path))
- return -1;
- }
-
- /*
- * remove/add lockspace members
- */
-
- rv = update_dir_members(name);
- if (rv)
- return rv;
-
- old_members = dir_members;
- old_count = dir_members_count;
-
- for (i = 0; i < old_count; i++) {
- id = old_members[i];
- if (id_exists(id, new_count, new_members))
- continue;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s/nodes/%d",
- SPACES_DIR, name, id);
-
- log_debug("set_members rmdir \"%s\"", path);
-
- rv = rmdir(path);
- if (rv) {
- log_error("%s: rmdir failed: %d", path, errno);
- goto out;
- }
- }
-
- /*
- * remove lockspace dir after we've removed all the nodes
- * (when we're shutting down and adding no new nodes)
- */
-
- if (!new_count) {
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s", SPACES_DIR, name);
-
- log_debug("set_members lockspace rmdir \"%s\"", path);
-
- rv = rmdir(path);
- if (rv)
- log_error("%s: rmdir failed: %d", path, errno);
- }
-
- for (i = 0; i < new_count; i++) {
- id = new_members[i];
-
- do_renew = 0;
-
- if (id_exists(id, renew_count, renew_members))
- do_renew = 1;
- else if (id_exists(id, old_count, old_members))
- continue;
-
- if (!is_cluster_member(id))
- update_cluster();
- /*
- * create node's dir
- */
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s/nodes/%d",
- SPACES_DIR, name, id);
-
- if (do_renew) {
- log_debug("set_members renew rmdir \"%s\"", path);
- rv = rmdir(path);
- if (rv) {
- log_error("%s: renew rmdir failed: %d",
- path, errno);
- goto out;
- }
- }
-
- log_debug("set_members mkdir \"%s\"", path);
-
- rv = create_path(path);
- if (rv)
- goto out;
-
- /*
- * set node's nodeid
- */
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s/nodes/%d/nodeid",
- SPACES_DIR, name, id);
-
- rv = fd = open(path, O_WRONLY);
- if (rv < 0) {
- log_error("%s: open failed: %d", path, errno);
- goto out;
- }
-
- memset(buf, 0, 32);
- snprintf(buf, 32, "%d", id);
-
- rv = do_write(fd, buf, strlen(buf));
- if (rv < 0) {
- log_error("%s: write failed: %d, %s", path, errno, buf);
- close(fd);
- goto out;
- }
- close(fd);
-
- /*
- * set node's weight
- */
-
- w = get_weight(id, name);
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s/nodes/%d/weight",
- SPACES_DIR, name, id);
-
- rv = fd = open(path, O_WRONLY);
- if (rv < 0) {
- log_error("%s: open failed: %d", path, errno);
- goto out;
- }
-
- memset(buf, 0, 32);
- snprintf(buf, 32, "%d", w);
-
- rv = do_write(fd, buf, strlen(buf));
- if (rv < 0) {
- log_error("%s: write failed: %d, %s", path, errno, buf);
- close(fd);
- goto out;
- }
- close(fd);
- }
-
- rv = 0;
- out:
- return rv;
-}
-
-#if 0
-char *str_ip(char *addr)
-{
- static char ip[256];
- struct sockaddr_in *sin = (struct sockaddr_in *) addr;
- memset(ip, 0, sizeof(ip));
- inet_ntop(AF_INET, &sin->sin_addr, ip, 256);
- return ip;
-}
-#endif
-
-static char *str_ip(char *addr)
-{
- static char str_ip_buf[INET6_ADDRSTRLEN];
- struct sockaddr_storage *ss = (struct sockaddr_storage *)addr;
- struct sockaddr_in *sin = (struct sockaddr_in *)addr;
- struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
- void *saddr;
-
- if (ss->ss_family == AF_INET6)
- saddr = &sin6->sin6_addr;
- else
- saddr = &sin->sin_addr;
-
- inet_ntop(ss->ss_family, saddr, str_ip_buf, sizeof(str_ip_buf));
- return str_ip_buf;
-}
-
-/* record the nodeids that are currently listed under
- config/dlm/cluster/comms/ so that we can remove all of them */
-
-static int update_comms_nodes(void)
-{
- char path[PATH_MAX];
- DIR *d;
- struct dirent *de;
- int i = 0;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, COMMS_DIR);
-
- d = opendir(path);
- if (!d) {
- log_debug("%s: opendir failed: %d", path, errno);
- return -1;
- }
-
- memset(comms_nodes, 0, sizeof(comms_nodes));
- comms_nodes_count = 0;
-
- while ((de = readdir(d))) {
- if (de->d_name[0] == '.')
- continue;
- comms_nodes[i++] = atoi(de->d_name);
- }
- closedir(d);
-
- comms_nodes_count = i;
- return 0;
-}
-
-/* clear out everything under config/dlm/cluster/comms/ */
-
-static void clear_configfs_comms(void)
-{
- char path[PATH_MAX];
- int i, rv;
-
- rv = update_comms_nodes();
- if (rv < 0)
- return;
-
- for (i = 0; i < comms_nodes_count; i++) {
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%d", COMMS_DIR, comms_nodes[i]);
-
- log_debug("clear_configfs_nodes rmdir \"%s\"", path);
-
- rv = rmdir(path);
- if (rv)
- log_error("%s: rmdir failed: %d", path, errno);
- }
-}
-
-static void clear_configfs_space_nodes(char *name)
-{
- char path[PATH_MAX];
- int i, rv;
-
- rv = update_dir_members(name);
- if (rv < 0)
- return;
-
- for (i = 0; i < dir_members_count; i++) {
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s/nodes/%d",
- SPACES_DIR, name, dir_members[i]);
-
- log_debug("clear_configfs_space_nodes rmdir \"%s\"", path);
-
- rv = rmdir(path);
- if (rv)
- log_error("%s: rmdir failed: %d", path, errno);
- }
-}
-
-/* clear out everything under config/dlm/cluster/spaces/ */
-
-static void clear_configfs_spaces(void)
-{
- char path[PATH_MAX];
- DIR *d;
- struct dirent *de;
- int rv;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s", SPACES_DIR);
-
- d = opendir(path);
- if (!d) {
- log_debug("%s: opendir failed: %d", path, errno);
- return;
- }
-
- while ((de = readdir(d))) {
- if (de->d_name[0] == '.')
- continue;
-
- clear_configfs_space_nodes(de->d_name);
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%s", SPACES_DIR, de->d_name);
-
- log_debug("clear_configfs_spaces rmdir \"%s\"", path);
-
- rv = rmdir(path);
- if (rv)
- log_error("%s: rmdir failed: %d", path, errno);
- }
- closedir(d);
-}
-
-static int add_configfs_base(void)
-{
- int rv = 0;
-
- if (!path_exists("/sys/kernel/config")) {
- log_error("No /sys/kernel/config, is configfs loaded?");
- return -1;
- }
-
- if (!path_exists("/sys/kernel/config/dlm")) {
- log_error("No /sys/kernel/config/dlm, is the dlm loaded?");
- return -1;
- }
-
- if (!path_exists("/sys/kernel/config/dlm/cluster"))
- rv = create_path("/sys/kernel/config/dlm/cluster");
-
- return rv;
-}
-
-int add_configfs_node(int nodeid, char *addr, int addrlen, int local)
-{
- char path[PATH_MAX];
- char padded_addr[sizeof(struct sockaddr_storage)];
- char buf[32];
- int rv, fd;
-
- log_debug("set_configfs_node %d %s local %d",
- nodeid, str_ip(addr), local);
-
- /*
- * create comm dir for this node
- */
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%d", COMMS_DIR, nodeid);
-
- rv = create_path(path);
- if (rv)
- return -1;
-
- /*
- * set the nodeid
- */
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%d/nodeid", COMMS_DIR, nodeid);
-
- fd = open(path, O_WRONLY);
- if (fd < 0) {
- log_error("%s: open failed: %d", path, errno);
- return -1;
- }
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%d", nodeid);
-
- rv = do_write(fd, buf, strlen(buf));
- if (rv < 0) {
- log_error("%s: write failed: %d, %s", path, errno, buf);
- close(fd);
- return -1;
- }
- close(fd);
-
- /*
- * set the address
- */
-
- memset(padded_addr, 0, sizeof(padded_addr));
- memcpy(padded_addr, addr, addrlen);
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%d/addr", COMMS_DIR, nodeid);
-
- fd = open(path, O_WRONLY);
- if (fd < 0) {
- log_error("%s: open failed: %d", path, errno);
- return -1;
- }
-
- rv = do_write(fd, padded_addr, sizeof(struct sockaddr_storage));
- if (rv < 0) {
- log_error("%s: write failed: %d %d", path, errno, rv);
- close(fd);
- return -1;
- }
- close(fd);
-
- /*
- * set local
- */
-
- if (!local)
- goto out;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%d/local", COMMS_DIR, nodeid);
-
- fd = open(path, O_WRONLY);
- if (fd < 0) {
- log_error("%s: open failed: %d", path, errno);
- return -1;
- }
-
- rv = do_write(fd, (void *)"1", strlen("1"));
- if (rv < 0) {
- log_error("%s: write failed: %d", path, errno);
- close(fd);
- return -1;
- }
- close(fd);
- out:
- return 0;
-}
-
-void del_configfs_node(int nodeid)
-{
- char path[PATH_MAX];
- int rv;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/%d", COMMS_DIR, nodeid);
-
- log_debug("del_configfs_node rmdir \"%s\"", path);
-
- rv = rmdir(path);
- if (rv)
- log_error("%s: rmdir failed: %d", path, errno);
-}
-
-static int set_configfs_protocol(int proto)
-{
- char path[PATH_MAX];
- char buf[32];
- int fd, rv;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/protocol", CLUSTER_DIR);
-
- fd = open(path, O_WRONLY);
- if (fd < 0) {
- log_error("%s: open failed: %d", path, errno);
- return fd;
- }
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%d", proto);
-
- rv = do_write(fd, buf, strlen(buf));
- if (rv < 0) {
- log_error("%s: write failed: %d", path, errno);
- return rv;
- }
- close(fd);
- log_debug("set protocol %d", proto);
- return 0;
-}
-
-static int set_configfs_timewarn(int cs)
-{
- char path[PATH_MAX];
- char buf[32];
- int fd, rv;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/timewarn_cs", CLUSTER_DIR);
-
- fd = open(path, O_WRONLY);
- if (fd < 0) {
- log_error("%s: open failed: %d", path, errno);
- return fd;
- }
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%d", cs);
-
- rv = do_write(fd, buf, strlen(buf));
- if (rv < 0) {
- log_error("%s: write failed: %d", path, errno);
- return rv;
- }
- close(fd);
- log_debug("set timewarn_cs %d", cs);
- return 0;
-}
-
-static int set_configfs_debug(int val)
-{
- char path[PATH_MAX];
- char buf[32];
- int fd, rv;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "%s/log_debug", CLUSTER_DIR);
-
- fd = open(path, O_WRONLY);
- if (fd < 0) {
- log_error("%s: open failed: %d", path, errno);
- return fd;
- }
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%d", val);
-
- rv = do_write(fd, buf, strlen(buf));
- if (rv < 0) {
- log_error("%s: write failed: %d", path, errno);
- return rv;
- }
- close(fd);
- log_debug("set log_debug %d", val);
- return 0;
-}
-
-#define NET_RMEM_DEFAULT 4194304
-#define NET_RMEM_MAX 4194304
-
-static int set_proc_rmem(void)
-{
- char path[PATH_MAX];
- char buf[32];
- int fd, rv;
-
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "/proc/sys/net/core/rmem_default");
-
- fd = open(path, O_RDWR);
- if (fd < 0) {
- log_error("%s: open failed: %d", path, errno);
- return fd;
- }
-
- memset(buf, 0, sizeof(buf));
-
- rv = read(fd, buf, sizeof(buf));
- if (rv < 0) {
- log_error("%s: read failed: %d", path, errno);
- close(fd);
- return rv;
- }
-
- if (atoi(buf) >= NET_RMEM_DEFAULT) {
- close(fd);
- goto next;
- }
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%d", NET_RMEM_DEFAULT);
-
- rv = do_write(fd, buf, strlen(buf));
- if (rv < 0) {
- log_error("%s: write failed: %d", path, errno);
- close(fd);
- return rv;
- }
-
- close(fd);
- log_debug("set %s %s", path, buf);
-
- next:
- memset(path, 0, PATH_MAX);
- snprintf(path, PATH_MAX, "/proc/sys/net/core/rmem_max");
-
- fd = open(path, O_RDWR);
- if (fd < 0) {
- log_error("%s: open failed: %d", path, errno);
- return fd;
- }
-
- memset(buf, 0, sizeof(buf));
-
- rv = read(fd, buf, sizeof(buf));
- if (rv < 0) {
- log_error("%s: read failed: %d", path, errno);
- close(fd);
- return rv;
- }
-
- if (atoi(buf) >= NET_RMEM_MAX) {
- close(fd);
- goto out;
- }
-
- memset(buf, 0, sizeof(buf));
- snprintf(buf, 32, "%d", NET_RMEM_MAX);
-
- rv = do_write(fd, buf, strlen(buf));
- if (rv < 0) {
- log_error("%s: write failed: %d", path, errno);
- close(fd);
- return rv;
- }
-
- close(fd);
- log_debug("set %s %s", path, buf);
- out:
- return 0;
-}
-
-void clear_configfs(void)
-{
- clear_configfs_comms();
- clear_configfs_spaces();
- rmdir("/sys/kernel/config/dlm/cluster");
-}
-
-int setup_configfs(void)
-{
- int rv;
-
- clear_configfs();
-
- rv = add_configfs_base();
- if (rv < 0)
- return rv;
-
- /* add configfs entries for existing nodes */
- update_cluster();
-
- /* the kernel has its own defaults for these values which we
- don't want to change unless these have been set; -1 means
- they have not been set on command line or config file */
-
- if (cfgk_debug != -1)
- set_configfs_debug(cfgk_debug);
- if (cfgk_timewarn != -1)
- set_configfs_timewarn(cfgk_timewarn);
-
- if (cfgk_protocol == PROTO_DETECT) {
- rv = detect_protocol();
- if (rv == PROTO_TCP || rv == PROTO_SCTP)
- cfgk_protocol = rv;
- }
-
- if (cfgk_protocol == PROTO_TCP || cfgk_protocol == PROTO_SCTP)
- set_configfs_protocol(cfgk_protocol);
-
- if (cfgk_protocol == PROTO_SCTP)
- set_proc_rmem();
-
- return 0;
-}
-
-static void find_minors(void)
-{
- FILE *fl;
- char name[256];
- uint32_t number;
- int found = 0;
- int c;
-
- control_minor = 0;
- monitor_minor = 0;
- plock_minor = 0;
- old_plock_minor = 0;
-
- if (!(fl = fopen("/proc/misc", "r"))) {
- log_error("/proc/misc fopen failed: %s", strerror(errno));
- return;
- }
-
- while (!feof(fl)) {
- if (fscanf(fl, "%d %255s\n", &number, &name[0]) == 2) {
-
- if (!strcmp(name, "dlm-control")) {
- control_minor = number;
- found++;
- } else if (!strcmp(name, "dlm-monitor")) {
- monitor_minor = number;
- found++;
- } else if (!strcmp(name, "dlm_plock")) {
- plock_minor = number;
- found++;
- } else if (!strcmp(name, "lock_dlm_plock")) {
- old_plock_minor = number;
- found++;
- }
-
- } else do {
- c = fgetc(fl);
- } while (c != EOF && c != '\n');
-
- if (found == 3)
- break;
- }
- fclose(fl);
-
- if (!found)
- log_error("Is dlm missing from kernel? No misc devices found.");
-}
-
-static int find_udev_device(const char *path, uint32_t minor)
-{
- struct stat st;
- int i;
-
- for (i = 0; i < 10; i++) {
- if (stat(path, &st) == 0 && minor(st.st_rdev) == minor)
- return 0;
- sleep(1);
- }
-
- log_error("cannot find device %s with minor %d", path, minor);
- return -1;
-}
-
-int setup_misc_devices(void)
-{
- int rv;
-
- find_minors();
-
- if (control_minor) {
- rv = find_udev_device("/dev/misc/dlm-control", control_minor);
- if (rv < 0)
- return rv;
- log_debug("found /dev/misc/dlm-control minor %u",
- control_minor);
- }
-
- if (monitor_minor) {
- rv = find_udev_device("/dev/misc/dlm-monitor", monitor_minor);
- if (rv < 0)
- return rv;
- log_debug("found /dev/misc/dlm-monitor minor %u",
- monitor_minor);
- }
-
- if (plock_minor) {
- rv = find_udev_device("/dev/misc/dlm_plock", plock_minor);
- if (rv < 0)
- return rv;
- log_debug("found /dev/misc/dlm_plock minor %u",
- plock_minor);
- }
-
- if (!plock_minor && old_plock_minor) {
- rv = find_udev_device("/dev/misc/lock_dlm_plock",
- old_plock_minor);
- if (rv < 0)
- return rv;
- log_debug("found /dev/misc/lock_dlm_plock minor %u",
- old_plock_minor);
- }
-
- return 0;
-}
-
diff --git a/group/dlm_controld/config.c b/group/dlm_controld/config.c
deleted file mode 100644
index 40b0c87..0000000
--- a/group/dlm_controld/config.c
+++ /dev/null
@@ -1,93 +0,0 @@
-#include "dlm_daemon.h"
-#include <libxml/tree.h>
-
-/* TODO:
- <dlm>
- <lockspace name="foo" nodir="1">
- <master nodeid="1" weight="2"/>
- <master nodeid="2" weight="3"/>
- </lockspace>
- </dlm>
-*/
-
-int get_weight(int nodeid, char *lockspace)
-{
- /* default weight is 1 */
- return 1;
-}
-
-static void proto_val(char *str, int *val)
-{
- if (!strncasecmp(str, "tcp", 3))
- *val = PROTO_TCP;
- else if (!strncasecmp(str, "sctp", 4))
- *val = PROTO_SCTP;
- else if (!strncasecmp(str, "detect", 6))
- *val = PROTO_DETECT;
- else {
- log_error("invalid protocol value %s", str);
- }
-}
-
-static void set_val(xmlNode *root, const char *name, int *opt, int *val)
-{
- xmlChar *str;
-
- str = xmlGetProp(root, BAD_CAST name);
- if (str && !(*opt)) {
- *val = atoi((char *)str);
- log_debug("config %s = %d", name, *val);
- }
-}
-
-void setup_config(int update)
-{
- xmlDoc *doc;
- xmlNode *root;
- xmlChar *str;
-
- if (!path_exists(DLM_CONFIG_FILE))
- return;
-
- doc = xmlParseFile(DLM_CONFIG_FILE);
- if (!doc) {
- log_error("xml parse error %d %s", errno, DLM_CONFIG_FILE);
- return;
- }
-
- root = xmlDocGetRootElement(doc);
- if (!root) {
- log_error("xml root error %d %s", errno, DLM_CONFIG_FILE);
- xmlFreeDoc(doc);
- return;
- }
-
- if (update)
- goto do_update;
-
- /* These config values are set from dlm.conf only if they haven't
- already been set on the command line. */
-
- str = xmlGetProp(root, BAD_CAST "protocol");
- if (str && !optk_protocol) {
- proto_val((char *)str, &cfgk_protocol);
- log_debug("config protocol = %d", cfgk_protocol);
- }
-
- set_val(root, "log_debug", &optk_debug, &cfgk_debug);
- set_val(root, "timewarn", &optk_timewarn, &cfgk_timewarn);
- set_val(root, "enable_fencing", &optd_enable_fencing,
&cfgd_enable_fencing);
- set_val(root, "enable_quorum", &optd_enable_quorum,
&cfgd_enable_quorum);
- set_val(root, "enable_plock", &optd_enable_plock,
&cfgd_enable_plock);
- set_val(root, "plock_ownership", &optd_plock_ownership,
&cfgd_plock_ownership);
- do_update:
- /* The following can be changed while running */
- set_val(root, "plock_debug", &optd_plock_debug, &cfgd_plock_debug);
- set_val(root, "plock_rate_limit", &optd_plock_rate_limit,
&cfgd_plock_rate_limit);
- set_val(root, "drop_resources_time", &optd_drop_resources_time,
&cfgd_drop_resources_time);
- set_val(root, "drop_resources_count", &optd_drop_resources_count,
&cfgd_drop_resources_count);
- set_val(root, "drop_resources_age", &optd_drop_resources_age,
&cfgd_drop_resources_age);
-
- xmlFreeDoc(doc);
-}
-
diff --git a/group/dlm_controld/config.h b/group/dlm_controld/config.h
deleted file mode 100644
index ab37eed..0000000
--- a/group/dlm_controld/config.h
+++ /dev/null
@@ -1,47 +0,0 @@
-#ifndef __CONFIG_DOT_H__
-#define __CONFIG_DOT_H__
-
-/* the kernel has default values for debug, timewarn and protocol;
- we only change them if new values are given on command line or in ccs */
-
-#define DEFAULT_DEBUG_LOGFILE 0
-#define DEFAULT_ENABLE_FENCING 1
-#define DEFAULT_ENABLE_QUORUM 0
-#define DEFAULT_ENABLE_PLOCK 1
-#define DEFAULT_PLOCK_DEBUG 0
-#define DEFAULT_PLOCK_RATE_LIMIT 0
-#define DEFAULT_PLOCK_OWNERSHIP 0
-#define DEFAULT_DROP_RESOURCES_TIME 10000 /* 10 sec */
-#define DEFAULT_DROP_RESOURCES_COUNT 10
-#define DEFAULT_DROP_RESOURCES_AGE 10000 /* 10 sec */
-
-extern int optk_debug;
-extern int optk_timewarn;
-extern int optk_protocol;
-extern int optd_debug_logfile;
-extern int optd_enable_fencing;
-extern int optd_enable_quorum;
-extern int optd_enable_plock;
-extern int optd_plock_debug;
-extern int optd_plock_rate_limit;
-extern int optd_plock_ownership;
-extern int optd_drop_resources_time;
-extern int optd_drop_resources_count;
-extern int optd_drop_resources_age;
-
-extern int cfgk_debug;
-extern int cfgk_timewarn;
-extern int cfgk_protocol;
-extern int cfgd_debug_logfile;
-extern int cfgd_enable_fencing;
-extern int cfgd_enable_quorum;
-extern int cfgd_enable_plock;
-extern int cfgd_plock_debug;
-extern int cfgd_plock_rate_limit;
-extern int cfgd_plock_ownership;
-extern int cfgd_drop_resources_time;
-extern int cfgd_drop_resources_count;
-extern int cfgd_drop_resources_age;
-
-#endif
-
diff --git a/group/dlm_controld/cpg.c b/group/dlm_controld/cpg.c
deleted file mode 100644
index b3c1454..0000000
--- a/group/dlm_controld/cpg.c
+++ /dev/null
@@ -1,2606 +0,0 @@
-#include "dlm_daemon.h"
-
-struct protocol_version {
- uint16_t major;
- uint16_t minor;
- uint16_t patch;
- uint16_t flags;
-};
-
-struct protocol {
- union {
- struct protocol_version dm_ver;
- uint16_t daemon_max[4];
- };
- union {
- struct protocol_version km_ver;
- uint16_t kernel_max[4];
- };
- union {
- struct protocol_version dr_ver;
- uint16_t daemon_run[4];
- };
- union {
- struct protocol_version kr_ver;
- uint16_t kernel_run[4];
- };
-};
-
-struct member {
- struct list_head list;
- int nodeid;
- int start; /* 1 if we received a start message for this change */
- int added; /* 1 if added by this change */
- int failed; /* 1 if failed in this change */
- int disallowed;
- uint32_t start_flags;
-};
-
-struct node {
- struct list_head list;
- int nodeid;
- int check_fencing;
- int check_quorum;
- int check_fs;
- int fs_notified;
- uint64_t add_time;
- uint64_t fail_time;
- uint64_t fence_time; /* for debug */
- uint64_t cluster_add_time;
- uint64_t cluster_remove_time;
- uint32_t fence_queries; /* for debug */
- uint32_t added_seq; /* for queries */
- uint32_t removed_seq; /* for queries */
- int failed_reason; /* for queries */
-
- struct protocol proto;
-};
-
-/* One of these change structs is created for every confchg a cpg gets. */
-
-#define CGST_WAIT_CONDITIONS 1
-#define CGST_WAIT_MESSAGES 2
-
-struct change {
- struct list_head list;
- struct list_head members;
- struct list_head removed; /* nodes removed by this change */
- int member_count;
- int joined_count;
- int remove_count;
- int failed_count;
- int state;
- int we_joined;
- uint32_t seq; /* used as a reference for debugging, and for queries */
- uint32_t combined_seq; /* for queries */
- uint64_t create_time;
-};
-
-struct ls_info {
- uint32_t ls_info_size;
- uint32_t id_info_size;
- uint32_t id_info_count;
-
- uint32_t started_count;
-
- int member_count;
- int joined_count;
- int remove_count;
- int failed_count;
-};
-
-struct id_info {
- int nodeid;
-};
-
-int message_flow_control_on;
-static cpg_handle_t cpg_handle_daemon;
-static int cpg_fd_daemon;
-static struct protocol our_protocol;
-static struct list_head daemon_nodes;
-static struct cpg_address daemon_member[MAX_NODES];
-static int daemon_member_count;
-
-static void log_config(const struct cpg_name *group_name,
- const struct cpg_address *member_list,
- size_t member_list_entries,
- const struct cpg_address *left_list,
- size_t left_list_entries,
- const struct cpg_address *joined_list,
- size_t joined_list_entries)
-{
- char m_buf[128];
- char j_buf[32];
- char l_buf[32];
- size_t i, len, pos;
- int ret;
-
- memset(m_buf, 0, sizeof(m_buf));
- memset(j_buf, 0, sizeof(j_buf));
- memset(l_buf, 0, sizeof(l_buf));
-
- len = sizeof(m_buf);
- pos = 0;
- for (i = 0; i < member_list_entries; i++) {
- ret = snprintf(m_buf + pos, len - pos, " %d",
- member_list[i].nodeid);
- if (ret >= len - pos)
- break;
- pos += ret;
- }
-
- len = sizeof(j_buf);
- pos = 0;
- for (i = 0; i < joined_list_entries; i++) {
- ret = snprintf(j_buf + pos, len - pos, " %d",
- joined_list[i].nodeid);
- if (ret >= len - pos)
- break;
- pos += ret;
- }
-
- len = sizeof(l_buf);
- pos = 0;
- for (i = 0; i < left_list_entries; i++) {
- ret = snprintf(l_buf + pos, len - pos, " %d",
- left_list[i].nodeid);
- if (ret >= len - pos)
- break;
- pos += ret;
- }
-
- log_debug("%s conf %zu %zu %zu memb%s join%s left%s", group_name->value,
- member_list_entries, joined_list_entries, left_list_entries,
- m_buf, j_buf, l_buf);
-}
-
-static void ls_info_in(struct ls_info *li)
-{
- li->ls_info_size = le32_to_cpu(li->ls_info_size);
- li->id_info_size = le32_to_cpu(li->id_info_size);
- li->id_info_count = le32_to_cpu(li->id_info_count);
- li->started_count = le32_to_cpu(li->started_count);
- li->member_count = le32_to_cpu(li->member_count);
- li->joined_count = le32_to_cpu(li->joined_count);
- li->remove_count = le32_to_cpu(li->remove_count);
- li->failed_count = le32_to_cpu(li->failed_count);
-}
-
-static void id_info_in(struct id_info *id)
-{
- id->nodeid = le32_to_cpu(id->nodeid);
-}
-
-static void ids_in(struct ls_info *li, struct id_info *ids)
-{
- struct id_info *id;
- int i;
-
- id = ids;
- for (i = 0; i < li->id_info_count; i++) {
- id_info_in(id);
- id = (struct id_info *)((char *)id + li->id_info_size);
- }
-}
-
-const char *msg_name(int type)
-{
- switch (type) {
- case DLM_MSG_PROTOCOL:
- return "protocol";
- case DLM_MSG_START:
- return "start";
- case DLM_MSG_PLOCK:
- return "plock";
- case DLM_MSG_PLOCK_OWN:
- return "plock_own";
- case DLM_MSG_PLOCK_DROP:
- return "plock_drop";
- case DLM_MSG_PLOCK_SYNC_LOCK:
- return "plock_sync_lock";
- case DLM_MSG_PLOCK_SYNC_WAITER:
- return "plock_sync_waiter";
- case DLM_MSG_PLOCKS_DATA:
- return "plocks_data";
- case DLM_MSG_PLOCKS_DONE:
- return "plocks_done";
- case DLM_MSG_DEADLK_CYCLE_START:
- return "deadlk_cycle_start";
- case DLM_MSG_DEADLK_CYCLE_END:
- return "deadlk_cycle_end";
- case DLM_MSG_DEADLK_CHECKPOINT_READY:
- return "deadlk_checkpoint_ready";
- case DLM_MSG_DEADLK_CANCEL_LOCK:
- return "deadlk_cancel_lock";
- default:
- return "unknown";
- }
-}
-
-static int _send_message(cpg_handle_t h, void *buf, int len, int type)
-{
- struct iovec iov;
- cpg_error_t error;
- int retries = 0;
-
- iov.iov_base = buf;
- iov.iov_len = len;
-
- retry:
- error = cpg_mcast_joined(h, CPG_TYPE_AGREED, &iov, 1);
- if (error == CPG_ERR_TRY_AGAIN) {
- retries++;
- usleep(1000);
- if (!(retries % 100))
- log_error("cpg_mcast_joined retry %d %s",
- retries, msg_name(type));
- goto retry;
- }
- if (error != CPG_OK) {
- log_error("cpg_mcast_joined error %d handle %llx %s",
- error, (unsigned long long)h, msg_name(type));
- return -1;
- }
-
- if (retries)
- log_debug("cpg_mcast_joined retried %d %s",
- retries, msg_name(type));
-
- return 0;
-}
-
-/* header fields caller needs to set: type, to_nodeid, flags, msgdata */
-
-void dlm_send_message(struct lockspace *ls, char *buf, int len)
-{
- struct dlm_header *hd = (struct dlm_header *) buf;
- int type = hd->type;
-
- hd->version[0] = cpu_to_le16(our_protocol.daemon_run[0]);
- hd->version[1] = cpu_to_le16(our_protocol.daemon_run[1]);
- hd->version[2] = cpu_to_le16(our_protocol.daemon_run[2]);
- hd->type = cpu_to_le16(hd->type);
- hd->nodeid = cpu_to_le32(our_nodeid);
- hd->to_nodeid = cpu_to_le32(hd->to_nodeid);
- hd->global_id = cpu_to_le32(ls->global_id);
- hd->flags = cpu_to_le32(hd->flags);
- hd->msgdata = cpu_to_le32(hd->msgdata);
- hd->msgdata2 = cpu_to_le32(hd->msgdata2);
-
- _send_message(ls->cpg_handle, buf, len, type);
-}
-
-static struct member *find_memb(struct change *cg, int nodeid)
-{
- struct member *memb;
-
- list_for_each_entry(memb, &cg->members, list) {
- if (memb->nodeid == nodeid)
- return memb;
- }
- return NULL;
-}
-
-static struct lockspace *find_ls_handle(cpg_handle_t h)
-{
- struct lockspace *ls;
-
- list_for_each_entry(ls, &lockspaces, list) {
- if (ls->cpg_handle == h)
- return ls;
- }
- return NULL;
-}
-
-static struct lockspace *find_ls_ci(int ci)
-{
- struct lockspace *ls;
-
- list_for_each_entry(ls, &lockspaces, list) {
- if (ls->cpg_client == ci)
- return ls;
- }
- return NULL;
-}
-
-static void free_cg(struct change *cg)
-{
- struct member *memb, *safe;
-
- list_for_each_entry_safe(memb, safe, &cg->members, list) {
- list_del(&memb->list);
- free(memb);
- }
- list_for_each_entry_safe(memb, safe, &cg->removed, list) {
- list_del(&memb->list);
- free(memb);
- }
- free(cg);
-}
-
-static void free_ls(struct lockspace *ls)
-{
- struct change *cg, *cg_safe;
- struct node *node, *node_safe;
-
- list_for_each_entry_safe(cg, cg_safe, &ls->changes, list) {
- list_del(&cg->list);
- free_cg(cg);
- }
-
- if (ls->started_change)
- free_cg(ls->started_change);
-
- list_for_each_entry_safe(node, node_safe, &ls->node_history, list) {
- list_del(&node->list);
- free(node);
- }
-
- free(ls);
-}
-
-
-/* Problem scenario:
- nodes A,B,C are in fence domain
- node C has gfs foo mounted
- node C fails
- nodes A,B begin fencing C (slow, not completed)
- node B mounts gfs foo
-
- We may end up having gfs foo mounted and being used on B before
- C has been fenced. C could wake up corrupt fs.
-
- So, we need to prevent any new gfs mounts while there are any
- outstanding, incomplete fencing operations.
-
- We also need to check that the specific failed nodes we know about have
- been fenced (since fenced may not even have been notified that the node
- has failed yet).
-
- So, check that:
- 1. has fenced fenced the node since we saw it fail?
- 2. fenced has no outstanding fencing ops
-
- For 1:
- - node X fails
- - we see node X fail and X has non-zero add_time,
- set check_fencing and record the fail time
- - wait for X to be removed from all dlm cpg's (probably not necessary)
- - check that the fencing time is later than the recorded time above
-
- Tracking fencing state when there are spurious partitions/merges...
-
- from a spurious leave/join of node X, a lockspace will see:
- - node X is a lockspace member
- - node X fails, may be waiting for all cpgs to see failure or for fencing to
- complete
- - node X joins the lockspace - we want to process the change as usual, but
- don't want to disrupt the code waiting for the fencing, and we want to
- continue running properly once the remerged node is properly reset
-
- ls->node_history
- when we see a node not in this list, add entry for it with zero add_time
- record the time we get a good start message from the node, add_time
- clear add_time if the node leaves
- if node fails with non-zero add_time, set check_fencing
- when a node is fenced, clear add_time and clear check_fencing
- if a node remerges after this, no good start message, no new add_time set
- if a node fails with zero add_time, it doesn't need fencing
- if a node remerges before it's been fenced, no good start message, no new
- add_time set
-*/
-
-static struct node *get_node_history(struct lockspace *ls, int nodeid)
-{
- struct node *node;
-
- list_for_each_entry(node, &ls->node_history, list) {
- if (node->nodeid == nodeid)
- return node;
- }
- return NULL;
-}
-
-static void node_history_init(struct lockspace *ls, int nodeid,
- struct change *cg)
-{
- struct node *node;
-
- node = get_node_history(ls, nodeid);
- if (node)
- goto out;
-
- node = malloc(sizeof(struct node));
- if (!node)
- return;
- memset(node, 0, sizeof(struct node));
-
- node->nodeid = nodeid;
- node->add_time = 0;
- list_add_tail(&node->list, &ls->node_history);
- out:
- if (cg)
- node->added_seq = cg->seq; /* for queries */
-}
-
-void node_history_cluster_add(int nodeid)
-{
- struct lockspace *ls;
- struct node *node;
-
- list_for_each_entry(ls, &lockspaces, list) {
- node_history_init(ls, nodeid, NULL);
-
- node = get_node_history(ls, nodeid);
- if (!node) {
- log_error("node_history_cluster_add no nodeid %d",
- nodeid);
- return;
- }
-
- node->cluster_add_time = time(NULL);
- }
-}
-
-void node_history_cluster_remove(int nodeid)
-{
- struct lockspace *ls;
- struct node *node;
-
- list_for_each_entry(ls, &lockspaces, list) {
- node = get_node_history(ls, nodeid);
- if (!node) {
- log_error("node_history_cluster_remove no nodeid %d",
- nodeid);
- return;
- }
-
- node->cluster_remove_time = time(NULL);
- }
-}
-
-static void node_history_start(struct lockspace *ls, int nodeid)
-{
- struct node *node;
-
- node = get_node_history(ls, nodeid);
- if (!node) {
- log_error("node_history_start no nodeid %d", nodeid);
- return;
- }
-
- node->add_time = time(NULL);
-}
-
-static void node_history_left(struct lockspace *ls, int nodeid,
- struct change *cg)
-{
- struct node *node;
-
- node = get_node_history(ls, nodeid);
- if (!node) {
- log_error("node_history_left no nodeid %d", nodeid);
- return;
- }
-
- node->add_time = 0;
- node->removed_seq = cg->seq; /* for queries */
-}
-
-static void node_history_fail(struct lockspace *ls, int nodeid,
- struct change *cg, int reason)
-{
- struct node *node;
-
- node = get_node_history(ls, nodeid);
- if (!node) {
- log_error("node_history_fail no nodeid %d", nodeid);
- return;
- }
-
- if (cfgd_enable_fencing && node->add_time) {
- node->check_fencing = 1;
- node->fence_time = 0;
- node->fence_queries = 0;
- node->fail_time = time(NULL);
- }
-
- /* fenced will take care of making sure the quorum value
- is adjusted for all the failures */
-
- if (cfgd_enable_quorum && !cfgd_enable_fencing)
- node->check_quorum = 1;
-
- if (ls->fs_registered) {
- log_group(ls, "check_fs nodeid %d set", nodeid);
- node->check_fs = 1;
- }
-
- node->removed_seq = cg->seq; /* for queries */
- node->failed_reason = reason; /* for queries */
-}
-
-static int check_fencing_done(struct lockspace *ls)
-{
- struct node *node;
- uint64_t last_fenced_time;
- int in_progress, wait_count = 0;
- int rv;
-
- if (!cfgd_enable_fencing) {
- log_group(ls, "check_fencing disabled");
- return 1;
- }
-
- list_for_each_entry(node, &ls->node_history, list) {
- if (!node->check_fencing)
- continue;
-
- /* check with fenced to see if the node has been
- fenced since node->add_time */
-
- rv = fence_node_time(node->nodeid, &last_fenced_time);
- if (rv < 0)
- log_error("fenced_node_info error %d", rv);
-
- /* need >= not just > because in at least one case
- we've seen fenced_time within the same second as
- fail_time: with external fencing, e.g. fence_node */
-
- if (last_fenced_time >= node->fail_time) {
- log_group(ls, "check_fencing %d done "
- "add %llu fail %llu last %llu",
- node->nodeid,
- (unsigned long long)node->add_time,
- (unsigned long long)node->fail_time,
- (unsigned long long)last_fenced_time);
- node->check_fencing = 0;
- node->add_time = 0;
- node->fence_time = last_fenced_time;
- } else {
- if (!node->fence_queries ||
- node->fence_time != last_fenced_time) {
- log_group(ls, "check_fencing %d wait "
- "add %llu fail %llu last %llu",
- node->nodeid,
- (unsigned long long)node->add_time,
- (unsigned long long)node->fail_time,
- (unsigned long long)last_fenced_time);
- node->fence_queries++;
- node->fence_time = last_fenced_time;
- }
- wait_count++;
- }
- }
-
- if (wait_count)
- return 0;
-
- /* now check if there are any outstanding fencing ops (for nodes
- we may not have seen in any lockspace), and return 0 if there
- are any */
-
- rv = fence_in_progress(&in_progress);
- if (rv < 0) {
- log_error("fenced_domain_info error %d", rv);
- return 0;
- }
-
- if (in_progress)
- return 0;
-
- log_group(ls, "check_fencing done");
- return 1;
-}
-
-static int check_quorum_done(struct lockspace *ls)
-{
- struct node *node;
- int wait_count = 0;
-
- if (!cfgd_enable_quorum) {
- log_group(ls, "check_quorum disabled");
- return 1;
- }
-
- /* wait for quorum system (cman) to see all the same nodes failed, so
- we know that cluster_quorate is adjusted for the same failures we've
- seen (see comment in fenced about the assumption here) */
-
- list_for_each_entry(node, &ls->node_history, list) {
- if (!node->check_quorum)
- continue;
-
- if (!is_cluster_member(node->nodeid)) {
- node->check_quorum = 0;
- } else {
- log_group(ls, "check_quorum nodeid %d is_cluster_member",
- node->nodeid);
- wait_count++;
- }
- }
-
- if (wait_count)
- return 0;
-
- if (!cluster_quorate) {
- log_group(ls, "check_quorum not quorate");
- return 0;
- }
-
- log_group(ls, "check_quorum done");
- return 1;
-}
-
-/* wait for local fs_controld to ack each failed node */
-
-static int check_fs_done(struct lockspace *ls)
-{
- struct node *node;
- int wait_count = 0;
-
- /* no corresponding fs for this lockspace */
- if (!ls->fs_registered) {
- log_group(ls, "check_fs none registered");
- return 1;
- }
-
- list_for_each_entry(node, &ls->node_history, list) {
- if (!node->check_fs)
- continue;
-
- if (node->fs_notified) {
- log_group(ls, "check_fs nodeid %d clear", node->nodeid);
- node->check_fs = 0;
- node->fs_notified = 0;
- } else {
- log_group(ls, "check_fs nodeid %d needs fs notify",
- node->nodeid);
- wait_count++;
- }
- }
-
- if (wait_count)
- return 0;
-
- log_group(ls, "check_fs done");
- return 1;
-}
-
-static int member_ids[MAX_NODES];
-static int member_count;
-static int renew_ids[MAX_NODES];
-static int renew_count;
-
-static void format_member_ids(struct lockspace *ls)
-{
- struct change *cg = list_first_entry(&ls->changes, struct change, list);
- struct member *memb;
-
- memset(member_ids, 0, sizeof(member_ids));
- member_count = 0;
-
- list_for_each_entry(memb, &cg->members, list)
- member_ids[member_count++] = memb->nodeid;
-}
-
-/* list of nodeids that have left and rejoined since last start_kernel;
- is any member of startcg in the left list of any other cg's?
- (if it is, then it presumably must be flagged added in another) */
-
-static void format_renew_ids(struct lockspace *ls)
-{
- struct change *cg, *startcg;
- struct member *memb, *leftmemb;
-
- startcg = list_first_entry(&ls->changes, struct change, list);
-
- memset(renew_ids, 0, sizeof(renew_ids));
- renew_count = 0;
-
- list_for_each_entry(memb, &startcg->members, list) {
- list_for_each_entry(cg, &ls->changes, list) {
- if (cg == startcg)
- continue;
- list_for_each_entry(leftmemb, &cg->removed, list) {
- if (memb->nodeid == leftmemb->nodeid) {
- renew_ids[renew_count++] = memb->nodeid;
- }
- }
- }
- }
-
-}
-
-static void start_kernel(struct lockspace *ls)
-{
- struct change *cg = list_first_entry(&ls->changes, struct change, list);
-
- if (!ls->kernel_stopped) {
- log_error("start_kernel cg %u not stopped", cg->seq);
- return;
- }
-
- log_group(ls, "start_kernel cg %u member_count %d",
- cg->seq, cg->member_count);
-
- /* needs to happen before setting control which starts recovery */
- if (ls->joining)
- set_sysfs_id(ls->name, ls->global_id);
-
- format_member_ids(ls);
- format_renew_ids(ls);
- set_configfs_members(ls->name, member_count, member_ids,
- renew_count, renew_ids);
- set_sysfs_control(ls->name, 1);
- ls->kernel_stopped = 0;
-
- if (ls->joining) {
- set_sysfs_event_done(ls->name, 0);
- ls->joining = 0;
- }
-}
-
-static void stop_kernel(struct lockspace *ls, uint32_t seq)
-{
- if (!ls->kernel_stopped) {
- log_group(ls, "stop_kernel cg %u", seq);
- set_sysfs_control(ls->name, 0);
- ls->kernel_stopped = 1;
- }
-}
-
-/* the first condition is that the local lockspace is stopped which we
- don't need to check for because stop_kernel(), which is synchronous,
- was done when the change was created */
-
-static int wait_conditions_done(struct lockspace *ls)
-{
- /* the fencing/quorum/fs conditions need to account for all the changes
- that have occured since the last change applied to dlm-kernel, not
- just the latest change */
-
- if (!check_fencing_done(ls)) {
- poll_fencing++;
- return 0;
- }
-
- /* fencing waits for quorum, so we don't need to check quorum for any
- reasons related to safety or protection, so enable_quorum defaults
- to 0. This does mean that lockspaces (and cluster fs's) can be
- started/enabled in an inquorate cluster if there are no outstanding
- fencing operations. Some users or apps may want lockspaces/fs's to
- only be enabled in a quorate cluster; enable_quorum can be set to 1
- to get that behavior. The main advantage of not waiting for quorum
- here is to allow lockspaces to be shut down (and cluster fs's
- unmounted) in an inquorate cluster. */
-
- if (!check_quorum_done(ls)) {
- poll_quorum++;
- return 0;
- }
-
- if (!check_fs_done(ls)) {
- poll_fs++;
- return 0;
- }
-
- return 1;
-}
-
-static int wait_messages_done(struct lockspace *ls)
-{
- struct change *cg = list_first_entry(&ls->changes, struct change, list);
- struct member *memb;
- int need = 0, total = 0;
-
- list_for_each_entry(memb, &cg->members, list) {
- if (!memb->start)
- need++;
- total++;
- }
-
- if (need) {
- log_group(ls, "wait_messages cg %u need %d of %d",
- cg->seq, need, total);
- return 0;
- }
-
- log_group(ls, "wait_messages cg %u got all %d", cg->seq, total);
- return 1;
-}
-
-static void cleanup_changes(struct lockspace *ls)
-{
- struct change *cg = list_first_entry(&ls->changes, struct change, list);
- struct change *safe;
-
- list_del(&cg->list);
- if (ls->started_change)
- free_cg(ls->started_change);
- ls->started_change = cg;
-
- ls->started_count++;
- if (!ls->started_count)
- ls->started_count++;
-
- cg->combined_seq = cg->seq; /* for queries */
-
- list_for_each_entry_safe(cg, safe, &ls->changes, list) {
- ls->started_change->combined_seq = cg->seq; /* for queries */
- list_del(&cg->list);
- free_cg(cg);
- }
-}
-
-/* There's a stream of confchg and messages. At one of these
- messages, the low node needs to store plocks and new nodes
- need to begin saving plock messages. A second message is
- needed to say that the plocks are ready to be read.
-
- When the last start message is recvd for a change, the low node
- stores plocks and the new nodes begin saving messages. When the
- store is done, low node sends plocks_stored message. When
- new nodes recv this, they read the plocks and their saved messages.
- plocks_stored message should identify a specific change, like start
- messages do; if it doesn't match ls->started_change, then it's ignored.
-
- If a confchg adding a new node arrives after plocks are stored but
- before plocks_stored msg recvd, then the message is ignored. The low
- node will send another plocks_stored message for the latest change
- (although it may be able to reuse the ckpt if no plock state has changed).
-*/
-
-static void set_plock_data_node(struct lockspace *ls)
-{
- struct change *cg = list_first_entry(&ls->changes, struct change, list);
- struct member *memb;
- int low = 0;
-
- list_for_each_entry(memb, &cg->members, list) {
- if (!(memb->start_flags & DLM_MFLG_HAVEPLOCK))
- continue;
-
- if (!low || memb->nodeid < low)
- low = memb->nodeid;
- }
-
- log_dlock(ls, "set_plock_data_node from %d to %d",
- ls->plock_data_node, low);
-
- ls->plock_data_node = low;
-}
-
-static struct id_info *get_id_struct(struct id_info *ids, int count, int size,
- int nodeid)
-{
- struct id_info *id = ids;
- int i;
-
- for (i = 0; i < count; i++) {
- if (id->nodeid == nodeid)
- return id;
- id = (struct id_info *)((char *)id + size);
- }
- return NULL;
-}
-
-/* do the change details in the message match the details of the given change */
-
-static int match_change(struct lockspace *ls, struct change *cg,
- struct dlm_header *hd, struct ls_info *li,
- struct id_info *ids)
-{
- struct id_info *id;
- struct member *memb;
- struct node *node;
- uint32_t seq = hd->msgdata;
- int i, members_mismatch;
-
- /* We can ignore messages if we're not in the list of members.
- The one known time this will happen is after we've joined
- the cpg, we can get messages for changes prior to the change
- in which we're added. */
-
- id = get_id_struct(ids, li->id_info_count, li->id_info_size,our_nodeid);
-
- if (!id) {
- log_group(ls, "match_change %d:%u skip %u we are not in members",
- hd->nodeid, seq, cg->seq);
- return 0;
- }
-
- memb = find_memb(cg, hd->nodeid);
- if (!memb) {
- log_group(ls, "match_change %d:%u skip %u sender not member",
- hd->nodeid, seq, cg->seq);
- return 0;
- }
-
- if (memb->start_flags & DLM_MFLG_NACK) {
- log_group(ls, "match_change %d:%u skip %u is nacked",
- hd->nodeid, seq, cg->seq);
- return 0;
- }
-
- if (memb->start && hd->type == DLM_MSG_START) {
- log_group(ls, "match_change %d:%u skip %u already start",
- hd->nodeid, seq, cg->seq);
- return 0;
- }
-
- /* a node's start can't match a change if the node joined the cluster
- more recently than the change was created */
-
- node = get_node_history(ls, hd->nodeid);
- if (!node) {
- log_group(ls, "match_change %d:%u skip cg %u no node history",
- hd->nodeid, seq, cg->seq);
- return 0;
- }
-
- if (node->cluster_add_time > cg->create_time) {
- log_group(ls, "match_change %d:%u skip cg %u created %llu "
- "cluster add %llu", hd->nodeid, seq, cg->seq,
- (unsigned long long)cg->create_time,
- (unsigned long long)node->cluster_add_time);
- return 0;
- }
-
- /* verify this is the right change by matching the counts
- and the nodeids of the current members */
-
- if (li->member_count != cg->member_count ||
- li->joined_count != cg->joined_count ||
- li->remove_count != cg->remove_count ||
- li->failed_count != cg->failed_count) {
- log_group(ls, "match_change %d:%u skip %u expect counts "
- "%d %d %d %d", hd->nodeid, seq, cg->seq,
- cg->member_count, cg->joined_count,
- cg->remove_count, cg->failed_count);
- return 0;
- }
-
- members_mismatch = 0;
- id = ids;
-
- for (i = 0; i < li->id_info_count; i++) {
- memb = find_memb(cg, id->nodeid);
- if (!memb) {
- log_group(ls, "match_change %d:%u skip %u no memb %d",
- hd->nodeid, seq, cg->seq, id->nodeid);
- members_mismatch = 1;
- break;
- }
- id = (struct id_info *)((char *)id + li->id_info_size);
- }
-
- if (members_mismatch)
- return 0;
-
- log_group(ls, "match_change %d:%u matches cg %u", hd->nodeid, seq,
- cg->seq);
- return 1;
-}
-
-/* Unfortunately, there's no really simple way to match a message with the
- specific change that it was sent for. We hope that by passing all the
- details of the change in the message, we will be able to uniquely match the
- it to the correct change. */
-
-/* A start message will usually be for the first (current) change on our list.
- In some cases it will be for a non-current change, and we can ignore it:
-
- 1. A,B,C get confchg1 adding C
- 2. C sends start for confchg1
- 3. A,B,C get confchg2 adding D
- 4. A,B,C,D recv start from C for confchg1 - ignored
- 5. C,D send start for confchg2
- 6. A,B send start for confchg2
- 7. A,B,C,D recv all start messages for confchg2, and start kernel
-
- In step 4, how do the nodes know whether the start message from C is
- for confchg1 or confchg2? Hopefully by comparing the counts and members. */
-
-static struct change *find_change(struct lockspace *ls, struct dlm_header *hd,
- struct ls_info *li, struct id_info *ids)
-{
- struct change *cg;
-
- list_for_each_entry_reverse(cg, &ls->changes, list) {
- if (!match_change(ls, cg, hd, li, ids))
- continue;
- return cg;
- }
-
- log_group(ls, "find_change %d:%u no match", hd->nodeid, hd->msgdata);
- return NULL;
-}
-
-static int is_added(struct lockspace *ls, int nodeid)
-{
- struct change *cg;
- struct member *memb;
-
- list_for_each_entry(cg, &ls->changes, list) {
- memb = find_memb(cg, nodeid);
- if (memb && memb->added)
- return 1;
- }
- return 0;
-}
-
-static void receive_start(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- struct change *cg;
- struct member *memb;
- struct ls_info *li;
- struct id_info *ids;
- uint32_t seq = hd->msgdata;
- int added;
-
- log_group(ls, "receive_start %d:%u len %d", hd->nodeid, seq, len);
-
- li = (struct ls_info *)((char *)hd + sizeof(struct dlm_header));
- ids = (struct id_info *)((char *)li + sizeof(struct ls_info));
-
- ls_info_in(li);
- ids_in(li, ids);
-
- cg = find_change(ls, hd, li, ids);
- if (!cg)
- return;
-
- memb = find_memb(cg, hd->nodeid);
- if (!memb) {
- /* this should never happen since match_change checks it */
- log_error("receive_start no member %d", hd->nodeid);
- return;
- }
-
- memb->start_flags = hd->flags;
-
- added = is_added(ls, hd->nodeid);
-
- if (added && li->started_count && ls->started_count) {
- log_error("receive_start %d:%u add node with started_count %u",
- hd->nodeid, seq, li->started_count);
-
- /* see comment in fence/fenced/cpg.c */
- memb->disallowed = 1;
- return;
- }
-
- if (memb->start_flags & DLM_MFLG_NACK) {
- log_group(ls, "receive_start %d:%u is NACK", hd->nodeid, seq);
- return;
- }
-
- node_history_start(ls, hd->nodeid);
- memb->start = 1;
-}
-
-static void receive_plocks_done(struct lockspace *ls, struct dlm_header *hd,
- int len)
-{
- struct ls_info *li;
- struct id_info *ids;
-
- log_dlock(ls, "receive_plocks_done %d:%u flags %x plocks_data %u need %d save
%d",
- hd->nodeid, hd->msgdata, hd->flags, hd->msgdata2,
- ls->need_plocks, ls->save_plocks);
-
- if (!ls->need_plocks)
- return;
-
- if (ls->need_plocks && !ls->save_plocks)
- return;
-
- if (!ls->started_change) {
- /* don't think this should happen */
- log_elock(ls, "receive_plocks_done %d:%u no started_change",
- hd->nodeid, hd->msgdata);
- return;
- }
-
- li = (struct ls_info *)((char *)hd + sizeof(struct dlm_header));
- ids = (struct id_info *)((char *)li + sizeof(struct ls_info));
- ls_info_in(li);
- ids_in(li, ids);
-
- if (!match_change(ls, ls->started_change, hd, li, ids)) {
- /* don't think this should happen */
- log_elock(ls, "receive_plocks_done %d:%u no match_change",
- hd->nodeid, hd->msgdata);
-
- /* remove/free anything we've saved from
- receive_plocks_data messages that weren't for us */
- clear_plocks_data(ls);
- return;
- }
-
- if (ls->recv_plocks_data_count != hd->msgdata2) {
- log_elock(ls, "receive_plocks_done plocks_data %u recv %u",
- hd->msgdata2, ls->recv_plocks_data_count);
- }
-
- process_saved_plocks(ls);
- ls->need_plocks = 0;
- ls->save_plocks = 0;
-
- log_dlock(ls, "receive_plocks_done %d:%u plocks_data_count %u",
- hd->nodeid, hd->msgdata, ls->recv_plocks_data_count);
-}
-
-static void send_info(struct lockspace *ls, struct change *cg, int type,
- uint32_t flags, uint32_t msgdata2)
-{
- struct dlm_header *hd;
- struct ls_info *li;
- struct id_info *id;
- struct member *memb;
- char *buf;
- int len, id_count;
-
- id_count = cg->member_count;
-
- len = sizeof(struct dlm_header) + sizeof(struct ls_info) +
- id_count * sizeof(struct id_info);
-
- buf = malloc(len);
- if (!buf) {
- log_error("send_info len %d no mem", len);
- return;
- }
- memset(buf, 0, len);
-
- hd = (struct dlm_header *)buf;
- li = (struct ls_info *)(buf + sizeof(*hd));
- id = (struct id_info *)(buf + sizeof(*hd) + sizeof(*li));
-
- /* fill in header (dlm_send_message handles part of header) */
-
- hd->type = type;
- hd->msgdata = cg->seq;
- hd->flags = flags;
- hd->msgdata2 = msgdata2;
-
- if (ls->joining)
- hd->flags |= DLM_MFLG_JOINING;
- if (!ls->need_plocks)
- hd->flags |= DLM_MFLG_HAVEPLOCK;
-
- /* fill in ls_info */
-
- li->ls_info_size = cpu_to_le32(sizeof(struct ls_info));
- li->id_info_size = cpu_to_le32(sizeof(struct id_info));
- li->id_info_count = cpu_to_le32(id_count);
- li->started_count = cpu_to_le32(ls->started_count);
- li->member_count = cpu_to_le32(cg->member_count);
- li->joined_count = cpu_to_le32(cg->joined_count);
- li->remove_count = cpu_to_le32(cg->remove_count);
- li->failed_count = cpu_to_le32(cg->failed_count);
-
- /* fill in id_info entries */
-
- list_for_each_entry(memb, &cg->members, list) {
- id->nodeid = cpu_to_le32(memb->nodeid);
- id++;
- }
-
- dlm_send_message(ls, buf, len);
-
- free(buf);
-}
-
-static void send_start(struct lockspace *ls, struct change *cg)
-{
- log_group(ls, "send_start %d:%u counts %u %d %d %d %d",
- our_nodeid, cg->seq, ls->started_count,
- cg->member_count, cg->joined_count, cg->remove_count,
- cg->failed_count);
-
- send_info(ls, cg, DLM_MSG_START, 0, 0);
-}
-
-static void send_plocks_done(struct lockspace *ls, struct change *cg, uint32_t
plocks_data)
-{
- log_dlock(ls, "send_plocks_done %d:%u counts %u %d %d %d %d plocks_data %u",
- our_nodeid, cg->seq, ls->started_count,
- cg->member_count, cg->joined_count, cg->remove_count,
- cg->failed_count, plocks_data);
-
- send_info(ls, cg, DLM_MSG_PLOCKS_DONE, 0, plocks_data);
-}
-
-static int same_members(struct change *cg1, struct change *cg2)
-{
- struct member *memb;
-
- list_for_each_entry(memb, &cg1->members, list) {
- if (!find_memb(cg2, memb->nodeid))
- return 0;
- }
- return 1;
-}
-
-static void send_nacks(struct lockspace *ls, struct change *startcg)
-{
- struct change *cg;
-
- list_for_each_entry(cg, &ls->changes, list) {
- if (cg->seq < startcg->seq &&
- cg->member_count == startcg->member_count &&
- cg->joined_count == startcg->joined_count &&
- cg->remove_count == startcg->remove_count &&
- cg->failed_count == startcg->failed_count &&
- same_members(cg, startcg)) {
- log_group(ls, "send nack old cg %u new cg %u",
- cg->seq, startcg->seq);
- send_info(ls, cg, DLM_MSG_START, DLM_MFLG_NACK, 0);
- }
- }
-}
-
-static int nodes_added(struct lockspace *ls)
-{
- struct change *cg;
-
- list_for_each_entry(cg, &ls->changes, list) {
- if (cg->joined_count)
- return 1;
- }
- return 0;
-}
-
-static void prepare_plocks(struct lockspace *ls)
-{
- struct change *cg = list_first_entry(&ls->changes, struct change, list);
- struct member *memb;
- uint32_t plocks_data;
-
- if (!cfgd_enable_plock || ls->disable_plock)
- return;
-
- log_dlock(ls, "prepare_plocks");
-
- /* if we're the only node in the lockspace, then we are the data_node
- and we don't need plocks */
-
- if (cg->member_count == 1) {
- list_for_each_entry(memb, &cg->members, list) {
- if (memb->nodeid != our_nodeid) {
- log_elock(ls, "prepare_plocks other member %d",
- memb->nodeid);
- }
- }
- ls->plock_data_node = our_nodeid;
- ls->need_plocks = 0;
- return;
- }
-
- /* the low node that indicated it had plock state in its last
- start message is the data_node */
-
- set_plock_data_node(ls);
-
- /* there is no node with plock state, so there's no syncing to do */
-
- if (!ls->plock_data_node) {
- ls->need_plocks = 0;
- ls->save_plocks = 0;
- return;
- }
-
- /* We save all plock messages received after our own confchg and
- apply them after we receive the plocks_done message from the
- data_node. */
-
- if (ls->need_plocks) {
- log_dlock(ls, "save_plocks start");
- ls->save_plocks = 1;
- return;
- }
-
- if (ls->plock_data_node != our_nodeid)
- return;
-
- if (nodes_added(ls))
- send_all_plocks_data(ls, cg->seq, &plocks_data);
-
- send_plocks_done(ls, cg, plocks_data);
-}
-
-static void apply_changes(struct lockspace *ls)
-{
- struct change *cg;
-
- if (list_empty(&ls->changes))
- return;
- cg = list_first_entry(&ls->changes, struct change, list);
-
- switch (cg->state) {
-
- case CGST_WAIT_CONDITIONS:
- if (wait_conditions_done(ls)) {
- send_nacks(ls, cg);
- send_start(ls, cg);
- cg->state = CGST_WAIT_MESSAGES;
- }
- break;
-
- case CGST_WAIT_MESSAGES:
- if (wait_messages_done(ls)) {
- start_kernel(ls);
- prepare_plocks(ls);
- cleanup_changes(ls);
- }
- break;
-
- default:
- log_error("apply_changes invalid state %d", cg->state);
- }
-}
-
-void process_lockspace_changes(void)
-{
- struct lockspace *ls, *safe;
-
- poll_fencing = 0;
- poll_quorum = 0;
- poll_fs = 0;
-
- list_for_each_entry_safe(ls, safe, &lockspaces, list) {
- if (!list_empty(&ls->changes))
- apply_changes(ls);
- }
-}
-
-static int add_change(struct lockspace *ls,
- const struct cpg_address *member_list,
- size_t member_list_entries,
- const struct cpg_address *left_list,
- size_t left_list_entries,
- const struct cpg_address *joined_list,
- size_t joined_list_entries,
- struct change **cg_out)
-{
- struct change *cg;
- struct member *memb;
- int i, error;
-
- cg = malloc(sizeof(struct change));
- if (!cg)
- goto fail_nomem;
- memset(cg, 0, sizeof(struct change));
- INIT_LIST_HEAD(&cg->members);
- INIT_LIST_HEAD(&cg->removed);
- cg->state = CGST_WAIT_CONDITIONS;
- cg->create_time = time(NULL);
- cg->seq = ++ls->change_seq;
- if (!cg->seq)
- cg->seq = ++ls->change_seq;
-
- cg->member_count = member_list_entries;
- cg->joined_count = joined_list_entries;
- cg->remove_count = left_list_entries;
-
- for (i = 0; i < member_list_entries; i++) {
- memb = malloc(sizeof(struct member));
- if (!memb)
- goto fail_nomem;
- memset(memb, 0, sizeof(struct member));
- memb->nodeid = member_list[i].nodeid;
- list_add_tail(&memb->list, &cg->members);
- }
-
- for (i = 0; i < left_list_entries; i++) {
- memb = malloc(sizeof(struct member));
- if (!memb)
- goto fail_nomem;
- memset(memb, 0, sizeof(struct member));
- memb->nodeid = left_list[i].nodeid;
- if (left_list[i].reason == CPG_REASON_NODEDOWN ||
- left_list[i].reason == CPG_REASON_PROCDOWN) {
- memb->failed = 1;
- cg->failed_count++;
- }
- list_add_tail(&memb->list, &cg->removed);
-
- if (memb->failed)
- node_history_fail(ls, memb->nodeid, cg,
- left_list[i].reason);
- else
- node_history_left(ls, memb->nodeid, cg);
-
- log_group(ls, "add_change cg %u remove nodeid %d reason %d",
- cg->seq, memb->nodeid, left_list[i].reason);
-
- if (left_list[i].reason == CPG_REASON_PROCDOWN)
- kick_node_from_cluster(memb->nodeid);
- }
-
- for (i = 0; i < joined_list_entries; i++) {
- memb = find_memb(cg, joined_list[i].nodeid);
- if (!memb) {
- log_error("no member %d", joined_list[i].nodeid);
- error = -ENOENT;
- goto fail;
- }
- memb->added = 1;
-
- if (memb->nodeid == our_nodeid)
- cg->we_joined = 1;
- else
- node_history_init(ls, memb->nodeid, cg);
-
- log_group(ls, "add_change cg %u joined nodeid %d", cg->seq,
- memb->nodeid);
- }
-
- if (cg->we_joined) {
- log_group(ls, "add_change cg %u we joined", cg->seq);
- list_for_each_entry(memb, &cg->members, list)
- node_history_init(ls, memb->nodeid, cg);
- }
-
- log_group(ls, "add_change cg %u counts member %d joined %d remove %d "
- "failed %d", cg->seq, cg->member_count, cg->joined_count,
- cg->remove_count, cg->failed_count);
-
- list_add(&cg->list, &ls->changes);
- *cg_out = cg;
- return 0;
-
- fail_nomem:
- log_error("no memory");
- error = -ENOMEM;
- fail:
- free_cg(cg);
- return error;
-}
-
-static int we_left(const struct cpg_address *left_list,
- size_t left_list_entries)
-{
- int i;
-
- for (i = 0; i < left_list_entries; i++) {
- if (left_list[i].nodeid == our_nodeid)
- return 1;
- }
- return 0;
-}
-
-static void confchg_cb(cpg_handle_t handle,
- const struct cpg_name *group_name,
- const struct cpg_address *member_list,
- size_t member_list_entries,
- const struct cpg_address *left_list,
- size_t left_list_entries,
- const struct cpg_address *joined_list,
- size_t joined_list_entries)
-{
- struct lockspace *ls;
- struct change *cg;
- struct member *memb;
- int rv;
-
- log_config(group_name, member_list, member_list_entries,
- left_list, left_list_entries,
- joined_list, joined_list_entries);
-
- ls = find_ls_handle(handle);
- if (!ls) {
- log_error("confchg_cb no lockspace for cpg %s",
- group_name->value);
- return;
- }
-
- if (ls->leaving && we_left(left_list, left_list_entries)) {
- /* we called cpg_leave(), and this should be the final
- cpg callback we receive */
- log_group(ls, "confchg for our leave");
- stop_kernel(ls, 0);
- set_configfs_members(ls->name, 0, NULL, 0, NULL);
- set_sysfs_event_done(ls->name, 0);
- cpg_finalize(ls->cpg_handle);
- client_dead(ls->cpg_client);
- purge_plocks(ls, our_nodeid, 1);
- list_del(&ls->list);
- free_ls(ls);
- return;
- }
-
- rv = add_change(ls, member_list, member_list_entries,
- left_list, left_list_entries,
- joined_list, joined_list_entries, &cg);
- if (rv)
- return;
-
- stop_kernel(ls, cg->seq);
-
- list_for_each_entry(memb, &cg->removed, list)
- purge_plocks(ls, memb->nodeid, 0);
-
- apply_changes(ls);
-
-#if 0
- deadlk_confchg(ls, member_list, member_list_entries,
- left_list, left_list_entries,
- joined_list, joined_list_entries);
-#endif
-}
-
-static void dlm_header_in(struct dlm_header *hd)
-{
- hd->version[0] = le16_to_cpu(hd->version[0]);
- hd->version[1] = le16_to_cpu(hd->version[1]);
- hd->version[2] = le16_to_cpu(hd->version[2]);
- hd->type = le16_to_cpu(hd->type);
- hd->nodeid = le32_to_cpu(hd->nodeid);
- hd->to_nodeid = le32_to_cpu(hd->to_nodeid);
- hd->global_id = le32_to_cpu(hd->global_id);
- hd->flags = le32_to_cpu(hd->flags);
- hd->msgdata = le32_to_cpu(hd->msgdata);
- hd->msgdata2 = le32_to_cpu(hd->msgdata2);
-}
-
-/* after our join confchg, we want to ignore plock messages (see need_plocks
- checks below) until the point in time where the ckpt_node saves plock
- state (final start message received); at this time we want to shift from
- ignoring plock messages to saving plock messages to apply on top of the
- plock state that we read. */
-
-static void deliver_cb(cpg_handle_t handle,
- const struct cpg_name *group_name,
- uint32_t nodeid, uint32_t pid,
- void *data, size_t len)
-{
- struct lockspace *ls;
- struct dlm_header *hd;
- int ignore_plock;
-
- ls = find_ls_handle(handle);
- if (!ls) {
- log_error("deliver_cb no ls for cpg %s", group_name->value);
- return;
- }
-
- if (len < sizeof(*hd)) {
- log_error("deliver_cb short message %zd", len);
- return;
- }
-
- hd = (struct dlm_header *)data;
- dlm_header_in(hd);
-
- if (hd->version[0] != our_protocol.daemon_run[0] ||
- hd->version[1] != our_protocol.daemon_run[1]) {
- log_error("reject message from %d version %u.%u.%u vs %u.%u.%u",
- nodeid, hd->version[0], hd->version[1],
- hd->version[2], our_protocol.daemon_run[0],
- our_protocol.daemon_run[1],
- our_protocol.daemon_run[2]);
- return;
- }
-
- if (hd->nodeid != nodeid) {
- log_error("bad msg nodeid %d %d", hd->nodeid, nodeid);
- return;
- }
-
- ignore_plock = 0;
-
- switch (hd->type) {
- case DLM_MSG_START:
- receive_start(ls, hd, len);
- break;
-
- case DLM_MSG_PLOCK:
- if (ls->disable_plock)
- break;
- if (ls->need_plocks && !ls->save_plocks) {
- ignore_plock = 1;
- break;
- }
- if (cfgd_enable_plock)
- receive_plock(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_plock %d",
- hd->type, nodeid, cfgd_enable_plock);
- break;
-
- case DLM_MSG_PLOCK_OWN:
- if (ls->disable_plock)
- break;
- if (ls->need_plocks && !ls->save_plocks) {
- ignore_plock = 1;
- break;
- }
- if (cfgd_enable_plock && cfgd_plock_ownership)
- receive_own(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_plock %d owner %d",
- hd->type, nodeid, cfgd_enable_plock,
- cfgd_plock_ownership);
- break;
-
- case DLM_MSG_PLOCK_DROP:
- if (ls->disable_plock)
- break;
- if (ls->need_plocks && !ls->save_plocks) {
- ignore_plock = 1;
- break;
- }
- if (cfgd_enable_plock && cfgd_plock_ownership)
- receive_drop(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_plock %d owner %d",
- hd->type, nodeid, cfgd_enable_plock,
- cfgd_plock_ownership);
- break;
-
- case DLM_MSG_PLOCK_SYNC_LOCK:
- case DLM_MSG_PLOCK_SYNC_WAITER:
- if (ls->disable_plock)
- break;
- if (ls->need_plocks && !ls->save_plocks) {
- ignore_plock = 1;
- break;
- }
- if (cfgd_enable_plock && cfgd_plock_ownership)
- receive_sync(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_plock %d owner %d",
- hd->type, nodeid, cfgd_enable_plock,
- cfgd_plock_ownership);
- break;
-
- case DLM_MSG_PLOCKS_DATA:
- if (ls->disable_plock)
- break;
- if (cfgd_enable_plock)
- receive_plocks_data(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_plock %d",
- hd->type, nodeid, cfgd_enable_plock);
- break;
-
- case DLM_MSG_PLOCKS_DONE:
- if (ls->disable_plock)
- break;
- if (cfgd_enable_plock)
- receive_plocks_done(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_plock %d",
- hd->type, nodeid, cfgd_enable_plock);
- break;
-
-#if 0
- case DLM_MSG_DEADLK_CYCLE_START:
- if (cfgd_enable_deadlk)
- receive_cycle_start(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_deadlk %d",
- hd->type, nodeid, cfgd_enable_deadlk);
- break;
-
- case DLM_MSG_DEADLK_CYCLE_END:
- if (cfgd_enable_deadlk)
- receive_cycle_end(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_deadlk %d",
- hd->type, nodeid, cfgd_enable_deadlk);
- break;
-
- case DLM_MSG_DEADLK_CHECKPOINT_READY:
- if (cfgd_enable_deadlk)
- receive_checkpoint_ready(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_deadlk %d",
- hd->type, nodeid, cfgd_enable_deadlk);
- break;
-
- case DLM_MSG_DEADLK_CANCEL_LOCK:
- if (cfgd_enable_deadlk)
- receive_cancel_lock(ls, hd, len);
- else
- log_error("msg %d nodeid %d enable_deadlk %d",
- hd->type, nodeid, cfgd_enable_deadlk);
- break;
-#endif
-
- default:
- log_error("unknown msg type %d", hd->type);
- }
-
- if (ignore_plock)
- log_plock(ls, "msg %s nodeid %d need_plock ignore",
- msg_name(hd->type), nodeid);
-
- apply_changes(ls);
-}
-
-static cpg_callbacks_t cpg_callbacks = {
- .cpg_deliver_fn = deliver_cb,
- .cpg_confchg_fn = confchg_cb,
-};
-
-void update_flow_control_status(void)
-{
- cpg_flow_control_state_t flow_control_state;
- cpg_error_t error;
-
- error = cpg_flow_control_state_get(cpg_handle_daemon,
- &flow_control_state);
- if (error != CPG_OK) {
- log_error("cpg_flow_control_state_get %d", error);
- return;
- }
-
- if (flow_control_state == CPG_FLOW_CONTROL_ENABLED) {
- if (message_flow_control_on == 0) {
- log_debug("flow control on");
- }
- message_flow_control_on = 1;
- } else {
- if (message_flow_control_on) {
- log_debug("flow control off");
- }
- message_flow_control_on = 0;
- }
-}
-
-static void process_cpg_lockspace(int ci)
-{
- struct lockspace *ls;
- cpg_error_t error;
-
- ls = find_ls_ci(ci);
- if (!ls) {
- log_error("process_lockspace_cpg no lockspace for ci %d", ci);
- return;
- }
-
- error = cpg_dispatch(ls->cpg_handle, CPG_DISPATCH_ALL);
- if (error != CPG_OK) {
- log_error("cpg_dispatch error %d", error);
- return;
- }
-
- update_flow_control_status();
-}
-
-/* received an "online" uevent from dlm-kernel */
-
-int dlm_join_lockspace(struct lockspace *ls)
-{
- cpg_error_t error;
- cpg_handle_t h;
- struct cpg_name name;
- int i = 0, fd, ci, rv;
- int unused;
-
- rv = fence_in_progress(&unused);
- if (cfgd_enable_fencing && rv < 0) {
- log_error("dlm_join_lockspace no fence domain");
- rv = -1;
- goto fail_free;
- }
-
- error = cpg_initialize(&h, &cpg_callbacks);
- if (error != CPG_OK) {
- log_error("cpg_initialize error %d", error);
- rv = -1;
- goto fail_free;
- }
-
- cpg_fd_get(h, &fd);
-
- ci = client_add(fd, process_cpg_lockspace, NULL);
-
- list_add(&ls->list, &lockspaces);
-
- ls->cpg_handle = h;
- ls->cpg_client = ci;
- ls->cpg_fd = fd;
- ls->kernel_stopped = 1;
- ls->need_plocks = 1;
- ls->joining = 1;
-
- memset(&name, 0, sizeof(name));
- sprintf(name.value, "dlm:ls:%s", ls->name);
- name.length = strlen(name.value) + 1;
-
- /* TODO: allow global_id to be set in cluster.conf? */
- ls->global_id = cpgname_to_crc(name.value, name.length);
-
- retry:
- error = cpg_join(h, &name);
- if (error == CPG_ERR_TRY_AGAIN) {
- sleep(1);
- if (!(++i % 10))
- log_error("cpg_join error retrying");
- goto retry;
- }
- if (error != CPG_OK) {
- log_error("cpg_join error %d", error);
- cpg_finalize(h);
- rv = -1;
- goto fail;
- }
-
- return 0;
-
- fail:
- list_del(&ls->list);
- client_dead(ci);
- cpg_finalize(h);
- fail_free:
- set_sysfs_event_done(ls->name, rv);
- free_ls(ls);
- return rv;
-}
-
-/* received an "offline" uevent from dlm-kernel */
-
-int dlm_leave_lockspace(struct lockspace *ls)
-{
- cpg_error_t error;
- struct cpg_name name;
- int i = 0;
-
- ls->leaving = 1;
-
- memset(&name, 0, sizeof(name));
- sprintf(name.value, "dlm:ls:%s", ls->name);
- name.length = strlen(name.value) + 1;
-
- retry:
- error = cpg_leave(ls->cpg_handle, &name);
- if (error == CPG_ERR_TRY_AGAIN) {
- sleep(1);
- if (!(++i % 10))
- log_error("cpg_leave error retrying");
- goto retry;
- }
- if (error != CPG_OK)
- log_error("cpg_leave error %d", error);
-
- return 0;
-}
-
-static struct node *get_node_daemon(int nodeid)
-{
- struct node *node;
-
- list_for_each_entry(node, &daemon_nodes, list) {
- if (node->nodeid == nodeid)
- return node;
- }
- return NULL;
-}
-
-static void add_node_daemon(int nodeid)
-{
- struct node *node;
-
- if (get_node_daemon(nodeid))
- return;
-
- node = malloc(sizeof(struct node));
- if (!node) {
- log_error("add_node_daemon no mem");
- return;
- }
- memset(node, 0, sizeof(struct node));
- node->nodeid = nodeid;
- list_add_tail(&node->list, &daemon_nodes);
-}
-
-static void pv_in(struct protocol_version *pv)
-{
- pv->major = le16_to_cpu(pv->major);
- pv->minor = le16_to_cpu(pv->minor);
- pv->patch = le16_to_cpu(pv->patch);
- pv->flags = le16_to_cpu(pv->flags);
-}
-
-static void pv_out(struct protocol_version *pv)
-{
- pv->major = cpu_to_le16(pv->major);
- pv->minor = cpu_to_le16(pv->minor);
- pv->patch = cpu_to_le16(pv->patch);
- pv->flags = cpu_to_le16(pv->flags);
-}
-
-static void protocol_in(struct protocol *proto)
-{
- pv_in(&proto->dm_ver);
- pv_in(&proto->km_ver);
- pv_in(&proto->dr_ver);
- pv_in(&proto->kr_ver);
-}
-
-static void protocol_out(struct protocol *proto)
-{
- pv_out(&proto->dm_ver);
- pv_out(&proto->km_ver);
- pv_out(&proto->dr_ver);
- pv_out(&proto->kr_ver);
-}
-
-/* go through member list saved in last confchg, see if we have received a
- proto message from each */
-
-static int all_protocol_messages(void)
-{
- struct node *node;
- int i;
-
- if (!daemon_member_count)
- return 0;
-
- for (i = 0; i < daemon_member_count; i++) {
- node = get_node_daemon(daemon_member[i].nodeid);
- if (!node) {
- log_error("all_protocol_messages no node %d",
- daemon_member[i].nodeid);
- return 0;
- }
-
- if (!node->proto.daemon_max[0])
- return 0;
- }
- return 1;
-}
-
-static int pick_min_protocol(struct protocol *proto)
-{
- uint16_t mind[4];
- uint16_t mink[4];
- struct node *node;
- int i;
-
- memset(&mind, 0, sizeof(mind));
- memset(&mink, 0, sizeof(mink));
-
- /* first choose the minimum major */
-
- for (i = 0; i < daemon_member_count; i++) {
- node = get_node_daemon(daemon_member[i].nodeid);
- if (!node) {
- log_error("pick_min_protocol no node %d",
- daemon_member[i].nodeid);
- return -1;
- }
-
- if (!mind[0] || node->proto.daemon_max[0] < mind[0])
- mind[0] = node->proto.daemon_max[0];
-
- if (!mink[0] || node->proto.kernel_max[0] < mink[0])
- mink[0] = node->proto.kernel_max[0];
- }
-
- if (!mind[0] || !mink[0]) {
- log_error("pick_min_protocol zero major number");
- return -1;
- }
-
- /* second pick the minimum minor with the chosen major */
-
- for (i = 0; i < daemon_member_count; i++) {
- node = get_node_daemon(daemon_member[i].nodeid);
- if (!node)
- continue;
-
- if (mind[0] == node->proto.daemon_max[0]) {
- if (!mind[1] || node->proto.daemon_max[1] < mind[1])
- mind[1] = node->proto.daemon_max[1];
- }
-
- if (mink[0] == node->proto.kernel_max[0]) {
- if (!mink[1] || node->proto.kernel_max[1] < mink[1])
- mink[1] = node->proto.kernel_max[1];
- }
- }
-
- if (!mind[1] || !mink[1]) {
- log_error("pick_min_protocol zero minor number");
- return -1;
- }
-
- /* third pick the minimum patch with the chosen major.minor */
-
- for (i = 0; i < daemon_member_count; i++) {
- node = get_node_daemon(daemon_member[i].nodeid);
- if (!node)
- continue;
-
- if (mind[0] == node->proto.daemon_max[0] &&
- mind[1] == node->proto.daemon_max[1]) {
- if (!mind[2] || node->proto.daemon_max[2] < mind[2])
- mind[2] = node->proto.daemon_max[2];
- }
-
- if (mink[0] == node->proto.kernel_max[0] &&
- mink[1] == node->proto.kernel_max[1]) {
- if (!mink[2] || node->proto.kernel_max[2] < mink[2])
- mink[2] = node->proto.kernel_max[2];
- }
- }
-
- if (!mind[2] || !mink[2]) {
- log_error("pick_min_protocol zero patch number");
- return -1;
- }
-
- memcpy(&proto->daemon_run, &mind, sizeof(mind));
- memcpy(&proto->kernel_run, &mink, sizeof(mink));
- return 0;
-}
-
-static void receive_protocol(struct dlm_header *hd, int len)
-{
- struct protocol *p;
- struct node *node;
-
- p = (struct protocol *)((char *)hd + sizeof(struct dlm_header));
- protocol_in(p);
-
- if (len < sizeof(struct dlm_header) + sizeof(struct protocol)) {
- log_error("receive_protocol invalid len %d from %d",
- len, hd->nodeid);
- return;
- }
-
- /* zero is an invalid version value */
-
- if (!p->daemon_max[0] || !p->daemon_max[1] || !p->daemon_max[2] ||
- !p->kernel_max[0] || !p->kernel_max[1] || !p->kernel_max[2]) {
- log_error("receive_protocol invalid max value from %d "
- "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
- p->daemon_max[0], p->daemon_max[1], p->daemon_max[2],
- p->kernel_max[0], p->kernel_max[1], p->kernel_max[2]);
- return;
- }
-
- /* the run values will be zero until a version is set, after
- which none of the run values can be zero */
-
- if (p->daemon_run[0] && (!p->daemon_run[1] || !p->daemon_run[2] ||
- !p->kernel_run[0] || !p->kernel_run[1] || !p->kernel_run[2])) {
- log_error("receive_protocol invalid run value from %d "
- "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
- p->daemon_run[0], p->daemon_run[1], p->daemon_run[2],
- p->kernel_run[0], p->kernel_run[1], p->kernel_run[2]);
- return;
- }
-
- /* if we have zero run values, and this msg has non-zero run values,
- then adopt them as ours; otherwise save this proto message */
-
- if (our_protocol.daemon_run[0])
- return;
-
- if (p->daemon_run[0]) {
- memcpy(&our_protocol.daemon_run, &p->daemon_run,
- sizeof(struct protocol_version));
- memcpy(&our_protocol.kernel_run, &p->kernel_run,
- sizeof(struct protocol_version));
- log_debug("run protocol from nodeid %d", hd->nodeid);
- return;
- }
-
- /* save this node's proto so we can tell when we've got all, and
- use it to select a minimum protocol from all */
-
- node = get_node_daemon(hd->nodeid);
- if (!node) {
- log_error("receive_protocol no node %d", hd->nodeid);
- return;
- }
- memcpy(&node->proto, p, sizeof(struct protocol));
-}
-
-static void send_protocol(struct protocol *proto)
-{
- struct dlm_header *hd;
- struct protocol *pr;
- char *buf;
- int len;
-
- len = sizeof(struct dlm_header) + sizeof(struct protocol);
- buf = malloc(len);
- if (!buf) {
- log_error("send_protocol no mem %d", len);
- return;
- }
- memset(buf, 0, len);
-
- hd = (struct dlm_header *)buf;
- pr = (struct protocol *)(buf + sizeof(*hd));
-
- hd->type = cpu_to_le16(DLM_MSG_PROTOCOL);
- hd->nodeid = cpu_to_le32(our_nodeid);
-
- memcpy(pr, proto, sizeof(struct protocol));
- protocol_out(pr);
-
- _send_message(cpg_handle_daemon, buf, len, DLM_MSG_PROTOCOL);
-}
-
-int set_protocol(void)
-{
- struct protocol proto;
- struct pollfd pollfd;
- int sent_proposal = 0;
- int rv;
-
- memset(&pollfd, 0, sizeof(pollfd));
- pollfd.fd = cpg_fd_daemon;
- pollfd.events = POLLIN;
-
- while (1) {
- if (our_protocol.daemon_run[0])
- break;
-
- if (!sent_proposal && all_protocol_messages()) {
- /* propose a protocol; look through info from all
- nodes and pick the min for both daemon and kernel,
- and propose that */
-
- sent_proposal = 1;
-
- /* copy our max values */
- memcpy(&proto, &our_protocol, sizeof(struct protocol));
-
- rv = pick_min_protocol(&proto);
- if (rv < 0)
- return rv;
-
- log_debug("set_protocol member_count %d propose "
- "daemon %u.%u.%u kernel %u.%u.%u",
- daemon_member_count,
- proto.daemon_run[0], proto.daemon_run[1],
- proto.daemon_run[2], proto.kernel_run[0],
- proto.kernel_run[1], proto.kernel_run[2]);
-
- send_protocol(&proto);
- }
-
- /* only process messages/events from daemon cpg until protocol
- is established */
-
- rv = poll(&pollfd, 1, -1);
- if (rv == -1 && errno == EINTR) {
- if (daemon_quit)
- return -1;
- continue;
- }
- if (rv < 0) {
- log_error("set_protocol poll errno %d", errno);
- return -1;
- }
-
- if (pollfd.revents & POLLIN)
- process_cpg_daemon(0);
- if (pollfd.revents & (POLLERR | POLLHUP | POLLNVAL)) {
- log_error("set_protocol poll revents %u",
- pollfd.revents);
- return -1;
- }
- }
-
- if (our_protocol.daemon_run[0] != our_protocol.daemon_max[0] ||
- our_protocol.daemon_run[1] > our_protocol.daemon_max[1]) {
- log_error("incompatible daemon protocol run %u.%u.%u max %u.%u.%u",
- our_protocol.daemon_run[0],
- our_protocol.daemon_run[1],
- our_protocol.daemon_run[2],
- our_protocol.daemon_max[0],
- our_protocol.daemon_max[1],
- our_protocol.daemon_max[2]);
- return -1;
- }
-
- if (our_protocol.kernel_run[0] != our_protocol.kernel_max[0] ||
- our_protocol.kernel_run[1] > our_protocol.kernel_max[1]) {
- log_error("incompatible kernel protocol run %u.%u.%u max %u.%u.%u",
- our_protocol.kernel_run[0],
- our_protocol.kernel_run[1],
- our_protocol.kernel_run[2],
- our_protocol.kernel_max[0],
- our_protocol.kernel_max[1],
- our_protocol.kernel_max[2]);
- return -1;
- }
-
- log_debug("daemon run %u.%u.%u max %u.%u.%u "
- "kernel run %u.%u.%u max %u.%u.%u",
- our_protocol.daemon_run[0],
- our_protocol.daemon_run[1],
- our_protocol.daemon_run[2],
- our_protocol.daemon_max[0],
- our_protocol.daemon_max[1],
- our_protocol.daemon_max[2],
- our_protocol.kernel_run[0],
- our_protocol.kernel_run[1],
- our_protocol.kernel_run[2],
- our_protocol.kernel_max[0],
- our_protocol.kernel_max[1],
- our_protocol.kernel_max[2]);
-
- send_protocol(&our_protocol);
- return 0;
-}
-
-static void deliver_cb_daemon(cpg_handle_t handle,
- const struct cpg_name *group_name,
- uint32_t nodeid, uint32_t pid,
- void *data, size_t len)
-{
- struct dlm_header *hd;
-
- if (len < sizeof(*hd)) {
- log_error("deliver_cb short message %zd", len);
- return;
- }
-
- hd = (struct dlm_header *)data;
- dlm_header_in(hd);
-
- switch (hd->type) {
- case DLM_MSG_PROTOCOL:
- receive_protocol(hd, len);
- break;
- default:
- log_error("deliver_cb_daemon unknown msg type %d", hd->type);
- }
-}
-
-static void confchg_cb_daemon(cpg_handle_t handle,
- const struct cpg_name *group_name,
- const struct cpg_address *member_list,
- size_t member_list_entries,
- const struct cpg_address *left_list,
- size_t left_list_entries,
- const struct cpg_address *joined_list,
- size_t joined_list_entries)
-{
- int i;
-
- log_config(group_name, member_list, member_list_entries,
- left_list, left_list_entries,
- joined_list, joined_list_entries);
-
- if (joined_list_entries)
- send_protocol(&our_protocol);
-
- memset(&daemon_member, 0, sizeof(daemon_member));
- daemon_member_count = member_list_entries;
-
- for (i = 0; i < member_list_entries; i++) {
- daemon_member[i] = member_list[i];
- add_node_daemon(member_list[i].nodeid);
- }
-}
-
-static cpg_callbacks_t cpg_callbacks_daemon = {
- .cpg_deliver_fn = deliver_cb_daemon,
- .cpg_confchg_fn = confchg_cb_daemon,
-};
-
-void process_cpg_daemon(int ci)
-{
- cpg_error_t error;
-
- error = cpg_dispatch(cpg_handle_daemon, CPG_DISPATCH_ALL);
- if (error != CPG_OK)
- log_error("daemon cpg_dispatch error %d", error);
-}
-
-int setup_cpg_daemon(void)
-{
- cpg_error_t error;
- struct cpg_name name;
- int i = 0;
-
- INIT_LIST_HEAD(&daemon_nodes);
-
- memset(&our_protocol, 0, sizeof(our_protocol));
- our_protocol.daemon_max[0] = 2;
- our_protocol.daemon_max[1] = 1;
- our_protocol.daemon_max[2] = 1;
- our_protocol.kernel_max[0] = 1;
- our_protocol.kernel_max[1] = 1;
- our_protocol.kernel_max[2] = 1;
-
- error = cpg_initialize(&cpg_handle_daemon, &cpg_callbacks_daemon);
- if (error != CPG_OK) {
- log_error("daemon cpg_initialize error %d", error);
- return -1;
- }
-
- cpg_fd_get(cpg_handle_daemon, &cpg_fd_daemon);
-
- memset(&name, 0, sizeof(name));
- sprintf(name.value, "dlm:controld");
- name.length = strlen(name.value) + 1;
-
- retry:
- error = cpg_join(cpg_handle_daemon, &name);
- if (error == CPG_ERR_TRY_AGAIN) {
- sleep(1);
- if (!(++i % 10))
- log_error("daemon cpg_join error retrying");
- goto retry;
- }
- if (error != CPG_OK) {
- log_error("daemon cpg_join error %d", error);
- goto fail;
- }
-
- log_debug("setup_cpg_daemon %d", cpg_fd_daemon);
- return cpg_fd_daemon;
-
- fail:
- cpg_finalize(cpg_handle_daemon);
- return -1;
-}
-
-void close_cpg_daemon(void)
-{
- struct lockspace *ls;
- cpg_error_t error;
- struct cpg_name name;
- int i = 0;
-
- if (!cpg_handle_daemon)
- return;
- if (cluster_down)
- goto fin;
-
- memset(&name, 0, sizeof(name));
- sprintf(name.value, "dlm:controld");
- name.length = strlen(name.value) + 1;
-
- retry:
- error = cpg_leave(cpg_handle_daemon, &name);
- if (error == CPG_ERR_TRY_AGAIN) {
- sleep(1);
- if (!(++i % 10))
- log_error("daemon cpg_leave error retrying");
- goto retry;
- }
- if (error != CPG_OK)
- log_error("daemon cpg_leave error %d", error);
- fin:
- list_for_each_entry(ls, &lockspaces, list) {
- if (ls->cpg_handle)
- cpg_finalize(ls->cpg_handle);
- }
- cpg_finalize(cpg_handle_daemon);
-}
-
-/* fs_controld has seen nodedown for nodeid; it's now ok for dlm to do
- recovery for the failed node */
-
-int set_fs_notified(struct lockspace *ls, int nodeid)
-{
- struct node *node;
-
- /* this shouldn't happen */
- node = get_node_history(ls, nodeid);
- if (!node) {
- log_error("set_fs_notified no nodeid %d", nodeid);
- return -ESRCH;
- }
-
- if (!find_memb(ls->started_change, nodeid)) {
- log_group(ls, "set_fs_notified %d not in ls", nodeid);
- return 0;
- }
-
- /* this can happen, we haven't seen a nodedown for this node yet,
- but we should soon */
- if (!node->check_fs) {
- log_group(ls, "set_fs_notified %d zero check_fs", nodeid);
- return -EAGAIN;
- }
-
- log_group(ls, "set_fs_notified nodeid %d", nodeid);
- node->fs_notified = 1;
- return 0;
-}
-
-int set_lockspace_info(struct lockspace *ls, struct dlmc_lockspace *lockspace)
-{
- struct change *cg, *last = NULL;
-
- strncpy(lockspace->name, ls->name, DLM_LOCKSPACE_LEN);
- lockspace->global_id = ls->global_id;
-
- if (ls->joining)
- lockspace->flags |= DLMC_LF_JOINING;
- if (ls->leaving)
- lockspace->flags |= DLMC_LF_LEAVING;
- if (ls->kernel_stopped)
- lockspace->flags |= DLMC_LF_KERNEL_STOPPED;
- if (ls->fs_registered)
- lockspace->flags |= DLMC_LF_FS_REGISTERED;
- if (ls->need_plocks)
- lockspace->flags |= DLMC_LF_NEED_PLOCKS;
- if (ls->save_plocks)
- lockspace->flags |= DLMC_LF_SAVE_PLOCKS;
-
- if (!ls->started_change)
- goto next;
-
- cg = ls->started_change;
-
- lockspace->cg_prev.member_count = cg->member_count;
- lockspace->cg_prev.joined_count = cg->joined_count;
- lockspace->cg_prev.remove_count = cg->remove_count;
- lockspace->cg_prev.failed_count = cg->failed_count;
- lockspace->cg_prev.combined_seq = cg->combined_seq;
- lockspace->cg_prev.seq = cg->seq;
-
- next:
- if (list_empty(&ls->changes))
- goto out;
-
- list_for_each_entry(cg, &ls->changes, list)
- last = cg;
-
- cg = list_first_entry(&ls->changes, struct change, list);
-
- lockspace->cg_next.member_count = cg->member_count;
- lockspace->cg_next.joined_count = cg->joined_count;
- lockspace->cg_next.remove_count = cg->remove_count;
- lockspace->cg_next.failed_count = cg->failed_count;
- lockspace->cg_next.combined_seq = last->seq;
- lockspace->cg_next.seq = cg->seq;
-
- if (cg->state == CGST_WAIT_CONDITIONS)
- lockspace->cg_next.wait_condition = 4;
- if (poll_fencing)
- lockspace->cg_next.wait_condition = 1;
- else if (poll_quorum)
- lockspace->cg_next.wait_condition = 2;
- else if (poll_fs)
- lockspace->cg_next.wait_condition = 3;
-
- if (cg->state == CGST_WAIT_MESSAGES)
- lockspace->cg_next.wait_messages = 1;
- out:
- return 0;
-}
-
-static int _set_node_info(struct lockspace *ls, struct change *cg, int nodeid,
- struct dlmc_node *node)
-{
- struct member *m = NULL;
- struct node *n;
-
- node->nodeid = nodeid;
-
- if (cg)
- m = find_memb(cg, nodeid);
- if (!m)
- goto history;
-
- node->flags |= DLMC_NF_MEMBER;
-
- if (m->start)
- node->flags |= DLMC_NF_START;
- if (m->disallowed)
- node->flags |= DLMC_NF_DISALLOWED;
-
- history:
- n = get_node_history(ls, nodeid);
- if (!n)
- goto out;
-
- if (n->check_fencing)
- node->flags |= DLMC_NF_CHECK_FENCING;
- if (n->check_quorum)
- node->flags |= DLMC_NF_CHECK_QUORUM;
- if (n->check_fs)
- node->flags |= DLMC_NF_CHECK_FS;
-
- node->added_seq = n->added_seq;
- node->removed_seq = n->removed_seq;
- node->failed_reason = n->failed_reason;
- out:
- return 0;
-}
-
-int set_node_info(struct lockspace *ls, int nodeid, struct dlmc_node *node)
-{
- struct change *cg;
-
- if (!list_empty(&ls->changes)) {
- cg = list_first_entry(&ls->changes, struct change, list);
- return _set_node_info(ls, cg, nodeid, node);
- }
-
- return _set_node_info(ls, ls->started_change, nodeid, node);
-}
-
-int set_lockspaces(int *count, struct dlmc_lockspace **lss_out)
-{
- struct lockspace *ls;
- struct dlmc_lockspace *lss, *lsp;
- int ls_count = 0;
-
- list_for_each_entry(ls, &lockspaces, list)
- ls_count++;
-
- lss = malloc(ls_count * sizeof(struct dlmc_lockspace));
- if (!lss)
- return -ENOMEM;
- memset(lss, 0, ls_count * sizeof(struct dlmc_lockspace));
-
- lsp = lss;
- list_for_each_entry(ls, &lockspaces, list) {
- set_lockspace_info(ls, lsp++);
- }
-
- *count = ls_count;
- *lss_out = lss;
- return 0;
-}
-
-int set_lockspace_nodes(struct lockspace *ls, int option, int *node_count,
- struct dlmc_node **nodes_out)
-{
- struct change *cg;
- struct node *n;
- struct dlmc_node *nodes = NULL, *nodep;
- struct member *memb;
- int count = 0;
-
- if (option == DLMC_NODES_ALL) {
- if (!list_empty(&ls->changes))
- cg = list_first_entry(&ls->changes, struct change,list);
- else
- cg = ls->started_change;
-
- list_for_each_entry(n, &ls->node_history, list)
- count++;
-
- } else if (option == DLMC_NODES_MEMBERS) {
- if (!ls->started_change)
- goto out;
- cg = ls->started_change;
- count = cg->member_count;
-
- } else if (option == DLMC_NODES_NEXT) {
- if (list_empty(&ls->changes))
- goto out;
- cg = list_first_entry(&ls->changes, struct change, list);
- count = cg->member_count;
- } else
- goto out;
-
- nodes = malloc(count * sizeof(struct dlmc_node));
- if (!nodes)
- return -ENOMEM;
- memset(nodes, 0, count * sizeof(struct dlmc_node));
- nodep = nodes;
-
- if (option == DLMC_NODES_ALL) {
- list_for_each_entry(n, &ls->node_history, list)
- _set_node_info(ls, cg, n->nodeid, nodep++);
- } else {
- list_for_each_entry(memb, &cg->members, list)
- _set_node_info(ls, cg, memb->nodeid, nodep++);
- }
- out:
- *node_count = count;
- *nodes_out = nodes;
- return 0;
-}
-
diff --git a/group/dlm_controld/crc.c b/group/dlm_controld/crc.c
deleted file mode 100644
index ff8c1d3..0000000
--- a/group/dlm_controld/crc.c
+++ /dev/null
@@ -1,72 +0,0 @@
-#include "dlm_daemon.h"
-
-static const uint32_t crc_32_tab[] = {
- 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535,
0x9e6495a3,
- 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
0x90bf1d91,
- 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551,
0x83d385c7,
- 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63,
0x8d080df5,
- 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd,
0xa50ab56b,
- 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf,
0xabd13d59,
- 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
0xb8bda50f,
- 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab,
0xb6662d3d,
- 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5,
0xe8b8d433,
- 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97,
0xe6635c01,
- 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1,
0xf50fc457,
- 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
0xfbd44c65,
- 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d,
0xd3d6f4fb,
- 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f,
0xdd0d7cc9,
- 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409,
0xce61e49f,
- 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b,
0xc0ba6cad,
- 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
0x73dc1683,
- 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27,
0x7d079eb1,
- 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671,
0x6e6b06e7,
- 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43,
0x60b08ed5,
- 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd,
0x48b2364b,
- 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
0x4669be79,
- 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9,
0x5505262f,
- 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b,
0x5bdeae1d,
- 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785,
0x05005713,
- 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7,
0x0bdbdf21,
- 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
0x18b74777,
- 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3,
0x166ccf45,
- 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d,
0x3e6e77db,
- 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f,
0x30b5ffe9,
- 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729,
0x23d967bf,
- 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
0x2d02ef8d
-};
-
-/**
- *
- * Copied from:
- *
- * gfs2_disk_hash - hash an array of data
- * @data: the data to be hashed
- * @len: the length of data to be hashed
- *
- * This function must produce the same results as the one in the kernel:
- * crc32_le(0xFFFFFFFF, data, len) ^ 0xFFFFFFFF
- *
- * Take some data and convert it to a 32-bit hash.
- *
- * The hash function is a 32-bit CRC of the data. The algorithm uses
- * the crc_32_tab table above.
- *
- * This may not be the fastest hash function, but it does a fair bit better
- * at providing uniform results than the others I've looked at. That's
- * really important for efficient directories.
- *
- * Returns: the hash
- */
-
-uint32_t cpgname_to_crc(const char *data, int len)
-{
- uint32_t hash = 0xFFFFFFFF;
-
- for (; len--; data++)
- hash = crc_32_tab[(hash ^ *data) & 0xFF] ^ (hash >> 8);
-
- hash = ~hash;
-
- return hash;
-}
-
diff --git a/group/dlm_controld/deadlock.c b/group/dlm_controld/deadlock.c
deleted file mode 100644
index bd1d68c..0000000
--- a/group/dlm_controld/deadlock.c
+++ /dev/null
@@ -1,1550 +0,0 @@
-#include "dlm_daemon.h"
-#include "config.h"
-#include "libdlm.h"
-
-static SaCkptHandleT global_ckpt_h;
-static SaCkptCallbacksT callbacks = { 0, 0 };
-static SaVersionT version = { 'B', 1, 1 };
-static char section_buf[10 * 1024 * 1024]; /* 10MB of pack_lock's enough? */
-static uint32_t section_len;
-static uint32_t section_max;
-
-struct node {
- struct list_head list;
- int nodeid;
- int checkpoint_ready; /* we've read its ckpt */
- int in_cycle; /* participating in cycle */
-};
-
-enum {
- LOCAL_COPY = 1,
- MASTER_COPY = 2,
-};
-
-/* from linux/fs/dlm/dlm_internal.h */
-#define DLM_LKSTS_WAITING 1
-#define DLM_LKSTS_GRANTED 2
-#define DLM_LKSTS_CONVERT 3
-
-struct pack_lock {
- uint64_t xid;
- uint32_t id;
- int nodeid;
- uint32_t remid;
- int ownpid;
- uint32_t exflags;
- uint32_t flags;
- int8_t status;
- int8_t grmode;
- int8_t rqmode;
- int8_t copy;
-};
-
-struct dlm_rsb {
- struct list_head list;
- struct list_head locks;
- char name[DLM_RESNAME_MAXLEN];
- int len;
-};
-
-/* information is saved in the lkb, and lkb->lock, from the perspective of the
- local or master copy, not the process copy */
-
-struct dlm_lkb {
- struct list_head list; /* r->locks */
- struct pack_lock lock; /* data from debugfs/checkpoint */
- int home; /* node where the lock owner lives*/
- struct dlm_rsb *rsb; /* lock is on resource */
- struct trans *trans; /* lock owned by this transaction */
- struct list_head trans_list; /* tr->locks */
- struct trans *waitfor_trans; /* the trans that's holding the
- lock that's blocking us */
-};
-
-/* waitfor pointers alloc'ed 4 at at time */
-#define TR_NALLOC 4
-
-struct trans {
- struct list_head list;
- struct list_head locks;
- uint64_t xid;
- int others_waiting_on_us; /* count of trans's
- pointing to us in
- waitfor */
- int waitfor_alloc;
- int waitfor_count; /* count of in-use
- waitfor slots and
- num of trans's we're
- waiting on */
- struct trans **waitfor; /* waitfor_alloc trans
- pointers */
-};
-
-static const int __dlm_compat_matrix[8][8] = {
- /* UN NL CR CW PR PW EX PD */
- {1, 1, 1, 1, 1, 1, 1, 0}, /* UN */
- {1, 1, 1, 1, 1, 1, 1, 0}, /* NL */
- {1, 1, 1, 1, 1, 1, 0, 0}, /* CR */
- {1, 1, 1, 1, 0, 0, 0, 0}, /* CW */
- {1, 1, 1, 0, 1, 0, 0, 0}, /* PR */
- {1, 1, 1, 0, 0, 0, 0, 0}, /* PW */
- {1, 1, 0, 0, 0, 0, 0, 0}, /* EX */
- {0, 0, 0, 0, 0, 0, 0, 0} /* PD */
-};
-
-static inline int dlm_modes_compat(int mode1, int mode2)
-{
- return __dlm_compat_matrix[mode1 + 1][mode2 + 1];
-}
-
-static const char *status_str(int lksts)
-{
- switch (lksts) {
- case DLM_LKSTS_WAITING:
- return "W";
- case DLM_LKSTS_GRANTED:
- return "G";
- case DLM_LKSTS_CONVERT:
- return "C";
- }
- return "?";
-}
-
-static void free_resources(struct lockspace *ls)
-{
- struct dlm_rsb *r, *r_safe;
- struct dlm_lkb *lkb, *lkb_safe;
-
- list_for_each_entry_safe(r, r_safe, &ls->resources, list) {
- list_for_each_entry_safe(lkb, lkb_safe, &r->locks, list) {
- list_del(&lkb->list);
- if (!list_empty(&lkb->trans_list))
- list_del(&lkb->trans_list);
- free(lkb);
- }
- list_del(&r->list);
- free(r);
- }
-}
-
-static void free_transactions(struct lockspace *ls)
-{
- struct trans *tr, *tr_safe;
-
- list_for_each_entry_safe(tr, tr_safe, &ls->transactions, list) {
- list_del(&tr->list);
- if (tr->waitfor)
- free(tr->waitfor);
- free(tr);
- }
-}
-
-static void disable_deadlock(void)
-{
- log_error("FIXME: deadlock detection disabled");
-}
-
-void setup_deadlock(void)
-{
- SaAisErrorT rv;
-
- if (!cfgd_enable_deadlk)
- return;
-
- rv = saCkptInitialize(&global_ckpt_h, &callbacks, &version);
- if (rv != SA_AIS_OK)
- log_error("ckpt init error %d", rv);
-}
-
-static struct dlm_rsb *get_resource(struct lockspace *ls, char *name, int len)
-{
- struct dlm_rsb *r;
-
- list_for_each_entry(r, &ls->resources, list) {
- if (r->len == len && !strncmp(r->name, name, len))
- return r;
- }
-
- r = malloc(sizeof(struct dlm_rsb));
- if (!r) {
- log_error("get_resource: no memory");
- disable_deadlock();
- return NULL;
- }
- memset(r, 0, sizeof(struct dlm_rsb));
- memcpy(r->name, name, len);
- r->len = len;
- INIT_LIST_HEAD(&r->locks);
- list_add(&r->list, &ls->resources);
- return r;
-}
-
-static struct dlm_lkb *create_lkb(void)
-{
- struct dlm_lkb *lkb;
-
- lkb = malloc(sizeof(struct dlm_lkb));
- if (!lkb) {
- log_error("create_lkb: no memory");
- disable_deadlock();
- } else {
- memset(lkb, 0, sizeof(struct dlm_lkb));
- INIT_LIST_HEAD(&lkb->list);
- INIT_LIST_HEAD(&lkb->trans_list);
- }
- return lkb;
-}
-
-static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb)
-{
- list_add(&lkb->list, &r->locks);
- lkb->rsb = r;
-}
-
-/* from linux/fs/dlm/dlm_internal.h */
-#define IFL_MSTCPY 0x00010000
-
-/* called on a lock that's just been read from debugfs */
-
-static void set_copy(struct pack_lock *lock)
-{
- uint32_t id, remid;
-
- if (!lock->nodeid)
- lock->copy = LOCAL_COPY;
- else if (lock->flags & IFL_MSTCPY)
- lock->copy = MASTER_COPY;
- else {
- /* process copy lock is converted to a partial master copy
- lock that will be combined with the real master copy */
- lock->copy = MASTER_COPY;
- id = lock->id;
- remid = lock->remid;
- lock->id = remid;
- lock->remid = id;
- lock->nodeid = our_nodeid;
- }
-}
-
-/* xid is always zero in the real master copy, xid should always be non-zero
- in the partial master copy (what was a process copy) */
-/* TODO: confirm or enforce that the partial will always have non-zero xid */
-
-static int partial_master_copy(struct pack_lock *lock)
-{
- return (lock->xid != 0);
-}
-
-static struct dlm_lkb *get_lkb(struct dlm_rsb *r, struct pack_lock *lock)
-{
- struct dlm_lkb *lkb;
-
- if (lock->copy != MASTER_COPY)
- goto out;
-
- list_for_each_entry(lkb, &r->locks, list) {
- if (lkb->lock.nodeid == lock->nodeid &&
- lkb->lock.id == lock->id)
- return lkb;
- }
- out:
- return create_lkb();
-}
-
-static struct dlm_lkb *add_lock(struct lockspace *ls, struct dlm_rsb *r,
- int from_nodeid, struct pack_lock *lock)
-{
- struct dlm_lkb *lkb;
-
- lkb = get_lkb(r, lock);
- if (!lkb)
- return NULL;
-
- switch (lock->copy) {
- case LOCAL_COPY:
- lkb->lock.xid = lock->xid;
- lkb->lock.nodeid = lock->nodeid;
- lkb->lock.id = lock->id;
- lkb->lock.remid = lock->remid;
- lkb->lock.ownpid = lock->ownpid;
- lkb->lock.exflags = lock->exflags;
- lkb->lock.flags = lock->flags;
- lkb->lock.status = lock->status;
- lkb->lock.grmode = lock->grmode;
- lkb->lock.rqmode = lock->rqmode;
- lkb->lock.copy = LOCAL_COPY;
- lkb->home = from_nodeid;
-
- log_group(ls, "add %s local nodeid %d id %x remid %x xid %llx",
- r->name, lock->nodeid, lock->id, lock->remid,
- (unsigned long long)lock->xid);
- break;
-
- case MASTER_COPY:
- if (partial_master_copy(lock)) {
- lkb->lock.xid = lock->xid;
- lkb->lock.nodeid = lock->nodeid;
- lkb->lock.id = lock->id;
- lkb->lock.remid = lock->remid;
- lkb->lock.copy = MASTER_COPY;
- } else {
- /* only set xid from partial master copy above */
- lkb->lock.nodeid = lock->nodeid;
- lkb->lock.id = lock->id;
- lkb->lock.remid = lock->remid;
- lkb->lock.copy = MASTER_COPY;
- /* set other fields from real master copy */
- lkb->lock.ownpid = lock->ownpid;
- lkb->lock.exflags = lock->exflags;
- lkb->lock.flags = lock->flags;
- lkb->lock.status = lock->status;
- lkb->lock.grmode = lock->grmode;
- lkb->lock.rqmode = lock->rqmode;
- }
- lkb->home = lock->nodeid;
-
- log_group(ls, "add %s master nodeid %d id %x remid %x xid %llx",
- r->name, lock->nodeid, lock->id, lock->remid,
- (unsigned long long)lock->xid);
- break;
- }
-
- if (list_empty(&lkb->list))
- add_lkb(r, lkb);
- return lkb;
-}
-
-static void parse_r_name(char *line, char *name)
-{
- char *p;
- int i = 0;
- int begin = 0;
-
- for (p = line; ; p++) {
- if (*p == '"') {
- if (begin)
- break;
- begin = 1;
- continue;
- }
- if (begin)
- name[i++] = *p;
- }
-}
-
-#define LOCK_LINE_MAX 1024
-
-static int read_debugfs_locks(struct lockspace *ls)
-{
- FILE *file;
- char path[PATH_MAX];
- char line[LOCK_LINE_MAX];
- struct dlm_rsb *r;
- struct pack_lock lock;
- char r_name[65];
- unsigned long long xid;
- unsigned int waiting;
- int r_nodeid;
- int r_len;
- int rv;
-
- snprintf(path, PATH_MAX, "/sys/kernel/debug/dlm/%s_locks", ls->name);
-
- file = fopen(path, "r");
- if (!file)
- return -1;
-
- /* skip the header on the first line */
- if (!fgets(line, LOCK_LINE_MAX, file)) {
- log_error("Unable to read %s: %d", path, errno);
- goto out;
- }
-
- while (fgets(line, LOCK_LINE_MAX, file)) {
- memset(&lock, 0, sizeof(struct pack_lock));
-
- rv = sscanf(line, "%x %d %x %u %llu %x %x %hhd %hhd %hhd %u %d %d",
- &lock.id,
- &lock.nodeid,
- &lock.remid,
- &lock.ownpid,
- &xid,
- &lock.exflags,
- &lock.flags,
- &lock.status,
- &lock.grmode,
- &lock.rqmode,
- &waiting,
- &r_nodeid,
- &r_len);
-
- lock.xid = xid; /* hack to avoid warning */
-
- if (rv != 13) {
- log_error("invalid debugfs line %d: %s", rv, line);
- goto out;
- }
-
- memset(r_name, 0, sizeof(r_name));
- parse_r_name(line, r_name);
-
- r = get_resource(ls, r_name, r_len);
- if (!r)
- break;
-
- set_copy(&lock);
- add_lock(ls, r, our_nodeid, &lock);
- }
- out:
- fclose(file);
- return 0;
-}
-
-static int read_checkpoint_locks(struct lockspace *ls, int from_nodeid,
- char *numbuf, int buflen)
-{
- struct dlm_rsb *r;
- struct pack_lock *lock;
- int count = section_len / sizeof(struct pack_lock);
- int i;
-
- r = get_resource(ls, numbuf, buflen - 1);
- if (!r)
- return -1;
-
- lock = (struct pack_lock *) §ion_buf;
-
- for (i = 0; i < count; i++) {
- lock->xid = le64_to_cpu(lock->xid);
- lock->id = le32_to_cpu(lock->id);
- lock->nodeid = le32_to_cpu(lock->nodeid);
- lock->remid = le32_to_cpu(lock->remid);
- lock->ownpid = le32_to_cpu(lock->ownpid);
- lock->exflags = le32_to_cpu(lock->exflags);
- lock->flags = le32_to_cpu(lock->flags);
-
- add_lock(ls, r, from_nodeid, lock);
- lock++;
- }
- return 0;
-}
-
-static int pack_lkb_list(struct list_head *q, struct pack_lock **lockp)
-{
- struct dlm_lkb *lkb;
- struct pack_lock *lock = *lockp;
- int count = 0;
-
- list_for_each_entry(lkb, q, list) {
- if (count + 1 > section_max) {
- log_error("too many locks %d for ckpt buf", count);
- break;
- }
-
- lock->xid = cpu_to_le64(lkb->lock.xid);
- lock->id = cpu_to_le32(lkb->lock.id);
- lock->nodeid = cpu_to_le32(lkb->lock.nodeid);
- lock->remid = cpu_to_le32(lkb->lock.remid);
- lock->ownpid = cpu_to_le32(lkb->lock.ownpid);
- lock->exflags = cpu_to_le32(lkb->lock.exflags);
- lock->flags = cpu_to_le32(lkb->lock.flags);
- lock->status = lkb->lock.status;
- lock->grmode = lkb->lock.grmode;
- lock->rqmode = lkb->lock.rqmode;
- lock->copy = lkb->lock.copy;
-
- lock++;
- count++;
- }
- return count;
-}
-
-static void pack_section_buf(struct lockspace *ls, struct dlm_rsb *r)
-{
- struct pack_lock *lock;
- int count;
-
- memset(§ion_buf, 0, sizeof(section_buf));
- section_max = sizeof(section_buf) / sizeof(struct pack_lock);
-
- lock = (struct pack_lock *) §ion_buf;
-
- count = pack_lkb_list(&r->locks, &lock);
-
- section_len = count * sizeof(struct pack_lock);
-}
-
-static int _unlink_checkpoint(struct lockspace *ls, SaNameT *name)
-{
- SaCkptCheckpointHandleT h;
- SaCkptCheckpointDescriptorT s;
- SaAisErrorT rv;
- int ret = 0;
- int retries;
-
- h = (SaCkptCheckpointHandleT) ls->deadlk_ckpt_handle;
- log_group(ls, "unlink ckpt %llx", (unsigned long long)h);
-
- retries = 0;
- unlink_retry:
- rv = saCkptCheckpointUnlink(global_ckpt_h, name);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "unlink ckpt retry");
- sleep(1);
- if (retries++ < 10)
- goto unlink_retry;
- }
- if (rv == SA_AIS_OK)
- goto out_close;
- if (!h)
- goto out;
-
- log_error("unlink ckpt error %d %s", rv, ls->name);
- ret = -1;
-
- retries = 0;
- status_retry:
- rv = saCkptCheckpointStatusGet(h, &s);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "unlink ckpt status retry");
- sleep(1);
- if (retries++ < 10)
- goto status_retry;
- }
- if (rv != SA_AIS_OK) {
- log_error("unlink ckpt status error %d %s", rv, ls->name);
- goto out_close;
- }
-
- log_group(ls, "unlink ckpt status: size %llu, max sections %u, "
- "max section size %llu, section count %u, mem %u",
- (unsigned long long)s.checkpointCreationAttributes.checkpointSize,
- s.checkpointCreationAttributes.maxSections,
- (unsigned long long)s.checkpointCreationAttributes.maxSectionSize,
- s.numberOfSections, s.memoryUsed);
-
- out_close:
- retries = 0;
- close_retry:
- rv = saCkptCheckpointClose(h);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "unlink ckpt close retry");
- sleep(1);
- if (retries++ < 10)
- goto close_retry;
- }
- if (rv != SA_AIS_OK) {
- log_error("unlink ckpt %llx close err %d %s",
- (unsigned long long)h, rv, ls->name);
- }
- out:
- ls->deadlk_ckpt_handle = 0;
- return ret;
-}
-
-static int unlink_checkpoint(struct lockspace *ls)
-{
- SaNameT name;
- int len;
-
- len = snprintf((char *)name.value, SA_MAX_NAME_LENGTH, "dlmdeadlk.%s.%d",
- ls->name, our_nodeid);
- name.length = len;
-
- return _unlink_checkpoint(ls, &name);
-}
-
-static void read_checkpoint(struct lockspace *ls, int nodeid)
-{
- SaCkptCheckpointHandleT h;
- SaCkptSectionIterationHandleT itr;
- SaCkptSectionDescriptorT desc;
- SaCkptIOVectorElementT iov;
- SaNameT name;
- SaAisErrorT rv;
- char buf[DLM_RESNAME_MAXLEN];
- int len;
- int retries;
-
- if (nodeid == our_nodeid)
- return;
-
- log_group(ls, "read_checkpoint %d", nodeid);
-
- len = snprintf((char *)name.value, SA_MAX_NAME_LENGTH, "dlmdeadlk.%s.%d",
- ls->name, nodeid);
- name.length = len;
-
- retries = 0;
- open_retry:
- rv = saCkptCheckpointOpen(global_ckpt_h, &name, NULL,
- SA_CKPT_CHECKPOINT_READ, 0, &h);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "read_checkpoint: %d ckpt open retry", nodeid);
- sleep(1);
- if (retries++ < 10)
- goto open_retry;
- }
- if (rv != SA_AIS_OK) {
- log_error("read_checkpoint: %d ckpt open error %d", nodeid, rv);
- return;
- }
-
- retries = 0;
- init_retry:
- rv = saCkptSectionIterationInitialize(h, SA_CKPT_SECTIONS_ANY, 0, &itr);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "read_checkpoint: ckpt iterinit retry");
- sleep(1);
- if (retries++ < 10)
- goto init_retry;
- }
- if (rv != SA_AIS_OK) {
- log_error("read_checkpoint: %d ckpt iterinit error %d", nodeid, rv);
- goto out;
- }
-
- while (1) {
- retries = 0;
- next_retry:
- rv = saCkptSectionIterationNext(itr, &desc);
- if (rv == SA_AIS_ERR_NO_SECTIONS)
- break;
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "read_checkpoint: ckpt iternext retry");
- sleep(1);
- if (retries++ < 10)
- goto next_retry;
- }
- if (rv != SA_AIS_OK) {
- log_error("read_checkpoint: %d ckpt iternext error %d",
- nodeid, rv);
- goto out_it;
- }
-
- if (!desc.sectionSize)
- continue;
-
- iov.sectionId = desc.sectionId;
- iov.dataBuffer = §ion_buf;
- iov.dataSize = desc.sectionSize;
- iov.dataOffset = 0;
-
- memset(&buf, 0, sizeof(buf));
- snprintf(buf, sizeof(buf), "%s", desc.sectionId.id);
-
- log_group(ls, "read_checkpoint: section size %llu id %u \"%s\"",
- (unsigned long long)iov.dataSize,
- iov.sectionId.idLen, buf);
-
- retries = 0;
- read_retry:
- rv = saCkptCheckpointRead(h, &iov, 1, NULL);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "read_checkpoint: ckpt read retry");
- sleep(1);
- if (retries++ < 10)
- goto read_retry;
- }
- if (rv != SA_AIS_OK) {
- log_error("read_checkpoint: %d ckpt read error %d",
- nodeid, rv);
- goto out_it;
- }
-
- section_len = iov.readSize;
-
- if (!section_len)
- continue;
-
- if (section_len % sizeof(struct pack_lock)) {
- log_error("read_checkpoint: %d bad section len %d",
- nodeid, section_len);
- continue;
- }
-
- read_checkpoint_locks(ls, nodeid, (char *)desc.sectionId.id,
- desc.sectionId.idLen);
- }
-
- out_it:
- saCkptSectionIterationFinalize(itr);
- retries = 0;
- out:
- rv = saCkptCheckpointClose(h);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "read_checkpoint: unlink ckpt close retry");
- sleep(1);
- if (retries++ < 10)
- goto out;
- }
- if (rv != SA_AIS_OK)
- log_error("read_checkpoint: %d close error %d", nodeid, rv);
-}
-
-static void write_checkpoint(struct lockspace *ls)
-{
- SaCkptCheckpointCreationAttributesT attr;
- SaCkptCheckpointHandleT h;
- SaCkptSectionIdT section_id;
- SaCkptSectionCreationAttributesT section_attr;
- SaCkptCheckpointOpenFlagsT flags;
- SaNameT name;
- SaAisErrorT rv;
- char buf[DLM_RESNAME_MAXLEN];
- struct dlm_rsb *r;
- struct dlm_lkb *lkb;
- int r_count, lock_count, total_size, section_size, max_section_size;
- int len;
-
- len = snprintf((char *)name.value, SA_MAX_NAME_LENGTH, "dlmdeadlk.%s.%d",
- ls->name, our_nodeid);
- name.length = len;
-
- /* unlink an old checkpoint before we create a new one */
- if (ls->deadlk_ckpt_handle) {
- log_error("write_checkpoint: old ckpt");
- if (_unlink_checkpoint(ls, &name))
- return;
- }
-
- /* loop through all locks to figure out sizes to set in
- the attr fields */
-
- r_count = 0;
- lock_count = 0;
- total_size = 0;
- max_section_size = 0;
-
- list_for_each_entry(r, &ls->resources, list) {
- r_count++;
- section_size = 0;
- list_for_each_entry(lkb, &r->locks, list) {
- section_size += sizeof(struct pack_lock);
- lock_count++;
- }
- total_size += section_size;
- if (section_size > max_section_size)
- max_section_size = section_size;
- }
-
- log_group(ls, "write_checkpoint: r_count %d, lock_count %d",
- r_count, lock_count);
-
- log_group(ls, "write_checkpoint: total %d bytes, max_section %d bytes",
- total_size, max_section_size);
-
- attr.creationFlags = SA_CKPT_WR_ALL_REPLICAS;
- attr.checkpointSize = total_size;
- attr.retentionDuration = SA_TIME_MAX;
- attr.maxSections = r_count + 1; /* don't know why we need +1 */
- attr.maxSectionSize = max_section_size;
- attr.maxSectionIdSize = DLM_RESNAME_MAXLEN;
-
- flags = SA_CKPT_CHECKPOINT_READ |
- SA_CKPT_CHECKPOINT_WRITE |
- SA_CKPT_CHECKPOINT_CREATE;
-
- open_retry:
- rv = saCkptCheckpointOpen(global_ckpt_h, &name, &attr, flags, 0, &h);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "write_checkpoint: ckpt open retry");
- sleep(1);
- goto open_retry;
- }
- if (rv == SA_AIS_ERR_EXIST) {
- log_group(ls, "write_checkpoint: ckpt already exists");
- return;
- }
- if (rv != SA_AIS_OK) {
- log_group(ls, "write_checkpoint: ckpt open error %d", rv);
- return;
- }
-
- log_group(ls, "write_checkpoint: open ckpt handle %llx",
- (unsigned long long)h);
- ls->deadlk_ckpt_handle = (uint64_t) h;
-
- list_for_each_entry(r, &ls->resources, list) {
- memset(buf, 0, sizeof(buf));
- len = snprintf(buf, sizeof(buf), "%s", r->name);
-
- section_id.id = (void *)buf;
- section_id.idLen = len + 1;
- section_attr.sectionId = §ion_id;
- section_attr.expirationTime = SA_TIME_END;
-
- pack_section_buf(ls, r);
-
- log_group(ls, "write_checkpoint: section size %u id %u \"%s\"",
- section_len, section_id.idLen, buf);
-
- create_retry:
- rv = saCkptSectionCreate(h, §ion_attr, §ion_buf,
- section_len);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- log_group(ls, "write_checkpoint: ckpt create retry");
- sleep(1);
- goto create_retry;
- }
- if (rv == SA_AIS_ERR_EXIST) {
- /* this shouldn't happen in general */
- log_error("write_checkpoint: clearing old ckpt");
- saCkptCheckpointClose(h);
- _unlink_checkpoint(ls, &name);
- goto open_retry;
- }
- if (rv != SA_AIS_OK) {
- log_error("write_checkpoint: section create %d", rv);
- break;
- }
- }
-}
-
-static void send_message(struct lockspace *ls, int type,
- uint32_t to_nodeid, uint32_t msgdata)
-{
- struct dlm_header *hd;
- int len;
- char *buf;
-
- len = sizeof(struct dlm_header);
- buf = malloc(len);
- if (!buf) {
- log_error("send_message: no memory");
- disable_deadlock();
- return;
- }
- memset(buf, 0, len);
-
- hd = (struct dlm_header *)buf;
- hd->type = type;
- hd->to_nodeid = to_nodeid;
- hd->msgdata = msgdata;
-
- dlm_send_message(ls, buf, len);
-
- free(buf);
-}
-
-static void send_checkpoint_ready(struct lockspace *ls)
-{
- log_group(ls, "send_checkpoint_ready");
- send_message(ls, DLM_MSG_DEADLK_CHECKPOINT_READY, 0, 0);
-}
-
-void send_cycle_start(struct lockspace *ls)
-{
- log_group(ls, "send_cycle_start");
- send_message(ls, DLM_MSG_DEADLK_CYCLE_START, 0, 0);
-}
-
-static void send_cycle_end(struct lockspace *ls)
-{
- log_group(ls, "send_cycle_end");
- send_message(ls, DLM_MSG_DEADLK_CYCLE_END, 0, 0);
-}
-
-static void send_cancel_lock(struct lockspace *ls, struct trans *tr,
- struct dlm_lkb *lkb)
-{
- int to_nodeid;
- uint32_t lkid;
-
- if (!lkb->lock.nodeid)
- lkid = lkb->lock.id;
- else
- lkid = lkb->lock.remid;
- to_nodeid = lkb->home;
-
- log_group(ls, "send_cancel_lock to nodeid %d rsb %s id %x xid %llx",
- to_nodeid, lkb->rsb->name, lkid,
- (unsigned long long)lkb->lock.xid);
-
- send_message(ls, DLM_MSG_DEADLK_CANCEL_LOCK, to_nodeid, lkid);
-}
-
-static void dump_resources(struct lockspace *ls)
-{
- struct dlm_rsb *r;
- struct dlm_lkb *lkb;
-
- log_group(ls, "Resource dump:");
-
- list_for_each_entry(r, &ls->resources, list) {
- log_group(ls, "\"%s\" len %d", r->name, r->len);
- list_for_each_entry(lkb, &r->locks, list) {
- log_group(ls, " %s: nodeid %d id %08x remid %08x gr %s rq %s pid %u xid
%llx",
- status_str(lkb->lock.status),
- lkb->lock.nodeid,
- lkb->lock.id,
- lkb->lock.remid,
- dlm_mode_str(lkb->lock.grmode),
- dlm_mode_str(lkb->lock.rqmode),
- lkb->lock.ownpid,
- (unsigned long long)lkb->lock.xid);
- }
- }
-}
-
-static void find_deadlock(struct lockspace *ls);
-
-static void run_deadlock(struct lockspace *ls)
-{
- struct node *node;
- int not_ready = 0;
- int low = -1;
-
- if (ls->all_checkpoints_ready)
- log_group(ls, "WARNING: run_deadlock all_checkpoints_ready");
-
- list_for_each_entry(node, &ls->deadlk_nodes, list) {
- if (!node->in_cycle)
- continue;
- if (!node->checkpoint_ready)
- not_ready++;
-
- log_group(ls, "nodeid %d checkpoint_ready = %d",
- node->nodeid, node->checkpoint_ready);
- }
- if (not_ready)
- return;
-
- ls->all_checkpoints_ready = 1;
-
- list_for_each_entry(node, &ls->deadlk_nodes, list) {
- if (!node->in_cycle)
- continue;
- if (node->nodeid < low || low == -1)
- low = node->nodeid;
- }
- ls->deadlk_low_nodeid = low;
-
- if (low == our_nodeid)
- find_deadlock(ls);
- else
- log_group(ls, "defer resolution to low nodeid %d", low);
-}
-
-void receive_checkpoint_ready(struct lockspace *ls, struct dlm_header *hd,
- int len)
-{
- struct node *node;
- int nodeid = hd->nodeid;
-
- log_group(ls, "receive_checkpoint_ready from %d", nodeid);
-
- read_checkpoint(ls, nodeid);
-
- list_for_each_entry(node, &ls->deadlk_nodes, list) {
- if (node->nodeid == nodeid) {
- node->checkpoint_ready = 1;
- break;
- }
- }
-
- run_deadlock(ls);
-}
-
-void receive_cycle_start(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- struct node *node;
- int nodeid = hd->nodeid;
- int rv;
-
- log_group(ls, "receive_cycle_start from %d", nodeid);
-
- if (ls->cycle_running) {
- log_group(ls, "cycle already running");
- return;
- }
- ls->cycle_running = 1;
- gettimeofday(&ls->cycle_start_time, NULL);
-
- list_for_each_entry(node, &ls->deadlk_nodes, list)
- node->in_cycle = 1;
-
- rv = read_debugfs_locks(ls);
- if (rv < 0) {
- log_error("can't read dlm debugfs file: %s", strerror(errno));
- return;
- }
-
- write_checkpoint(ls);
- send_checkpoint_ready(ls);
-}
-
-static uint64_t dt_usec(struct timeval *start, struct timeval *stop)
-{
- uint64_t dt;
-
- dt = stop->tv_sec - start->tv_sec;
- dt *= 1000000;
- dt += stop->tv_usec - start->tv_usec;
- return dt;
-}
-
-/* TODO: nodes added during a cycle - what will they do with messages
- they recv from other nodes running the cycle? */
-
-void receive_cycle_end(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- struct node *node;
- int nodeid = hd->nodeid;
- uint64_t usec;
-
- if (!ls->cycle_running) {
- log_error("receive_cycle_end %s from %d: no cycle running",
- ls->name, nodeid);
- return;
- }
-
- gettimeofday(&ls->cycle_end_time, NULL);
- usec = dt_usec(&ls->cycle_start_time, &ls->cycle_end_time);
- log_group(ls, "receive_cycle_end: from %d cycle time %.2f s",
- nodeid, usec * 1.e-6);
-
- ls->cycle_running = 0;
- ls->all_checkpoints_ready = 0;
-
- list_for_each_entry(node, &ls->deadlk_nodes, list)
- node->checkpoint_ready = 0;
-
- free_resources(ls);
- free_transactions(ls);
- unlink_checkpoint(ls);
-}
-
-void receive_cancel_lock(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- dlm_lshandle_t h;
- int nodeid = hd->nodeid;
- uint32_t lkid = hd->msgdata;
- int rv;
-
- if (nodeid != our_nodeid)
- return;
-
- h = dlm_open_lockspace(ls->name);
- if (!h) {
- log_error("deadlock cancel %x from %d can't open lockspace %s",
- lkid, nodeid, ls->name);
- return;
- }
-
- log_group(ls, "receive_cancel_lock %x from %d", lkid, nodeid);
-
- rv = dlm_ls_deadlock_cancel(h, lkid, 0);
- if (rv < 0) {
- log_error("deadlock cancel %x from %x lib cancel errno %d",
- lkid, nodeid, errno);
- }
-
- dlm_close_lockspace(h);
-}
-
-static void node_joined(struct lockspace *ls, int nodeid)
-{
- struct node *node;
-
- node = malloc(sizeof(struct node));
- if (!node) {
- log_error("node_joined: no memory");
- disable_deadlock();
- return;
- }
- memset(node, 0, sizeof(struct node));
- node->nodeid = nodeid;
- list_add_tail(&node->list, &ls->deadlk_nodes);
- log_group(ls, "node %d joined deadlock cpg", nodeid);
-}
-
-static void node_left(struct lockspace *ls, int nodeid, int reason)
-{
- struct node *node, *safe;
-
- list_for_each_entry_safe(node, safe, &ls->deadlk_nodes, list) {
- if (node->nodeid != nodeid)
- continue;
-
- list_del(&node->list);
- free(node);
- log_group(ls, "node %d left deadlock cpg", nodeid);
- }
-}
-
-static void purge_locks(struct lockspace *ls, int nodeid);
-
-void deadlk_confchg(struct lockspace *ls,
- const struct cpg_address *member_list,
- size_t member_list_entries,
- const struct cpg_address *left_list,
- size_t left_list_entries,
- const struct cpg_address *joined_list,
- size_t joined_list_entries)
-{
- int i;
-
- if (!cfgd_enable_deadlk)
- return;
-
- if (!ls->deadlk_confchg_init) {
- ls->deadlk_confchg_init = 1;
- for (i = 0; i < member_list_entries; i++)
- node_joined(ls, member_list[i].nodeid);
- return;
- }
-
- /* nodes added during a cycle won't have node->in_cycle set so they
- won't be included in any of the cycle processing */
-
- for (i = 0; i < joined_list_entries; i++)
- node_joined(ls, joined_list[i].nodeid);
-
- for (i = 0; i < left_list_entries; i++)
- node_left(ls, left_list[i].nodeid, left_list[i].reason);
-
- if (!ls->cycle_running)
- return;
-
- if (!left_list_entries)
- return;
-
- if (!ls->all_checkpoints_ready) {
- run_deadlock(ls);
- return;
- }
-
- for (i = 0; i < left_list_entries; i++)
- purge_locks(ls, left_list[i].nodeid);
-
- for (i = 0; i < left_list_entries; i++) {
- if (left_list[i].nodeid != ls->deadlk_low_nodeid)
- continue;
- /* this will set a new low node which will call find_deadlock */
- run_deadlock(ls);
- break;
- }
-}
-
-/* would we ever call this after we've created the transaction lists?
- I don't think so; I think it can only be called between reading
- checkpoints */
-
-static void purge_locks(struct lockspace *ls, int nodeid)
-{
- struct dlm_rsb *r;
- struct dlm_lkb *lkb, *safe;
-
- list_for_each_entry(r, &ls->resources, list) {
- list_for_each_entry_safe(lkb, safe, &r->locks, list) {
- if (lkb->home == nodeid) {
- list_del(&lkb->list);
- if (list_empty(&lkb->trans_list))
- free(lkb);
- else
- log_group(ls, "purge %d %x on trans",
- nodeid, lkb->lock.id);
- }
- }
- }
-}
-
-static void add_lkb_trans(struct trans *tr, struct dlm_lkb *lkb)
-{
- list_add(&lkb->trans_list, &tr->locks);
- lkb->trans = tr;
-}
-
-static struct trans *get_trans(struct lockspace *ls, uint64_t xid)
-{
- struct trans *tr;
-
- list_for_each_entry(tr, &ls->transactions, list) {
- if (tr->xid == xid)
- return tr;
- }
-
- tr = malloc(sizeof(struct trans));
- if (!tr) {
- log_error("get_trans: no memory");
- disable_deadlock();
- return NULL;
- }
- memset(tr, 0, sizeof(struct trans));
- tr->xid = xid;
- tr->waitfor = NULL;
- tr->waitfor_alloc = 0;
- tr->waitfor_count = 0;
- INIT_LIST_HEAD(&tr->locks);
- list_add(&tr->list, &ls->transactions);
- return tr;
-}
-
-/* for each rsb, for each lock, find/create trans, add lkb to the trans list */
-
-static void create_trans_list(struct lockspace *ls)
-{
- struct dlm_rsb *r;
- struct dlm_lkb *lkb;
- struct trans *tr;
- int r_count = 0, lkb_count = 0;
-
- list_for_each_entry(r, &ls->resources, list) {
- r_count++;
- list_for_each_entry(lkb, &r->locks, list) {
- lkb_count++;
- tr = get_trans(ls, lkb->lock.xid);
- if (!tr)
- goto out;
- add_lkb_trans(tr, lkb);
- }
- }
- out:
- log_group(ls, "create_trans_list: r_count %d lkb_count %d",
- r_count, lkb_count);
-}
-
-static int locks_compat(struct dlm_lkb *waiting_lkb,
- struct dlm_lkb *granted_lkb)
-{
- if (waiting_lkb == granted_lkb) {
- log_debug("waiting and granted same lock");
- return 0;
- }
-
- if (waiting_lkb->trans->xid == granted_lkb->trans->xid) {
- log_debug("waiting and granted same trans %llx",
- (unsigned long long)waiting_lkb->trans->xid);
- return 0;
- }
-
- return dlm_modes_compat(granted_lkb->lock.grmode,
- waiting_lkb->lock.rqmode);
-}
-
-static int in_waitfor(struct trans *tr, struct trans *add_tr)
-{
- int i;
-
- for (i = 0; i < tr->waitfor_alloc; i++) {
- if (!tr->waitfor[i])
- continue;
- if (tr->waitfor[i] == add_tr)
- return 1;
- }
- return 0;
-}
-
-static void add_waitfor(struct lockspace *ls, struct dlm_lkb *waiting_lkb,
- struct dlm_lkb *granted_lkb)
-{
- struct trans *tr = waiting_lkb->trans;
- int i;
-
- if (locks_compat(waiting_lkb, granted_lkb))
- return;
-
- /* this shouldn't happen AFAIK */
- if (tr == granted_lkb->trans) {
- log_group(ls, "trans %llx waiting on self",
- (unsigned long long)tr->xid);
- return;
- }
-
- /* don't add the same trans to the waitfor list multiple times */
- if (tr->waitfor_count && in_waitfor(tr, granted_lkb->trans)) {
- log_group(ls, "trans %llx already waiting for trans %llx, "
- "waiting %x %s, granted %x %s",
- (unsigned long long)waiting_lkb->trans->xid,
- (unsigned long long)granted_lkb->trans->xid,
- waiting_lkb->lock.id, waiting_lkb->rsb->name,
- granted_lkb->lock.id, granted_lkb->rsb->name);
- return;
- }
-
- if (tr->waitfor_count == tr->waitfor_alloc) {
- struct trans **old_waitfor = tr->waitfor;
- tr->waitfor_alloc += TR_NALLOC;
- tr->waitfor = malloc(tr->waitfor_alloc * sizeof(tr));
- if (!tr->waitfor) {
- log_error("add_waitfor no mem %u", tr->waitfor_alloc);
- return;
- }
- memset(tr->waitfor, 0, tr->waitfor_alloc * sizeof(tr));
-
- /* copy then free old set of pointers */
- for (i = 0; i < tr->waitfor_count; i++)
- tr->waitfor[i] = old_waitfor[i];
- if (old_waitfor)
- free(old_waitfor);
- }
-
- tr->waitfor[tr->waitfor_count++] = granted_lkb->trans;
- granted_lkb->trans->others_waiting_on_us++;
- waiting_lkb->waitfor_trans = granted_lkb->trans;
-}
-
-/* for each trans, for each waiting lock, go to rsb of the lock,
- find granted locks on that rsb, then find the trans the
- granted lock belongs to, add that trans to our waitfor list */
-
-static void create_waitfor_graph(struct lockspace *ls)
-{
- struct dlm_lkb *waiting_lkb, *granted_lkb;
- struct dlm_rsb *r;
- struct trans *tr;
- int depend_count = 0;
-
- list_for_each_entry(tr, &ls->transactions, list) {
- list_for_each_entry(waiting_lkb, &tr->locks, trans_list) {
- if (waiting_lkb->lock.status == DLM_LKSTS_GRANTED)
- continue;
- /* waiting_lkb status is CONVERT or WAITING */
-
- r = waiting_lkb->rsb;
-
- list_for_each_entry(granted_lkb, &r->locks, list) {
- if (granted_lkb->lock.status==DLM_LKSTS_WAITING)
- continue;
- /* granted_lkb status is GRANTED or CONVERT */
- add_waitfor(ls, waiting_lkb, granted_lkb);
- depend_count++;
- }
- }
- }
-
- log_group(ls, "create_waitfor_graph: depend_count %d", depend_count);
-}
-
-/* Assume a transaction that's not waiting on any locks will complete, release
- all the locks it currently holds, and exit. Other transactions that were
- blocked waiting on the removed transaction's now-released locks may now be
- unblocked, complete, release all held locks and exit. Repeat this until
- no more transactions can be removed. If there are transactions remaining,
- then they are deadlocked. */
-
-static void remove_waitfor(struct trans *tr, struct trans *remove_tr)
-{
- int i;
-
- for (i = 0; i < tr->waitfor_alloc; i++) {
- if (!tr->waitfor_count)
- break;
-
- if (!tr->waitfor[i])
- continue;
-
- if (tr->waitfor[i] == remove_tr) {
- tr->waitfor[i] = NULL;
- tr->waitfor_count--;
- remove_tr->others_waiting_on_us--;
- }
- }
-}
-
-/* remove_tr is not waiting for anything, assume it completes and goes away
- and remove it from any other transaction's waitfor list */
-
-static void remove_trans(struct lockspace *ls, struct trans *remove_tr)
-{
- struct trans *tr;
-
- list_for_each_entry(tr, &ls->transactions, list) {
- if (tr == remove_tr)
- continue;
- if (!remove_tr->others_waiting_on_us)
- break;
- remove_waitfor(tr, remove_tr);
- }
-
- if (remove_tr->others_waiting_on_us)
- log_group(ls, "trans %llx removed others waiting %d",
- (unsigned long long)remove_tr->xid,
- remove_tr->others_waiting_on_us);
-}
-
-static int reduce_waitfor_graph(struct lockspace *ls)
-{
- struct trans *tr, *safe;
- int blocked = 0;
- int removed = 0;
-
- list_for_each_entry_safe(tr, safe, &ls->transactions, list) {
- if (tr->waitfor_count) {
- blocked++;
- continue;
- }
- remove_trans(ls, tr);
- list_del(&tr->list);
- if (tr->waitfor)
- free(tr->waitfor);
- free(tr);
- removed++;
- }
-
- log_group(ls, "reduce_waitfor_graph: %d blocked, %d removed",
- blocked, removed);
- return removed;
-}
-
-static void reduce_waitfor_graph_loop(struct lockspace *ls)
-{
- int removed;
-
- while (1) {
- removed = reduce_waitfor_graph(ls);
- if (!removed)
- break;
- }
-}
-
-static struct trans *find_trans_to_cancel(struct lockspace *ls)
-{
- struct trans *tr;
-
- list_for_each_entry(tr, &ls->transactions, list) {
- if (!tr->others_waiting_on_us)
- continue;
- return tr;
- }
- return NULL;
-}
-
-static void cancel_trans(struct lockspace *ls)
-{
- struct trans *tr;
- struct dlm_lkb *lkb;
- int removed;
-
- tr = find_trans_to_cancel(ls);
- if (!tr) {
- log_group(ls, "cancel_trans: no trans found");
- return;
- }
-
- list_for_each_entry(lkb, &tr->locks, trans_list) {
- if (lkb->lock.status == DLM_LKSTS_GRANTED)
- continue;
- send_cancel_lock(ls, tr, lkb);
-
- /* When this canceled trans has multiple locks all blocked by
- locks held by one other trans, that other trans is only
- added to tr->waitfor once, and only one of these waiting
- locks will have waitfor_trans set. So, the lkb with
- non-null waitfor_trans was the first one responsible
- for adding waitfor_trans to tr->waitfor.
-
- We could potentially forget about keeping track of lkb->
- waitfor_trans, forget about calling remove_waitfor()
- here and just set tr->waitfor_count = 0 after this loop.
- The loss would be that waitfor_trans->others_waiting_on_us
- would not get decremented. */
-
- if (lkb->waitfor_trans)
- remove_waitfor(tr, lkb->waitfor_trans);
- }
-
- /* this shouldn't happen, if it does something's not working right */
- if (tr->waitfor_count) {
- log_group(ls, "cancel_trans: %llx non-zero waitfor_count %d",
- (unsigned long long)tr->xid, tr->waitfor_count);
- }
-
- /* this should now remove the canceled trans since it now has a zero
- waitfor_count */
- removed = reduce_waitfor_graph(ls);
-
- if (!removed)
- log_group(ls, "canceled trans not removed from graph");
-
- /* now call reduce_waitfor_graph() in another loop and it
- should completely reduce */
-}
-
-static void dump_trans(struct lockspace *ls, struct trans *tr)
-{
- struct dlm_lkb *lkb;
- struct trans *wf;
- int i;
-
- log_group(ls, "trans xid %llx waitfor_count %d others_waiting_on_us %d",
- (unsigned long long)tr->xid, tr->waitfor_count,
- tr->others_waiting_on_us);
-
- log_group(ls, "locks:");
-
- list_for_each_entry(lkb, &tr->locks, trans_list) {
- log_group(ls, " %s: id %08x gr %s rq %s pid %u:%u \"%s\"",
- status_str(lkb->lock.status),
- lkb->lock.id,
- dlm_mode_str(lkb->lock.grmode),
- dlm_mode_str(lkb->lock.rqmode),
- lkb->home,
- lkb->lock.ownpid,
- lkb->rsb->name);
- }
-
- if (!tr->waitfor_count)
- return;
-
- log_group(ls, "waitfor:");
-
- for (i = 0; i < tr->waitfor_alloc; i++) {
- if (!tr->waitfor[i])
- continue;
- wf = tr->waitfor[i];
- log_group(ls, " xid %llx", (unsigned long long)wf->xid);
- }
-}
-
-static void dump_all_trans(struct lockspace *ls)
-{
- struct trans *tr;
-
- log_group(ls, "Transaction dump:");
-
- list_for_each_entry(tr, &ls->transactions, list)
- dump_trans(ls, tr);
-}
-
-static void find_deadlock(struct lockspace *ls)
-{
- if (list_empty(&ls->resources)) {
- log_group(ls, "no deadlock: no resources");
- goto out;
- }
-
- if (!list_empty(&ls->transactions)) {
- log_group(ls, "transactions list should be empty");
- goto out;
- }
-
- dump_resources(ls);
- create_trans_list(ls);
- create_waitfor_graph(ls);
- dump_all_trans(ls);
- reduce_waitfor_graph_loop(ls);
-
- if (list_empty(&ls->transactions)) {
- log_group(ls, "no deadlock: all transactions reduced");
- goto out;
- }
-
- log_group(ls, "found deadlock");
- dump_all_trans(ls);
-
- cancel_trans(ls);
- reduce_waitfor_graph_loop(ls);
-
- if (list_empty(&ls->transactions)) {
- log_group(ls, "resolved deadlock with cancel");
- goto out;
- }
-
- log_error("deadlock resolution failed");
- dump_all_trans(ls);
- out:
- send_cycle_end(ls);
-}
-
diff --git a/group/dlm_controld/dlm_controld.h b/group/dlm_controld/dlm_controld.h
deleted file mode 100644
index 73e4ecc..0000000
--- a/group/dlm_controld/dlm_controld.h
+++ /dev/null
@@ -1,38 +0,0 @@
-#ifndef __DLM_CONTROLD_DOT_H__
-#define __DLM_CONTROLD_DOT_H__
-
-/* This defines the interface between dlm_controld and libdlmcontrol, and
- should only be used by libdlmcontrol. */
-
-#define DLMC_SOCK_PATH "dlmc_sock"
-#define DLMC_QUERY_SOCK_PATH "dlmc_query_sock"
-
-#define DLMC_MAGIC 0xD13CD13C
-#define DLMC_VERSION 0x00010001
-
-#define DLMC_CMD_DUMP_DEBUG 1
-#define DLMC_CMD_DUMP_PLOCKS 2
-#define DLMC_CMD_LOCKSPACE_INFO 3
-#define DLMC_CMD_NODE_INFO 4
-#define DLMC_CMD_LOCKSPACES 5
-#define DLMC_CMD_LOCKSPACE_NODES 6
-#define DLMC_CMD_FS_REGISTER 7
-#define DLMC_CMD_FS_UNREGISTER 8
-#define DLMC_CMD_FS_NOTIFIED 9
-#define DLMC_CMD_DEADLOCK_CHECK 10
-#define DLMC_CMD_DUMP_LOG_PLOCK 11
-
-struct dlmc_header {
- unsigned int magic;
- unsigned int version;
- unsigned int command;
- unsigned int option;
- unsigned int len;
- int data; /* embedded command-specific data, for convenience */
- int unused1;
- int unsued2;
- char name[DLM_LOCKSPACE_LEN]; /* no terminating null space */
-};
-
-#endif
-
diff --git a/group/dlm_controld/dlm_daemon.h b/group/dlm_controld/dlm_daemon.h
deleted file mode 100644
index 0c89f67..0000000
--- a/group/dlm_controld/dlm_daemon.h
+++ /dev/null
@@ -1,326 +0,0 @@
-#ifndef __DLM_DAEMON_DOT_H__
-#define __DLM_DAEMON_DOT_H__
-
-#include <sys/types.h>
-#include <asm/types.h>
-#include <sys/uio.h>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
-#include <sys/utsname.h>
-#include <sys/poll.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <net/if.h>
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <stdarg.h>
-#include <fcntl.h>
-#include <netdb.h>
-#include <limits.h>
-#include <unistd.h>
-#include <time.h>
-#include <syslog.h>
-#include <sched.h>
-#include <signal.h>
-#include <sys/time.h>
-#include <dirent.h>
-
-#include <corosync/cpg.h>
-#include <liblogthread.h>
-
-#include <linux/dlmconstants.h>
-#include "libdlmcontrol.h"
-#include "dlm_controld.h"
-#include "config.h"
-#include "list.h"
-#include "rbtree.h"
-#include "linux_endian.h"
-
-/* TODO: cleanup */
-#define CLUSTERVARLIB "/var/lib/cluster"
-#define CLUSTERVARRUN "/var/run/cluster"
-#define LOGDIR "/var/log/cluster"
-#define VERSION "master"
-#define SYSLOGFACILITY LOG_LOCAL4
-#define SYSLOGLEVEL LOG_INFO
-#define REDHAT_COPYRIGHT "Copyright (C) Red Hat, Inc. 2004-2011 All rights
reserved."
-#define LOCKFILE_NAME CLUSTERVARRUN "/dlm_controld.pid"
-#define DAEMON_NAME "dlm_controld"
-#define DEFAULT_LOG_MODE LOG_MODE_OUTPUT_FILE|LOG_MODE_OUTPUT_SYSLOG
-#define DEFAULT_SYSLOG_FACILITY SYSLOGFACILITY
-#define DEFAULT_SYSLOG_PRIORITY SYSLOGLEVEL
-#define DEFAULT_LOGFILE_PRIORITY LOG_INFO /* ? */
-#define DEFAULT_LOGFILE LOGDIR "/" DAEMON_NAME ".log"
-#define DLM_CONFIG_FILE "/etc/dlm.conf"
-
-
-/* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h.
- Copied in libdlm.h so apps don't need to include the kernel header.
- The libcpg limit is larger at CPG_MAX_NAME_LENGTH 128. Our cpg name includes
- a "dlm:" prefix before the lockspace name. */
-
-/* Maximum members of a ls, should match CPG_MEMBERS_MAX in corosync/cpg.h.
- There are no max defines in dlm-kernel for lockspace members. */
-
-#define MAX_NODES 128
-
-/* Maximum number of IP addresses per node, when using SCTP and multi-ring in
- corosync In dlm-kernel this is DLM_MAX_ADDR_COUNT, currently 3. */
-
-#define MAX_NODE_ADDRESSES 4
-
-/* Max string length printed on a line, for debugging/dump output. */
-
-#define MAXLINE 256
-
-/* cfgk_protocol */
-
-#define PROTO_TCP 0
-#define PROTO_SCTP 1
-#define PROTO_DETECT 2
-
-extern int daemon_debug_opt;
-extern int daemon_quit;
-extern int cluster_down;
-extern int poll_fencing;
-extern int poll_quorum;
-extern int poll_fs;
-extern int poll_ignore_plock;
-extern int poll_drop_plock;
-extern int plock_fd;
-extern int plock_ci;
-extern struct list_head lockspaces;
-extern int cluster_quorate;
-extern uint32_t cluster_ringid_seq;
-extern int our_nodeid;
-extern uint32_t control_minor;
-extern uint32_t monitor_minor;
-extern uint32_t plock_minor;
-extern uint32_t old_plock_minor;
-
-#define LOG_DUMP_SIZE DLMC_DUMP_SIZE
-
-#define LOG_PLOCK 0x00010000
-
-void log_level(char *name_in, uint32_t level_in, const char *fmt, ...);
-
-#define log_error(fmt, args...) log_level(NULL, LOG_ERR, fmt, ##args)
-#define log_debug(fmt, args...) log_level(NULL, LOG_DEBUG, fmt, ##args)
-#define log_group(ls, fmt, args...) log_level((ls)->name, LOG_DEBUG, fmt, ##args)
-
-#define log_plock(ls, fmt, args...) log_level((ls)->name, LOG_PLOCK, fmt, ##args)
-#define log_dlock(ls, fmt, args...) log_level((ls)->name, LOG_PLOCK|LOG_DEBUG, fmt,
##args)
-#define log_elock(ls, fmt, args...) log_level((ls)->name, LOG_PLOCK|LOG_ERR, fmt,
##args)
-
-/* dlm_header types */
-enum {
- DLM_MSG_PROTOCOL = 1,
- DLM_MSG_START,
- DLM_MSG_PLOCK,
- DLM_MSG_PLOCK_OWN,
- DLM_MSG_PLOCK_DROP,
- DLM_MSG_PLOCK_SYNC_LOCK,
- DLM_MSG_PLOCK_SYNC_WAITER,
- DLM_MSG_PLOCKS_DONE,
- DLM_MSG_PLOCKS_DATA,
- DLM_MSG_DEADLK_CYCLE_START,
- DLM_MSG_DEADLK_CYCLE_END,
- DLM_MSG_DEADLK_CHECKPOINT_READY,
- DLM_MSG_DEADLK_CANCEL_LOCK
-};
-
-/* dlm_header flags */
-#define DLM_MFLG_JOINING 1 /* accompanies start, we are joining */
-#define DLM_MFLG_HAVEPLOCK 2 /* accompanies start, we have plock state */
-#define DLM_MFLG_NACK 4 /* accompanies start, prevent wrong match when
- two outstanding changes are the same */
-#define DLM_MFLG_PLOCK_SIG 8 /* msgdata2 is a plock signature */
-
-struct dlm_header {
- uint16_t version[3];
- uint16_t type; /* DLM_MSG_ */
- uint32_t nodeid; /* sender */
- uint32_t to_nodeid; /* recipient, 0 for all */
- uint32_t global_id; /* global unique id for this lockspace */
- uint32_t flags; /* DLM_MFLG_ */
- uint32_t msgdata; /* in-header payload depends on MSG type; lkid
- for deadlock, seq for lockspace membership */
- uint32_t msgdata2; /* second MSG-specific data */
- uint64_t pad;
-};
-
-struct lockspace {
- struct list_head list;
- char name[DLM_LOCKSPACE_LEN+1];
- uint32_t global_id;
-
- /* lockspace membership stuff */
-
- cpg_handle_t cpg_handle;
- int cpg_client;
- int cpg_fd;
- int joining;
- int leaving;
- int kernel_stopped;
- int fs_registered;
- uint32_t change_seq;
- uint32_t started_count;
- struct change *started_change;
- struct list_head changes;
- struct list_head node_history;
-
- /* plock stuff */
-
- int plock_data_node;
- int need_plocks;
- int save_plocks;
- int disable_plock;
- uint32_t recv_plocks_data_count;
- uint32_t associated_mg_id;
- struct list_head saved_messages;
- struct list_head plock_resources;
- struct rb_root plock_resources_root;
- time_t last_checkpoint_time;
- time_t last_plock_time;
- struct timeval drop_resources_last;
-
-#if 0
- /* deadlock stuff */
-
- int deadlk_low_nodeid;
- struct list_head deadlk_nodes;
- uint64_t deadlk_ckpt_handle;
- int deadlk_confchg_init;
- struct list_head transactions;
- struct list_head resources;
- struct timeval cycle_start_time;
- struct timeval cycle_end_time;
- struct timeval last_send_cycle_start;
- int cycle_running;
- int all_checkpoints_ready;
-#endif
-};
-
-/* action.c */
-void set_associated_id(uint32_t mg_id);
-int set_sysfs_control(char *name, int val);
-int set_sysfs_event_done(char *name, int val);
-int set_sysfs_id(char *name, uint32_t id);
-int set_configfs_members(char *name, int new_count, int *new_members,
- int renew_count, int *renew_members);
-int add_configfs_node(int nodeid, char *addr, int addrlen, int local);
-void del_configfs_node(int nodeid);
-void clear_configfs(void);
-int setup_configfs(void);
-int check_uncontrolled_lockspaces(void);
-int setup_misc_devices(void);
-int path_exists(const char *path);
-
-/* config.c */
-void setup_config(int update);
-int get_weight(int nodeid, char *lockspace);
-
-/* cpg.c */
-int setup_cpg_daemon(void);
-void close_cpg_daemon(void);
-void process_cpg_daemon(int ci);
-int set_protocol(void);
-void process_lockspace_changes(void);
-void dlm_send_message(struct lockspace *ls, char *buf, int len);
-int dlm_join_lockspace(struct lockspace *ls);
-int dlm_leave_lockspace(struct lockspace *ls);
-const char *msg_name(int type);
-void update_flow_control_status(void);
-void node_history_cluster_add(int nodeid);
-void node_history_cluster_remove(int nodeid);
-int set_node_info(struct lockspace *ls, int nodeid, struct dlmc_node *node);
-int set_lockspace_info(struct lockspace *ls, struct dlmc_lockspace *lockspace);
-int set_lockspaces(int *count, struct dlmc_lockspace **lss_out);
-int set_lockspace_nodes(struct lockspace *ls, int option, int *node_count,
- struct dlmc_node **nodes_out);
-int set_fs_notified(struct lockspace *ls, int nodeid);
-
-/* deadlock.c */
-void setup_deadlock(void);
-void send_cycle_start(struct lockspace *ls);
-void receive_checkpoint_ready(struct lockspace *ls, struct dlm_header *hd,
- int len);
-void receive_cycle_start(struct lockspace *ls, struct dlm_header *hd, int len);
-void receive_cycle_end(struct lockspace *ls, struct dlm_header *hd, int len);
-void receive_cancel_lock(struct lockspace *ls, struct dlm_header *hd, int len);
-void deadlk_confchg(struct lockspace *ls,
- const struct cpg_address *member_list,
- size_t member_list_entries,
- const struct cpg_address *left_list,
- size_t left_list_entries,
- const struct cpg_address *joined_list,
- size_t joined_list_entries);
-
-/* main.c */
-int do_read(int fd, void *buf, size_t count);
-int do_write(int fd, void *buf, size_t count);
-void client_dead(int ci);
-int client_add(int fd, void (*workfn)(int ci), void (*deadfn)(int ci));
-int client_fd(int ci);
-void client_ignore(int ci, int fd);
-void client_back(int ci, int fd);
-struct lockspace *find_ls(char *name);
-struct lockspace *find_ls_id(uint32_t id);
-const char *dlm_mode_str(int mode);
-void cluster_dead(int ci);
-
-/* member_cman.c */
-int setup_cluster(void);
-void close_cluster(void);
-void process_cluster(int ci);
-void update_cluster(void);
-int is_cluster_member(uint32_t nodeid);
-int setup_cluster_cfg(void);
-void close_cluster_cfg(void);
-void process_cluster_cfg(int ci);
-void kick_node_from_cluster(int nodeid);
-int fence_node_time(int nodeid, uint64_t *last_fenced_time);
-int fence_in_progress(int *count);
-
-/* netlink.c */
-int setup_netlink(void);
-void process_netlink(int ci);
-
-/* plock.c */
-int setup_plocks(void);
-void close_plocks(void);
-void process_plocks(int ci);
-void drop_resources_all(void);
-int limit_plocks(void);
-void receive_plock(struct lockspace *ls, struct dlm_header *hd, int len);
-void receive_own(struct lockspace *ls, struct dlm_header *hd, int len);
-void receive_sync(struct lockspace *ls, struct dlm_header *hd, int len);
-void receive_drop(struct lockspace *ls, struct dlm_header *hd, int len);
-void process_saved_plocks(struct lockspace *ls);
-void purge_plocks(struct lockspace *ls, int nodeid, int unmount);
-int copy_plock_state(struct lockspace *ls, char *buf, int *len_out);
-
-void send_all_plocks_data(struct lockspace *ls, uint32_t seq, uint32_t *plocks_data);
-void receive_plocks_data(struct lockspace *ls, struct dlm_header *hd, int len);
-void clear_plocks_data(struct lockspace *ls);
-
-/* logging.c */
-
-void init_logging(void);
-void setup_logging(void);
-void close_logging(void);
-void copy_log_dump(char *buf, int *len);
-void copy_log_dump_plock(char *buf, int *len);
-
-/* crc.c */
-uint32_t cpgname_to_crc(const char *data, int len);
-
-#endif
-
diff --git a/group/dlm_controld/logging.c b/group/dlm_controld/logging.c
deleted file mode 100644
index e6e82b7..0000000
--- a/group/dlm_controld/logging.c
+++ /dev/null
@@ -1,173 +0,0 @@
-#include "dlm_daemon.h"
-
-static int log_mode;
-static int syslog_facility;
-static int syslog_priority;
-static int logfile_priority;
-static char logfile[PATH_MAX];
-
-void init_logging(void)
-{
- log_mode = DEFAULT_LOG_MODE;
- syslog_facility = DEFAULT_SYSLOG_FACILITY;
- syslog_priority = DEFAULT_SYSLOG_PRIORITY;
- logfile_priority = DEFAULT_LOGFILE_PRIORITY;
- strcpy(logfile, DEFAULT_LOGFILE);
-
- /* logfile_priority is the only one of these options that
- can be controlled from command line or environment variable */
-
- if (cfgd_debug_logfile)
- logfile_priority = LOG_DEBUG;
-
- log_debug("logging mode %d syslog f %d p %d logfile p %d %s",
- log_mode, syslog_facility, syslog_priority,
- logfile_priority, logfile);
-
- logt_init(DAEMON_NAME, log_mode, syslog_facility, syslog_priority,
- logfile_priority, logfile);
-}
-
-void setup_logging(void)
-{
- /* TODO */
- /*
- ccs_read_logging(ccs_handle, DAEMON_NAME,
- &cfgd_debug_logfile, &log_mode,
- &syslog_facility, &syslog_priority,
- &logfile_priority, logfile);
- */
-
- log_debug("logging mode %d syslog f %d p %d logfile p %d %s",
- log_mode, syslog_facility, syslog_priority,
- logfile_priority, logfile);
-
- logt_conf(DAEMON_NAME, log_mode, syslog_facility, syslog_priority,
- logfile_priority, logfile);
-}
-
-void close_logging(void)
-{
- logt_exit();
-}
-
-#define NAME_ID_SIZE 32
-#define LOG_STR_LEN 512
-static char log_str[LOG_STR_LEN];
-
-static char log_dump[LOG_DUMP_SIZE];
-static unsigned int log_point;
-static unsigned int log_wrap;
-
-static char log_dump_plock[LOG_DUMP_SIZE];
-static unsigned int log_point_plock;
-static unsigned int log_wrap_plock;
-
-static void log_copy(char *buf, int *len, char *log_buf,
- unsigned int *point, unsigned int *wrap)
-{
- unsigned int p = *point;
- unsigned int w = *wrap;
- int tail_len;
-
- if (!w && !p) {
- *len = 0;
- } else if (*wrap) {
- tail_len = LOG_DUMP_SIZE - p;
- memcpy(buf, log_buf + p, tail_len);
- if (p)
- memcpy(buf+tail_len, log_buf, p);
- *len = LOG_DUMP_SIZE;
- } else {
- memcpy(buf, log_buf, p-1);
- *len = p-1;
- }
-}
-
-void copy_log_dump(char *buf, int *len)
-{
- log_copy(buf, len, log_dump, &log_point, &log_wrap);
-}
-
-void copy_log_dump_plock(char *buf, int *len)
-{
- log_copy(buf, len, log_dump_plock, &log_point_plock, &log_wrap_plock);
-}
-
-static void log_save_str(int level, int len, char *log_buf,
- unsigned int *point, unsigned int *wrap)
-{
- unsigned int p = *point;
- unsigned int w = *wrap;
- int i;
-
- if (len < LOG_DUMP_SIZE - p) {
- memcpy(log_buf + p, log_str, len);
- p += len;
-
- if (p == LOG_DUMP_SIZE) {
- p = 0;
- w = 1;
- }
- goto out;
- }
-
- for (i = 0; i < len; i++) {
- log_buf[p++] = log_str[i];
-
- if (p == LOG_DUMP_SIZE) {
- p = 0;
- w = 1;
- }
- }
- out:
- *point = p;
- *wrap = w;
-}
-
-void log_level(char *name_in, uint32_t level_in, const char *fmt, ...)
-{
- va_list ap;
- char name[NAME_ID_SIZE + 1];
- uint32_t level = level_in & 0x0000FFFF;
- uint32_t extra = level_in & 0xFFFF0000;
- int ret, pos = 0;
- int len = LOG_STR_LEN - 2;
- int plock = extra & LOG_PLOCK;
-
- memset(name, 0, sizeof(name));
-
- if (name_in)
- snprintf(name, NAME_ID_SIZE, "%s ", name_in);
-
- ret = snprintf(log_str + pos, len - pos, "%llu %s",
- (unsigned long long)time(NULL), name);
-
- pos += ret;
-
- va_start(ap, fmt);
- ret = vsnprintf(log_str + pos, len - pos, fmt, ap);
- va_end(ap);
-
- if (ret >= len - pos)
- pos = len - 1;
- else
- pos += ret;
-
- log_str[pos++] = '\n';
- log_str[pos++] = '\0';
-
- if (level)
- log_save_str(level, pos - 1, log_dump, &log_point, &log_wrap);
- if (plock)
- log_save_str(level, pos - 1, log_dump_plock, &log_point_plock,
&log_wrap_plock);
- if (level)
- logt_print(level, "%s", log_str);
-
- if (!daemon_debug_opt)
- return;
-
- if (level || (plock && cfgd_plock_debug))
- fprintf(stderr, "%s", log_str);
-}
-
diff --git a/group/dlm_controld/main.c b/group/dlm_controld/main.c
deleted file mode 100644
index b0d9b03..0000000
--- a/group/dlm_controld/main.c
+++ /dev/null
@@ -1,1274 +0,0 @@
-#include "dlm_daemon.h"
-#include <pthread.h>
-#include <linux/netlink.h>
-#include <linux/genetlink.h>
-#include <linux/dlm_netlink.h>
-
-#define CLIENT_NALLOC 32
-static int client_maxi;
-static int client_size = 0;
-static struct client *client = NULL;
-static struct pollfd *pollfd = NULL;
-static pthread_t query_thread;
-static pthread_mutex_t query_mutex;
-static struct list_head fs_register_list;
-static int kernel_monitor_fd;
-
-struct client {
- int fd;
- void *workfn;
- void *deadfn;
- struct lockspace *ls;
-};
-
-int do_read(int fd, void *buf, size_t count)
-{
- int rv, off = 0;
-
- while (off < count) {
- rv = read(fd, (char *)buf + off, count - off);
- if (rv == 0)
- return -1;
- if (rv == -1 && errno == EINTR)
- continue;
- if (rv == -1)
- return -1;
- off += rv;
- }
- return 0;
-}
-
-int do_write(int fd, void *buf, size_t count)
-{
- int rv, off = 0;
-
- retry:
- rv = write(fd, (char *)buf + off, count);
- if (rv == -1 && errno == EINTR)
- goto retry;
- if (rv < 0) {
- log_error("write errno %d", errno);
- return rv;
- }
-
- if (rv != count) {
- count -= rv;
- off += rv;
- goto retry;
- }
- return 0;
-}
-
-static void client_alloc(void)
-{
- int i;
-
- if (!client) {
- client = malloc(CLIENT_NALLOC * sizeof(struct client));
- pollfd = malloc(CLIENT_NALLOC * sizeof(struct pollfd));
- } else {
- client = realloc(client, (client_size + CLIENT_NALLOC) *
- sizeof(struct client));
- pollfd = realloc(pollfd, (client_size + CLIENT_NALLOC) *
- sizeof(struct pollfd));
- if (!pollfd)
- log_error("can't alloc for pollfd");
- }
- if (!client || !pollfd)
- log_error("can't alloc for client array");
-
- for (i = client_size; i < client_size + CLIENT_NALLOC; i++) {
- client[i].workfn = NULL;
- client[i].deadfn = NULL;
- client[i].fd = -1;
- pollfd[i].fd = -1;
- pollfd[i].revents = 0;
- }
- client_size += CLIENT_NALLOC;
-}
-
-void client_dead(int ci)
-{
- close(client[ci].fd);
- client[ci].workfn = NULL;
- client[ci].fd = -1;
- pollfd[ci].fd = -1;
-}
-
-int client_add(int fd, void (*workfn)(int ci), void (*deadfn)(int ci))
-{
- int i;
-
- if (!client)
- client_alloc();
- again:
- for (i = 0; i < client_size; i++) {
- if (client[i].fd == -1) {
- client[i].workfn = workfn;
- if (deadfn)
- client[i].deadfn = deadfn;
- else
- client[i].deadfn = client_dead;
- client[i].fd = fd;
- pollfd[i].fd = fd;
- pollfd[i].events = POLLIN;
- if (i > client_maxi)
- client_maxi = i;
- return i;
- }
- }
-
- client_alloc();
- goto again;
-}
-
-int client_fd(int ci)
-{
- return client[ci].fd;
-}
-
-void client_ignore(int ci, int fd)
-{
- pollfd[ci].fd = -1;
- pollfd[ci].events = 0;
-}
-
-void client_back(int ci, int fd)
-{
- pollfd[ci].fd = fd;
- pollfd[ci].events = POLLIN;
-}
-
-static void sigterm_handler(int sig)
-{
- daemon_quit = 1;
-}
-
-static struct lockspace *create_ls(char *name)
-{
- struct lockspace *ls;
-
- ls = malloc(sizeof(*ls));
- if (!ls)
- goto out;
- memset(ls, 0, sizeof(struct lockspace));
- strncpy(ls->name, name, DLM_LOCKSPACE_LEN);
-
- INIT_LIST_HEAD(&ls->changes);
- INIT_LIST_HEAD(&ls->node_history);
- INIT_LIST_HEAD(&ls->saved_messages);
- INIT_LIST_HEAD(&ls->plock_resources);
- ls->plock_resources_root = RB_ROOT;
-#if 0
- INIT_LIST_HEAD(&ls->deadlk_nodes);
- INIT_LIST_HEAD(&ls->transactions);
- INIT_LIST_HEAD(&ls->resources);
-#endif
- out:
- return ls;
-}
-
-struct lockspace *find_ls(char *name)
-{
- struct lockspace *ls;
-
- list_for_each_entry(ls, &lockspaces, list) {
- if ((strlen(ls->name) == strlen(name)) &&
- !strncmp(ls->name, name, strlen(name)))
- return ls;
- }
- return NULL;
-}
-
-struct lockspace *find_ls_id(uint32_t id)
-{
- struct lockspace *ls;
-
- list_for_each_entry(ls, &lockspaces, list) {
- if (ls->global_id == id)
- return ls;
- }
- return NULL;
-}
-
-struct fs_reg {
- struct list_head list;
- char name[DLM_LOCKSPACE_LEN+1];
-};
-
-static int fs_register_check(char *name)
-{
- struct fs_reg *fs;
- list_for_each_entry(fs, &fs_register_list, list) {
- if (!strcmp(name, fs->name))
- return 1;
- }
- return 0;
-}
-
-static int fs_register_add(char *name)
-{
- struct fs_reg *fs;
-
- if (fs_register_check(name))
- return -EALREADY;
-
- fs = malloc(sizeof(struct fs_reg));
- if (!fs)
- return -ENOMEM;
- strncpy(fs->name, name, DLM_LOCKSPACE_LEN);
- list_add(&fs->list, &fs_register_list);
- return 0;
-}
-
-static void fs_register_del(char *name)
-{
- struct fs_reg *fs;
- list_for_each_entry(fs, &fs_register_list, list) {
- if (!strcmp(name, fs->name)) {
- list_del(&fs->list);
- free(fs);
- return;
- }
- }
-}
-
-#define MAXARGS 8
-
-static char *get_args(char *buf, int *argc, char **argv, char sep, int want)
-{
- char *p = buf, *rp = NULL;
- int i;
-
- argv[0] = p;
-
- for (i = 1; i < MAXARGS; i++) {
- p = strchr(buf, sep);
- if (!p)
- break;
- *p = '\0';
-
- if (want == i) {
- rp = p + 1;
- break;
- }
-
- argv[i] = p + 1;
- buf = p + 1;
- }
- *argc = i;
-
- /* we ended by hitting \0, return the point following that */
- if (!rp)
- rp = strchr(buf, '\0') + 1;
-
- return rp;
-}
-
-const char *dlm_mode_str(int mode)
-{
- switch (mode) {
- case DLM_LOCK_IV:
- return "IV";
- case DLM_LOCK_NL:
- return "NL";
- case DLM_LOCK_CR:
- return "CR";
- case DLM_LOCK_CW:
- return "CW";
- case DLM_LOCK_PR:
- return "PR";
- case DLM_LOCK_PW:
- return "PW";
- case DLM_LOCK_EX:
- return "EX";
- }
- return "??";
-}
-
-/* recv "online" (join) and "offline" (leave) messages from dlm via
uevents */
-
-static void process_uevent(int ci)
-{
- struct lockspace *ls;
- char buf[MAXLINE];
- char *argv[MAXARGS], *act, *sys;
- int rv, argc = 0;
-
- memset(buf, 0, sizeof(buf));
- memset(argv, 0, sizeof(char *) * MAXARGS);
-
- retry_recv:
- rv = recv(client[ci].fd, &buf, sizeof(buf), 0);
- if (rv < 0) {
- if (errno == EINTR)
- goto retry_recv;
- if (errno != EAGAIN)
- log_error("uevent recv error %d errno %d", rv, errno);
- return;
- }
-
- if (!strstr(buf, "dlm"))
- return;
-
- log_debug("uevent: %s", buf);
-
- get_args(buf, &argc, argv, '/', 4);
- if (argc != 4)
- log_error("uevent message has %d args", argc);
- act = argv[0];
- sys = argv[2];
-
- if ((strlen(sys) != strlen("dlm")) || strcmp(sys, "dlm"))
- return;
-
- log_debug("kernel: %s %s", act, argv[3]);
-
- rv = 0;
-
- if (!strcmp(act, "online@")) {
- ls = find_ls(argv[3]);
- if (ls) {
- rv = -EEXIST;
- goto out;
- }
-
- ls = create_ls(argv[3]);
- if (!ls) {
- rv = -ENOMEM;
- goto out;
- }
-
- if (fs_register_check(ls->name))
- ls->fs_registered = 1;
-
- rv = dlm_join_lockspace(ls);
- if (rv) {
- /* ls already freed */
- goto out;
- }
-
- } else if (!strcmp(act, "offline@")) {
- ls = find_ls(argv[3]);
- if (!ls) {
- rv = -ENOENT;
- goto out;
- }
-
- dlm_leave_lockspace(ls);
- }
- out:
- if (rv < 0)
- log_error("process_uevent %s error %d errno %d",
- act, rv, errno);
-}
-
-static int setup_uevent(void)
-{
- struct sockaddr_nl snl;
- int s, rv;
-
- s = socket(AF_NETLINK, SOCK_DGRAM, NETLINK_KOBJECT_UEVENT);
- if (s < 0) {
- log_error("uevent netlink socket");
- return s;
- }
-
- memset(&snl, 0, sizeof(snl));
- snl.nl_family = AF_NETLINK;
- snl.nl_pid = getpid();
- snl.nl_groups = 1;
-
- rv = bind(s, (struct sockaddr *) &snl, sizeof(snl));
- if (rv < 0) {
- log_error("uevent bind error %d errno %d", rv, errno);
- close(s);
- return rv;
- }
-
- return s;
-}
-
-static void init_header(struct dlmc_header *h, int cmd, char *name, int result,
- int extra_len)
-{
- memset(h, 0, sizeof(struct dlmc_header));
-
- h->magic = DLMC_MAGIC;
- h->version = DLMC_VERSION;
- h->len = sizeof(struct dlmc_header) + extra_len;
- h->command = cmd;
- h->data = result;
-
- if (name)
- strncpy(h->name, name, DLM_LOCKSPACE_LEN);
-}
-
-static char copy_buf[LOG_DUMP_SIZE];
-
-static void query_dump_debug(int fd)
-{
- struct dlmc_header h;
- int len = 0;
-
- copy_log_dump(copy_buf, &len);
-
- init_header(&h, DLMC_CMD_DUMP_DEBUG, NULL, 0, len);
- send(fd, &h, sizeof(h), MSG_NOSIGNAL);
-
- if (len)
- send(fd, copy_buf, len, MSG_NOSIGNAL);
-}
-
-static void query_dump_log_plock(int fd)
-{
- struct dlmc_header h;
- int len = 0;
-
- copy_log_dump_plock(copy_buf, &len);
-
- init_header(&h, DLMC_CMD_DUMP_DEBUG, NULL, 0, len);
- send(fd, &h, sizeof(h), MSG_NOSIGNAL);
-
- if (len)
- send(fd, copy_buf, len, MSG_NOSIGNAL);
-}
-
-static void query_dump_plocks(int fd, char *name)
-{
- struct lockspace *ls;
- struct dlmc_header h;
- int len = 0;
- int rv;
-
- ls = find_ls(name);
- if (!ls) {
- rv = -ENOENT;
- goto out;
- }
-
- rv = copy_plock_state(ls, copy_buf, &len);
- out:
- init_header(&h, DLMC_CMD_DUMP_PLOCKS, name, rv, len);
- send(fd, &h, sizeof(h), MSG_NOSIGNAL);
-
- if (len)
- send(fd, copy_buf, len, MSG_NOSIGNAL);
-}
-
-/* combines a header and the data and sends it back to the client in
- a single do_write() call */
-
-static void do_reply(int fd, int cmd, char *name, int result, int option,
- char *buf, int buflen)
-{
- struct dlmc_header *h;
- char *reply;
- int reply_len;
-
- reply_len = sizeof(struct dlmc_header) + buflen;
- reply = malloc(reply_len);
- if (!reply)
- return;
- memset(reply, 0, reply_len);
- h = (struct dlmc_header *)reply;
-
- init_header(h, cmd, name, result, buflen);
- h->option = option;
-
- if (buf && buflen)
- memcpy(reply + sizeof(struct dlmc_header), buf, buflen);
-
- do_write(fd, reply, reply_len);
-
- free(reply);
-}
-
-static void query_lockspace_info(int fd, char *name)
-{
- struct lockspace *ls;
- struct dlmc_lockspace lockspace;
- int rv;
-
- ls = find_ls(name);
- if (!ls) {
- rv = -ENOENT;
- goto out;
- }
-
- memset(&lockspace, 0, sizeof(lockspace));
-
- rv = set_lockspace_info(ls, &lockspace);
- out:
- do_reply(fd, DLMC_CMD_LOCKSPACE_INFO, name, rv, 0,
- (char *)&lockspace, sizeof(lockspace));
-}
-
-static void query_node_info(int fd, char *name, int nodeid)
-{
- struct lockspace *ls;
- struct dlmc_node node;
- int rv;
-
- ls = find_ls(name);
- if (!ls) {
- rv = -ENOENT;
- goto out;
- }
-
- memset(&node, 0, sizeof(node));
-
- rv = set_node_info(ls, nodeid, &node);
- out:
- do_reply(fd, DLMC_CMD_NODE_INFO, name, rv, 0,
- (char *)&node, sizeof(node));
-}
-
-static void query_lockspaces(int fd, int max)
-{
- int ls_count = 0;
- struct dlmc_lockspace *lss = NULL;
- int rv, result;
-
- rv = set_lockspaces(&ls_count, &lss);
- if (rv < 0) {
- result = rv;
- ls_count = 0;
- goto out;
- }
-
- if (ls_count > max) {
- result = -E2BIG;
- ls_count = max;
- } else {
- result = ls_count;
- }
- out:
- do_reply(fd, DLMC_CMD_LOCKSPACES, NULL, result, 0,
- (char *)lss, ls_count * sizeof(struct dlmc_lockspace));
-
- if (lss)
- free(lss);
-}
-
-static void query_lockspace_nodes(int fd, char *name, int option, int max)
-{
- struct lockspace *ls;
- int node_count = 0;
- struct dlmc_node *nodes = NULL;
- int rv, result;
-
- ls = find_ls(name);
- if (!ls) {
- result = -ENOENT;
- node_count = 0;
- goto out;
- }
-
- rv = set_lockspace_nodes(ls, option, &node_count, &nodes);
- if (rv < 0) {
- result = rv;
- node_count = 0;
- goto out;
- }
-
- /* node_count is the number of structs copied/returned; the caller's
- max may be less than that, in which case we copy as many as they
- asked for and return -E2BIG */
-
- if (node_count > max) {
- result = -E2BIG;
- node_count = max;
- } else {
- result = node_count;
- }
- out:
- do_reply(fd, DLMC_CMD_LOCKSPACE_NODES, name, result, 0,
- (char *)nodes, node_count * sizeof(struct dlmc_node));
-
- if (nodes)
- free(nodes);
-}
-
-static void process_connection(int ci)
-{
- struct dlmc_header h;
- char *extra = NULL;
- int rv, extra_len;
- struct lockspace *ls;
-
- rv = do_read(client[ci].fd, &h, sizeof(h));
- if (rv < 0) {
- log_debug("connection %d read error %d", ci, rv);
- goto out;
- }
-
- if (h.magic != DLMC_MAGIC) {
- log_debug("connection %d magic error %x", ci, h.magic);
- goto out;
- }
-
- if ((h.version & 0xFFFF0000) != (DLMC_VERSION & 0xFFFF0000)) {
- log_debug("connection %d version error %x", ci, h.version);
- goto out;
- }
-
- if (h.len > sizeof(h)) {
- extra_len = h.len - sizeof(h);
- extra = malloc(extra_len);
- if (!extra) {
- log_error("process_connection no mem %d", extra_len);
- goto out;
- }
- memset(extra, 0, extra_len);
-
- rv = do_read(client[ci].fd, extra, extra_len);
- if (rv < 0) {
- log_debug("connection %d extra read error %d", ci, rv);
- goto out;
- }
- }
-
- switch (h.command) {
- case DLMC_CMD_FS_REGISTER:
- rv = fs_register_add(h.name);
- ls = find_ls(h.name);
- if (ls)
- ls->fs_registered = 1;
- do_reply(client[ci].fd, DLMC_CMD_FS_REGISTER, h.name, rv, 0,
- NULL, 0);
- break;
-
- case DLMC_CMD_FS_UNREGISTER:
- fs_register_del(h.name);
- ls = find_ls(h.name);
- if (ls)
- ls->fs_registered = 0;
- break;
-
- case DLMC_CMD_FS_NOTIFIED:
- ls = find_ls(h.name);
- if (ls)
- rv = set_fs_notified(ls, h.data);
- else
- rv = -ENOENT;
- /* pass back the nodeid provided by caller in option field */
- do_reply(client[ci].fd, DLMC_CMD_FS_NOTIFIED, h.name, rv,
- h.data, NULL, 0);
- break;
-
-#if 0
- case DLMC_CMD_DEADLOCK_CHECK:
- ls = find_ls(h.name);
- if (ls)
- send_cycle_start(ls);
- client_dead(ci);
- break;
-#endif
- default:
- log_error("process_connection %d unknown command %d",
- ci, h.command);
- }
- out:
- if (extra)
- free(extra);
-}
-
-static void process_listener(int ci)
-{
- int fd, i;
-
- fd = accept(client[ci].fd, NULL, NULL);
- if (fd < 0) {
- log_error("process_listener: accept error %d %d", fd, errno);
- return;
- }
-
- i = client_add(fd, process_connection, NULL);
-
- log_debug("client connection %d fd %d", i, fd);
-}
-
-static int setup_listener(const char *sock_path)
-{
- struct sockaddr_un addr;
- socklen_t addrlen;
- int rv, s;
-
- /* we listen for new client connections on socket s */
-
- s = socket(AF_LOCAL, SOCK_STREAM, 0);
- if (s < 0) {
- log_error("socket error %d %d", s, errno);
- return s;
- }
-
- memset(&addr, 0, sizeof(addr));
- addr.sun_family = AF_LOCAL;
- strcpy(&addr.sun_path[1], sock_path);
- addrlen = sizeof(sa_family_t) + strlen(addr.sun_path+1) + 1;
-
- rv = bind(s, (struct sockaddr *) &addr, addrlen);
- if (rv < 0) {
- log_error("bind error %d %d", rv, errno);
- close(s);
- return rv;
- }
-
- rv = listen(s, 5);
- if (rv < 0) {
- log_error("listen error %d %d", rv, errno);
- close(s);
- return rv;
- }
- return s;
-}
-
-static void query_lock(void)
-{
- pthread_mutex_lock(&query_mutex);
-}
-
-static void query_unlock(void)
-{
- pthread_mutex_unlock(&query_mutex);
-}
-
-/* This is a thread, so we have to be careful, don't call log_ functions.
- We need a thread to process queries because the main thread may block
- for long periods when writing to sysfs to stop dlm-kernel (any maybe
- other places). */
-
-static void *process_queries(void *arg)
-{
- struct dlmc_header h;
- int s, f, rv;
-
- rv = setup_listener(DLMC_QUERY_SOCK_PATH);
- if (rv < 0)
- return NULL;
-
- s = rv;
-
- for (;;) {
- f = accept(s, NULL, NULL);
- if (f < 0)
- return NULL;
-
- rv = do_read(f, &h, sizeof(h));
- if (rv < 0) {
- goto out;
- }
-
- if (h.magic != DLMC_MAGIC) {
- goto out;
- }
-
- if ((h.version & 0xFFFF0000) != (DLMC_VERSION & 0xFFFF0000)) {
- goto out;
- }
-
- query_lock();
-
- switch (h.command) {
- case DLMC_CMD_DUMP_DEBUG:
- query_dump_debug(f);
- break;
- case DLMC_CMD_DUMP_LOG_PLOCK:
- query_dump_log_plock(f);
- break;
- case DLMC_CMD_DUMP_PLOCKS:
- query_dump_plocks(f, h.name);
- break;
- case DLMC_CMD_LOCKSPACE_INFO:
- query_lockspace_info(f, h.name);
- break;
- case DLMC_CMD_NODE_INFO:
- query_node_info(f, h.name, h.data);
- break;
- case DLMC_CMD_LOCKSPACES:
- query_lockspaces(f, h.data);
- break;
- case DLMC_CMD_LOCKSPACE_NODES:
- query_lockspace_nodes(f, h.name, h.option, h.data);
- break;
- default:
- break;
- }
- query_unlock();
-
- out:
- close(f);
- }
-}
-
-static int setup_queries(void)
-{
- int rv;
-
- pthread_mutex_init(&query_mutex, NULL);
-
- rv = pthread_create(&query_thread, NULL, process_queries, NULL);
- if (rv < 0) {
- log_error("can't create query thread");
- return rv;
- }
- return 0;
-}
-
-/* The dlm in kernels before 2.6.28 do not have the monitor device. We
- keep this fd open as long as we're running. If we exit/terminate while
- lockspaces exist in the kernel, the kernel will detect a close on this
- fd and stop the lockspaces. */
-
-static void setup_monitor(void)
-{
- if (!monitor_minor)
- return;
-
- kernel_monitor_fd = open("/dev/misc/dlm-monitor", O_RDONLY);
- log_debug("/dev/misc/dlm-monitor fd %d", kernel_monitor_fd);
-}
-
-void cluster_dead(int ci)
-{
- if (!cluster_down)
- log_error("cluster is down, exiting");
- daemon_quit = 1;
- cluster_down = 1;
-}
-
-static void loop(void)
-{
- struct lockspace *ls;
- int poll_timeout = -1;
- int rv, i;
- void (*workfn) (int ci);
- void (*deadfn) (int ci);
-
- rv = setup_queries();
- if (rv < 0)
- goto out;
-
- rv = setup_listener(DLMC_SOCK_PATH);
- if (rv < 0)
- goto out;
- client_add(rv, process_listener, NULL);
-
- rv = setup_cluster_cfg();
- if (rv < 0)
- goto out;
- if (rv > 0)
- client_add(rv, process_cluster_cfg, cluster_dead);
-
- rv = setup_cluster();
- if (rv < 0)
- goto out;
- client_add(rv, process_cluster, cluster_dead);
-
- setup_config(0);
-
- setup_logging();
-
- rv = check_uncontrolled_lockspaces();
- if (rv < 0)
- goto out;
-
- rv = setup_misc_devices();
- if (rv < 0)
- goto out;
-
- setup_monitor();
-
- rv = setup_configfs(); /* calls update_cluster() */
- if (rv < 0)
- goto out;
-
- rv = setup_uevent();
- if (rv < 0)
- goto out;
- client_add(rv, process_uevent, NULL);
-
- rv = setup_cpg_daemon();
- if (rv < 0)
- goto out;
- client_add(rv, process_cpg_daemon, cluster_dead);
-
- rv = set_protocol();
- if (rv < 0)
- goto out;
-
-#if 0
- if (cfgd_enable_deadlk) {
- rv = setup_netlink();
- if (rv < 0)
- goto out;
- client_add(rv, process_netlink, NULL);
-
- setup_deadlock();
- }
-#endif
-
- rv = setup_plocks();
- if (rv < 0)
- goto out;
- plock_fd = rv;
- plock_ci = client_add(rv, process_plocks, NULL);
-
- for (;;) {
- rv = poll(pollfd, client_maxi + 1, poll_timeout);
- if (rv == -1 && errno == EINTR) {
- if (daemon_quit && list_empty(&lockspaces))
- goto out;
- log_error("shutdown ignored, active lockspaces");
- daemon_quit = 0;
- continue;
- }
- if (rv < 0) {
- log_error("poll errno %d", errno);
- goto out;
- }
-
- query_lock();
-
- for (i = 0; i <= client_maxi; i++) {
- if (client[i].fd < 0)
- continue;
- if (pollfd[i].revents & POLLIN) {
- workfn = client[i].workfn;
- workfn(i);
- }
- if (pollfd[i].revents & (POLLERR | POLLHUP | POLLNVAL)) {
- deadfn = client[i].deadfn;
- deadfn(i);
- }
- }
- query_unlock();
-
- if (daemon_quit)
- break;
-
- query_lock();
-
- poll_timeout = -1;
-
- if (poll_fencing || poll_quorum || poll_fs) {
- process_lockspace_changes();
- poll_timeout = 1000;
- }
-
- if (poll_ignore_plock) {
- if (!limit_plocks()) {
- poll_ignore_plock = 0;
- client_back(plock_ci, plock_fd);
- }
- poll_timeout = 1000;
- }
-
- if (poll_drop_plock) {
- drop_resources_all();
- if (poll_drop_plock)
- poll_timeout = 1000;
- }
-
- query_unlock();
- }
- out:
- log_debug("shutdown");
- close_plocks();
- close_cpg_daemon();
- clear_configfs();
- close_logging();
- close_cluster();
- close_cluster_cfg();
-
- list_for_each_entry(ls, &lockspaces, list)
- log_error("abandoned lockspace %s", ls->name);
-}
-
-static void lockfile(void)
-{
- int fd, error;
- struct flock lock;
- char buf[33];
-
- memset(buf, 0, 33);
-
- fd = open(LOCKFILE_NAME, O_CREAT|O_WRONLY,
- S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
- if (fd < 0) {
- fprintf(stderr, "cannot open/create lock file %s\n",
- LOCKFILE_NAME);
- exit(EXIT_FAILURE);
- }
-
- lock.l_type = F_WRLCK;
- lock.l_start = 0;
- lock.l_whence = SEEK_SET;
- lock.l_len = 0;
-
- error = fcntl(fd, F_SETLK, &lock);
- if (error) {
- fprintf(stderr, "dlm_controld is already running\n");
- exit(EXIT_FAILURE);
- }
-
- error = ftruncate(fd, 0);
- if (error) {
- fprintf(stderr, "cannot clear lock file %s\n", LOCKFILE_NAME);
- exit(EXIT_FAILURE);
- }
-
- sprintf(buf, "%d\n", getpid());
-
- error = write(fd, buf, strlen(buf));
- if (error <= 0) {
- fprintf(stderr, "cannot write lock file %s\n", LOCKFILE_NAME);
- exit(EXIT_FAILURE);
- }
-}
-
-static void print_usage(void)
-{
- printf("Usage:\n");
- printf("\n");
- printf("dlm_controld [options]\n");
- printf("\n");
- printf("Options:\n");
- printf("\n");
- printf(" -D Enable debugging to stderr and don't fork\n");
- printf(" -L Enable debugging to log file\n");
- printf(" -K Enable kernel dlm debugging messages\n");
- printf(" -r <num> dlm kernel lowcomms protocol, 0 tcp, 1 sctp, 2
detect\n");
- printf(" 2 selects tcp if corosync rrp_mode is \"none\",
otherwise sctp\n");
- printf(" Default is 2\n");
- printf(" -f <num> Enable (1) or disable (0) fencing recovery
dependency\n");
- printf(" Default is %d\n", DEFAULT_ENABLE_FENCING);
- printf(" -q <num> Enable (1) or disable (0) quorum recovery
dependency\n");
- printf(" Default is %d\n", DEFAULT_ENABLE_QUORUM);
-#if 0
- printf(" -d <num> Enable (1) or disable (0) deadlock detection
code\n");
- printf(" Default is %d\n", DEFAULT_ENABLE_DEADLK);
-#endif
- printf(" -p <num> Enable (1) or disable (0) plock code for cluster
fs\n");
- printf(" Default is %d\n", DEFAULT_ENABLE_PLOCK);
- printf(" -P Enable plock debugging\n");
- printf(" -l <limit> Limit the rate of plock operations\n");
- printf(" Default is %d, set to 0 for no limit\n", DEFAULT_PLOCK_RATE_LIMIT);
- printf(" -o <n> Enable (1) or disable (0) plock ownership\n");
- printf(" Default is %d\n", DEFAULT_PLOCK_OWNERSHIP);
- printf(" -t <ms> plock ownership drop resources time
(milliseconds)\n");
- printf(" Default is %u\n", DEFAULT_DROP_RESOURCES_TIME);
- printf(" -c <num> plock ownership drop resources count\n");
- printf(" Default is %u\n", DEFAULT_DROP_RESOURCES_COUNT);
- printf(" -a <ms> plock ownership drop resources age
(milliseconds)\n");
- printf(" Default is %u\n", DEFAULT_DROP_RESOURCES_AGE);
- printf(" -h Print this help, then exit\n");
- printf(" -V Print program version information, then exit\n");
-}
-
-#define OPTION_STRING "LDKf:q:p:Pl:o:t:c:a:hVr:"
-
-static void read_arguments(int argc, char **argv)
-{
- int cont = 1;
- int optchar;
-
- while (cont) {
- optchar = getopt(argc, argv, OPTION_STRING);
-
- switch (optchar) {
- case 'D':
- daemon_debug_opt = 1;
- break;
-
- case 'L':
- optd_debug_logfile = 1;
- cfgd_debug_logfile = 1;
- break;
-
- case 'K':
- optk_debug = 1;
- cfgk_debug = 1;
- break;
-
- case 'r':
- optk_protocol = 1;
- cfgk_protocol = atoi(optarg);
- break;
-
- case 'f':
- optd_enable_fencing = 1;
- cfgd_enable_fencing = atoi(optarg);
- break;
-
- case 'q':
- optd_enable_quorum = 1;
- cfgd_enable_quorum = atoi(optarg);
- break;
-
- case 'p':
- optd_enable_plock = 1;
- cfgd_enable_plock = atoi(optarg);
- break;
-
- case 'P':
- optd_plock_debug = 1;
- cfgd_plock_debug = 1;
- break;
-
- case 'l':
- optd_plock_rate_limit = 1;
- cfgd_plock_rate_limit = atoi(optarg);
- break;
-
- case 'o':
- optd_plock_ownership = 1;
- cfgd_plock_ownership = atoi(optarg);
- break;
-
- case 't':
- optd_drop_resources_time = 1;
- cfgd_drop_resources_time = atoi(optarg);
- break;
-
- case 'c':
- optd_drop_resources_count = 1;
- cfgd_drop_resources_count = atoi(optarg);
- break;
-
- case 'a':
- optd_drop_resources_age = 1;
- cfgd_drop_resources_age = atoi(optarg);
- break;
-
- case 'h':
- print_usage();
- exit(EXIT_SUCCESS);
- break;
-
- case 'V':
- printf("dlm_controld %s (built %s %s)\n",
- VERSION, __DATE__, __TIME__);
- printf("%s\n", REDHAT_COPYRIGHT);
- exit(EXIT_SUCCESS);
- break;
-
- case ':':
- case '?':
- fprintf(stderr, "Please use '-h' for usage.\n");
- exit(EXIT_FAILURE);
- break;
-
- case EOF:
- cont = 0;
- break;
-
- default:
- fprintf(stderr, "unknown option: %c\n", optchar);
- exit(EXIT_FAILURE);
- break;
- };
- }
-
- if (getenv("DLM_CONTROLD_DEBUG")) {
- optd_debug_logfile = 1;
- cfgd_debug_logfile = 1;
- }
-}
-
-static void set_scheduler(void)
-{
- struct sched_param sched_param;
- int rv;
-
- rv = sched_get_priority_max(SCHED_RR);
- if (rv != -1) {
- sched_param.sched_priority = rv;
- rv = sched_setscheduler(0, SCHED_RR, &sched_param);
- if (rv == -1)
- log_error("could not set SCHED_RR priority %d err %d",
- sched_param.sched_priority, errno);
- } else {
- log_error("could not get maximum scheduler priority err %d",
- errno);
- }
-}
-
-int main(int argc, char **argv)
-{
- INIT_LIST_HEAD(&lockspaces);
- INIT_LIST_HEAD(&fs_register_list);
-
- read_arguments(argc, argv);
-
- if (!daemon_debug_opt) {
- if (daemon(0, 0) < 0) {
- perror("daemon error");
- exit(EXIT_FAILURE);
- }
- }
- lockfile();
- init_logging();
- log_level(NULL, LOG_INFO, "dlm_controld %s started", VERSION);
- signal(SIGTERM, sigterm_handler);
- set_scheduler();
-
- loop();
-
- return 0;
-}
-
-int daemon_debug_opt;
-int daemon_quit;
-int cluster_down;
-int poll_fencing;
-int poll_quorum;
-int poll_fs;
-int poll_ignore_plock;
-int poll_drop_plock;
-int plock_fd;
-int plock_ci;
-struct list_head lockspaces;
-int cluster_quorate;
-int our_nodeid;
-uint32_t cluster_ringid_seq;
-uint32_t control_minor;
-uint32_t monitor_minor;
-uint32_t plock_minor;
-uint32_t old_plock_minor;
-
-/* was a config value set on command line?, 0 or 1.
- optk is a kernel option, optd is a daemon option */
-
-int optk_debug;
-int optk_timewarn;
-int optk_protocol;
-int optd_debug_logfile;
-int optd_enable_fencing;
-int optd_enable_quorum;
-int optd_enable_plock;
-int optd_plock_debug;
-int optd_plock_rate_limit;
-int optd_plock_ownership;
-int optd_drop_resources_time;
-int optd_drop_resources_count;
-int optd_drop_resources_age;
-
-/* actual config value from command line, cluster.conf, or default.
- cfgk is a kernel config value, cfgd is a daemon config value */
-
-int cfgk_debug = -1;
-int cfgk_timewarn = -1;
-int cfgk_protocol = PROTO_DETECT;
-int cfgd_debug_logfile = DEFAULT_DEBUG_LOGFILE;
-int cfgd_enable_fencing = DEFAULT_ENABLE_FENCING;
-int cfgd_enable_quorum = DEFAULT_ENABLE_QUORUM;
-int cfgd_enable_plock = DEFAULT_ENABLE_PLOCK;
-int cfgd_plock_debug = DEFAULT_PLOCK_DEBUG;
-int cfgd_plock_rate_limit = DEFAULT_PLOCK_RATE_LIMIT;
-int cfgd_plock_ownership = DEFAULT_PLOCK_OWNERSHIP;
-int cfgd_drop_resources_time = DEFAULT_DROP_RESOURCES_TIME;
-int cfgd_drop_resources_count = DEFAULT_DROP_RESOURCES_COUNT;
-int cfgd_drop_resources_age = DEFAULT_DROP_RESOURCES_AGE;
-
diff --git a/group/dlm_controld/member_cman.c b/group/dlm_controld/member_cman.c
deleted file mode 100644
index 909c249..0000000
--- a/group/dlm_controld/member_cman.c
+++ /dev/null
@@ -1,267 +0,0 @@
-#include "dlm_daemon.h"
-#include <corosync/corotypes.h>
-#include <corosync/cfg.h>
-#include <corosync/quorum.h>
-#include "libfenced.h"
-
-static corosync_cfg_handle_t ch;
-static quorum_handle_t qh;
-static uint32_t old_nodes[MAX_NODES];
-static int old_node_count;
-static uint32_t quorum_nodes[MAX_NODES];
-static int quorum_node_count;
-
-static int is_member(uint32_t *node_list, int count, uint32_t nodeid)
-{
- int i;
-
- for (i = 0; i < count; i++) {
- if (node_list[i] == nodeid)
- return 1;
- }
- return 0;
-}
-
-static int is_old_member(uint32_t nodeid)
-{
- return is_member(old_nodes, old_node_count, nodeid);
-}
-
-int is_cluster_member(uint32_t nodeid)
-{
- return is_member(quorum_nodes, quorum_node_count, nodeid);
-}
-
-static void quorum_callback(quorum_handle_t h, uint32_t quorate,
- uint64_t ring_seq, uint32_t node_list_entries,
- uint32_t *node_list)
-{
- corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES];
- corosync_cfg_node_address_t *addrptr = addrs;
- cs_error_t err;
- int i, j, num_addrs;
-
-
- cluster_quorate = quorate;
- cluster_ringid_seq = (uint32_t)ring_seq;
-
- log_debug("cluster quorum %u seq %u nodes %u",
- cluster_quorate, cluster_ringid_seq, node_list_entries);
-
- old_node_count = quorum_node_count;
- memcpy(&old_nodes, &quorum_nodes, sizeof(old_nodes));
-
- quorum_node_count = 0;
- memset(&quorum_nodes, 0, sizeof(quorum_nodes));
-
- for (i = 0; i < node_list_entries; i++)
- quorum_nodes[quorum_node_count++] = node_list[i];
-
- for (i = 0; i < old_node_count; i++) {
- if (!is_cluster_member(old_nodes[i])) {
- log_debug("cluster node %u removed seq %u",
- old_nodes[i], cluster_ringid_seq);
- node_history_cluster_remove(old_nodes[i]);
- del_configfs_node(old_nodes[i]);
- }
- }
-
- for (i = 0; i < quorum_node_count; i++) {
- if (!is_old_member(quorum_nodes[i])) {
- log_debug("cluster node %u added seq %u",
- quorum_nodes[i], cluster_ringid_seq);
- node_history_cluster_add(quorum_nodes[i]);
-
- err = corosync_cfg_get_node_addrs(ch, quorum_nodes[i],
- MAX_NODE_ADDRESSES,
- &num_addrs, addrs);
- if (err != CS_OK) {
- log_error("corosync_cfg_get_node_addrs failed "
- "nodeid %u", quorum_nodes[i]);
- continue;
- }
-
- for (j = 0; j < num_addrs; j++) {
- add_configfs_node(quorum_nodes[i],
- addrptr[j].address,
- addrptr[j].address_length,
- (quorum_nodes[i] ==
- our_nodeid));
- }
- }
- }
-}
-
-static quorum_callbacks_t quorum_callbacks =
-{
- .quorum_notify_fn = quorum_callback,
-};
-
-void process_cluster(int ci)
-{
- cs_error_t err;
-
- err = quorum_dispatch(qh, CS_DISPATCH_ALL);
- if (err != CS_OK)
- cluster_dead(0);
-}
-
-/* Force re-read of quorum nodes */
-void update_cluster(void)
-{
- cs_error_t err;
-
- err = quorum_dispatch(qh, CS_DISPATCH_ONE);
- if (err != CS_OK)
- cluster_dead(0);
-}
-
-int setup_cluster(void)
-{
- cs_error_t err;
- int fd;
-
- err = quorum_initialize(&qh, &quorum_callbacks);
- if (err != CS_OK) {
- log_error("quorum init error %d", err);
- return -1;
- }
-
- err = quorum_fd_get(qh, &fd);
- if (err != CS_OK) {
- log_error("quorum fd_get error %d", err);
- goto fail;
- }
-
- err = quorum_trackstart(qh, CS_TRACK_CHANGES);
- if (err != CS_OK) {
- log_error("quorum trackstart error %d", err);
- goto fail;
- }
-
- old_node_count = 0;
- memset(&old_nodes, 0, sizeof(old_nodes));
- quorum_node_count = 0;
- memset(&quorum_nodes, 0, sizeof(quorum_nodes));
-
- return fd;
- fail:
- quorum_finalize(qh);
- return -1;
-}
-
-void close_cluster(void)
-{
- quorum_trackstop(qh);
- quorum_finalize(qh);
-}
-
-void kick_node_from_cluster(int nodeid)
-{
- if (!nodeid) {
- log_error("telling corosync to shut down cluster locally");
- corosync_cfg_try_shutdown(ch,
- COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE);
- } else {
- log_error("telling corosync to remove nodeid %d from cluster",
- nodeid);
- corosync_cfg_kill_node(ch, nodeid, "dlm_controld");
- }
-}
-
-static void shutdown_callback(corosync_cfg_handle_t h,
- corosync_cfg_shutdown_flags_t flags)
-{
- if (flags & COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) {
- if (list_empty(&lockspaces))
- corosync_cfg_replyto_shutdown(ch,
- COROSYNC_CFG_SHUTDOWN_FLAG_YES);
- else {
- log_debug("no to corosync shutdown");
- corosync_cfg_replyto_shutdown(ch,
- COROSYNC_CFG_SHUTDOWN_FLAG_NO);
- }
- }
-}
-
-static corosync_cfg_callbacks_t cfg_callbacks =
-{
- .corosync_cfg_shutdown_callback = shutdown_callback,
- .corosync_cfg_state_track_callback = NULL,
-};
-
-void process_cluster_cfg(int ci)
-{
- cs_error_t err;
-
- err = corosync_cfg_dispatch(ch, CS_DISPATCH_ALL);
- if (err != CS_OK)
- cluster_dead(0);
-}
-
-int setup_cluster_cfg(void)
-{
- cs_error_t err;
- unsigned int nodeid;
- int fd;
-
- err = corosync_cfg_initialize(&ch, &cfg_callbacks);
- if (err != CS_OK) {
- log_error("corosync cfg init error %d", err);
- return -1;
- }
-
- err = corosync_cfg_fd_get(ch, &fd);
- if (err != CS_OK) {
- log_error("corosync cfg fd_get error %d", err);
- corosync_cfg_finalize(ch);
- return -1;
- }
-
- err = corosync_cfg_local_get(ch, &nodeid);
- if (err != CS_OK) {
- log_error("corosync cfg local_get error %d", err);
- corosync_cfg_finalize(ch);
- return -1;
- }
- our_nodeid = nodeid;
- log_debug("our_nodeid %d", our_nodeid);
-
- return fd;
-}
-
-void close_cluster_cfg(void)
-{
- corosync_cfg_finalize(ch);
-}
-
-int fence_node_time(int nodeid, uint64_t *last_fenced_time)
-{
- struct fenced_node nodeinfo;
- int rv;
-
- memset(&nodeinfo, 0, sizeof(nodeinfo));
-
- rv = fenced_node_info(nodeid, &nodeinfo);
- if (rv < 0)
- return rv;
-
- *last_fenced_time = nodeinfo.last_fenced_time;
- return 0;
-}
-
-int fence_in_progress(int *count)
-{
- struct fenced_domain domain;
- int rv;
-
- memset(&domain, 0, sizeof(domain));
-
- rv = fenced_domain_info(&domain);
- if (rv < 0)
- return rv;
-
- *count = domain.victim_count;
- return 0;
-}
-
diff --git a/group/dlm_controld/netlink.c b/group/dlm_controld/netlink.c
deleted file mode 100644
index 63122f7..0000000
--- a/group/dlm_controld/netlink.c
+++ /dev/null
@@ -1,225 +0,0 @@
-#include "dlm_daemon.h"
-#include "config.h"
-#include <linux/dlm.h>
-#include <linux/netlink.h>
-#include <linux/genetlink.h>
-#include <linux/dlm_netlink.h>
-
-#define DEADLOCK_CHECK_SECS 10
-
-/* FIXME: look into using libnl/libnetlink */
-
-#define GENLMSG_DATA(glh) ((void *)((char *)NLMSG_DATA(glh) + GENL_HDRLEN))
-#define GENLMSG_PAYLOAD(glh) (NLMSG_PAYLOAD(glh, 0) - GENL_HDRLEN)
-#define NLA_DATA(na) ((void *)((char*)(na) + NLA_HDRLEN))
-#define NLA_PAYLOAD(len) (len - NLA_HDRLEN)
-
-/* Maximum size of response requested or message sent */
-#define MAX_MSG_SIZE 1024
-
-struct msgtemplate {
- struct nlmsghdr n;
- struct genlmsghdr g;
- char buf[MAX_MSG_SIZE];
-};
-
-static int send_genetlink_cmd(int sd, uint16_t nlmsg_type, uint32_t nlmsg_pid,
- uint8_t genl_cmd, uint16_t nla_type,
- void *nla_data, int nla_len)
-{
- struct nlattr *na;
- struct sockaddr_nl nladdr;
- int r, buflen;
- char *buf;
-
- struct msgtemplate msg;
-
- msg.n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
- msg.n.nlmsg_type = nlmsg_type;
- msg.n.nlmsg_flags = NLM_F_REQUEST;
- msg.n.nlmsg_seq = 0;
- msg.n.nlmsg_pid = nlmsg_pid;
- msg.g.cmd = genl_cmd;
- msg.g.version = 0x1;
- na = (struct nlattr *) GENLMSG_DATA(&msg);
- na->nla_type = nla_type;
- na->nla_len = nla_len + 1 + NLA_HDRLEN;
- if (nla_data)
- memcpy(NLA_DATA(na), nla_data, nla_len);
- msg.n.nlmsg_len += NLMSG_ALIGN(na->nla_len);
-
- buf = (char *) &msg;
- buflen = msg.n.nlmsg_len ;
- memset(&nladdr, 0, sizeof(nladdr));
- nladdr.nl_family = AF_NETLINK;
- while ((r = sendto(sd, buf, buflen, 0, (struct sockaddr *) &nladdr,
- sizeof(nladdr))) < buflen) {
- if (r > 0) {
- buf += r;
- buflen -= r;
- } else if (errno != EAGAIN)
- return -1;
- }
- return 0;
-}
-
-/*
- * Probe the controller in genetlink to find the family id
- * for the DLM family
- */
-static int get_family_id(int sd)
-{
- char genl_name[100];
- struct {
- struct nlmsghdr n;
- struct genlmsghdr g;
- char buf[256];
- } ans;
-
- int id = 0, rc;
- struct nlattr *na;
- int rep_len;
-
- strcpy(genl_name, DLM_GENL_NAME);
- rc = send_genetlink_cmd(sd, GENL_ID_CTRL, getpid(), CTRL_CMD_GETFAMILY,
- CTRL_ATTR_FAMILY_NAME, (void *)genl_name,
- strlen(DLM_GENL_NAME)+1);
-
- rep_len = recv(sd, &ans, sizeof(ans), 0);
- if (ans.n.nlmsg_type == NLMSG_ERROR ||
- (rep_len < 0) || !NLMSG_OK((&ans.n), rep_len))
- return 0;
-
- na = (struct nlattr *) GENLMSG_DATA(&ans);
- na = (struct nlattr *) ((char *) na + NLA_ALIGN(na->nla_len));
- if (na->nla_type == CTRL_ATTR_FAMILY_ID) {
- id = *(uint16_t *) NLA_DATA(na);
- }
- return id;
-}
-
-/* genetlink messages are timewarnings used as part of deadlock detection */
-
-int setup_netlink(void)
-{
- struct sockaddr_nl snl;
- int s, rv;
- uint16_t id;
-
- s = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
- if (s < 0) {
- log_error("generic netlink socket");
- return s;
- }
-
- memset(&snl, 0, sizeof(snl));
- snl.nl_family = AF_NETLINK;
-
- rv = bind(s, (struct sockaddr *) &snl, sizeof(snl));
- if (rv < 0) {
- log_error("gen netlink bind error %d errno %d", rv, errno);
- close(s);
- return rv;
- }
-
- id = get_family_id(s);
- if (!id) {
- log_error("Error getting family id, errno %d", errno);
- close(s);
- return -1;
- }
-
- rv = send_genetlink_cmd(s, id, getpid(), DLM_CMD_HELLO, 0, NULL, 0);
- if (rv < 0) {
- log_error("error sending hello cmd, errno %d", errno);
- close(s);
- return -1;
- }
-
- return s;
-}
-
-static void process_timewarn(struct dlm_lock_data *data)
-{
- struct lockspace *ls;
- struct timeval now;
- unsigned int sec;
-
- ls = find_ls_id(data->lockspace_id);
- if (!ls)
- return;
-
- data->resource_name[data->resource_namelen] = '\0';
-
- log_group(ls, "timewarn: lkid %x pid %d name %s",
- data->id, data->ownpid, data->resource_name);
-
- /* Problem: we don't want to get a timewarn, assume it's resolved
- by the current cycle, but in fact it's from a deadlock that
- formed after the checkpoints for the current cycle. Then we'd
- have to hope for another warning (that may not come) to trigger
- a new cycle to catch the deadlock. If our last cycle ckpt
- was say N (~5?) sec before we receive the timewarn, then we
- can be confident that the cycle included the lock in question.
- Otherwise, we're not sure if the warning is for a new deadlock
- that's formed since our last cycle ckpt (unless it's a long
- enough time since the last cycle that we're confident it *is*
- a new deadlock). When there is a deadlock, I suspect it will
- be common to receive warnings before, during, and possibly
- after the cycle that resolves it. Wonder if we should record
- timewarns and match them with deadlock cycles so we can tell
- which timewarns are addressed by a given cycle and which aren't. */
-
-
- gettimeofday(&now, NULL);
-
- /* don't send a new start until at least SECS after the last
- we sent, and at least SECS after the last completed cycle */
-
- sec = now.tv_sec - ls->last_send_cycle_start.tv_sec;
-
- if (sec < DEADLOCK_CHECK_SECS) {
- log_group(ls, "skip send: recent send cycle %d sec", sec);
- return;
- }
-
- sec = now.tv_sec - ls->cycle_end_time.tv_sec;
-
- if (sec < DEADLOCK_CHECK_SECS) {
- log_group(ls, "skip send: recent cycle end %d sec", sec);
- return;
- }
-
- gettimeofday(&ls->last_send_cycle_start, NULL);
-
- if (cfgd_enable_deadlk)
- send_cycle_start(ls);
-}
-
-void process_netlink(int ci)
-{
- struct msgtemplate msg;
- struct nlattr *na;
- int len;
- int fd;
-
- fd = client_fd(ci);
-
- len = recv(fd, &msg, sizeof(msg), 0);
-
- if (len < 0) {
- log_error("nonfatal netlink error: errno %d", errno);
- return;
- }
-
- if (msg.n.nlmsg_type == NLMSG_ERROR || !NLMSG_OK((&msg.n), len)) {
- struct nlmsgerr *err = NLMSG_DATA(&msg);
- log_error("fatal netlink error: errno %d", err->error);
- return;
- }
-
- na = (struct nlattr *) GENLMSG_DATA(&msg);
-
- process_timewarn((struct dlm_lock_data *) NLA_DATA(na));
-}
-
diff --git a/group/dlm_controld/plock.c b/group/dlm_controld/plock.c
deleted file mode 100644
index c38b998..0000000
--- a/group/dlm_controld/plock.c
+++ /dev/null
@@ -1,2197 +0,0 @@
-#include "dlm_daemon.h"
-#include <linux/dlm_plock.h>
-
-/* FIXME: remove this once everyone is using the version of
- * dlm_plock.h which defines it */
-
-#ifndef DLM_PLOCK_FL_CLOSE
-#warning DLM_PLOCK_FL_CLOSE undefined. Enabling build workaround.
-#define DLM_PLOCK_FL_CLOSE 1
-#define DLM_PLOCK_BUILD_WORKAROUND 1
-#endif
-
-static uint32_t plock_read_count;
-static uint32_t plock_recv_count;
-static uint32_t plock_rate_delays;
-static struct timeval plock_read_time;
-static struct timeval plock_recv_time;
-static struct timeval plock_rate_last;
-
-static int plock_device_fd = -1;
-static int need_fsid_translation = 0;
-
-extern int message_flow_control_on;
-
-#define RD_CONTINUE 0x00000001
-
-struct resource_data {
- uint64_t number;
- int owner;
- uint32_t lock_count;
- uint32_t flags;
- uint32_t pad;
-};
-
-struct plock_data {
- uint64_t start;
- uint64_t end;
- uint64_t owner;
- uint32_t pid;
- uint32_t nodeid;
- uint8_t ex;
- uint8_t waiter;
- uint16_t pad1;
- uint32_t pad;
-};
-
-#define R_GOT_UNOWN 0x00000001 /* have received owner=0 message */
-#define R_SEND_UNOWN 0x00000002 /* have sent owner=0 message */
-#define R_SEND_OWN 0x00000004 /* have sent owner=our_nodeid message */
-#define R_PURGE_UNOWN 0x00000008 /* set owner=0 in purge */
-#define R_SEND_DROP 0x00000010
-
-struct resource {
- struct list_head list; /* list of resources */
- uint64_t number;
- int owner; /* nodeid or 0 for unowned */
- uint32_t flags;
- struct timeval last_access;
- struct list_head locks; /* one lock for each range */
- struct list_head waiters;
- struct list_head pending; /* discovering r owner */
- struct rb_node rb_node;
-};
-
-#define P_SYNCING 0x00000001 /* plock has been sent as part of sync but not
- yet received */
-
-struct posix_lock {
- struct list_head list; /* resource locks or waiters list */
- uint32_t pid;
- uint64_t owner;
- uint64_t start;
- uint64_t end;
- int ex;
- int nodeid;
- uint32_t flags;
-};
-
-struct lock_waiter {
- struct list_head list;
- uint32_t flags;
- struct dlm_plock_info info;
-};
-
-struct save_msg {
- struct list_head list;
- int nodeid;
- int len;
- int type;
- char buf[0];
-};
-
-
-static void send_own(struct lockspace *ls, struct resource *r, int owner);
-static void save_pending_plock(struct lockspace *ls, struct resource *r,
- struct dlm_plock_info *in);
-
-
-static int got_unown(struct resource *r)
-{
- return !!(r->flags & R_GOT_UNOWN);
-}
-
-static void info_bswap_out(struct dlm_plock_info *i)
-{
- i->version[0] = cpu_to_le32(i->version[0]);
- i->version[1] = cpu_to_le32(i->version[1]);
- i->version[2] = cpu_to_le32(i->version[2]);
- i->pid = cpu_to_le32(i->pid);
- i->nodeid = cpu_to_le32(i->nodeid);
- i->rv = cpu_to_le32(i->rv);
- i->fsid = cpu_to_le32(i->fsid);
- i->number = cpu_to_le64(i->number);
- i->start = cpu_to_le64(i->start);
- i->end = cpu_to_le64(i->end);
- i->owner = cpu_to_le64(i->owner);
-}
-
-static void info_bswap_in(struct dlm_plock_info *i)
-{
- i->version[0] = le32_to_cpu(i->version[0]);
- i->version[1] = le32_to_cpu(i->version[1]);
- i->version[2] = le32_to_cpu(i->version[2]);
- i->pid = le32_to_cpu(i->pid);
- i->nodeid = le32_to_cpu(i->nodeid);
- i->rv = le32_to_cpu(i->rv);
- i->fsid = le32_to_cpu(i->fsid);
- i->number = le64_to_cpu(i->number);
- i->start = le64_to_cpu(i->start);
- i->end = le64_to_cpu(i->end);
- i->owner = le64_to_cpu(i->owner);
-}
-
-static const char *op_str(int optype)
-{
- switch (optype) {
- case DLM_PLOCK_OP_LOCK:
- return "LK";
- case DLM_PLOCK_OP_UNLOCK:
- return "UN";
- case DLM_PLOCK_OP_GET:
- return "GET";
- default:
- return "??";
- }
-}
-
-static const char *ex_str(int optype, int ex)
-{
- if (optype == DLM_PLOCK_OP_UNLOCK || optype == DLM_PLOCK_OP_GET)
- return "-";
- if (ex)
- return "WR";
- else
- return "RD";
-}
-
-/*
- * In kernels before 2.6.26, plocks came from gfs2's lock_dlm module.
- * Reading plocks from there as well should allow us to use cluster3
- * on old (RHEL5) kernels. In this case, the fsid we read in plock_info
- * structs is the mountgroup id, which we need to translate to the ls id.
- */
-
-int setup_plocks(void)
-{
- plock_read_count = 0;
- plock_recv_count = 0;
- plock_rate_delays = 0;
- gettimeofday(&plock_read_time, NULL);
- gettimeofday(&plock_recv_time, NULL);
- gettimeofday(&plock_rate_last, NULL);
-
- if (plock_minor) {
- plock_device_fd = open("/dev/misc/dlm_plock", O_RDWR);
- } else if (old_plock_minor) {
- log_debug("setup_plocks using old lock_dlm interface");
- need_fsid_translation = 1;
- plock_device_fd = open("/dev/misc/lock_dlm_plock", O_RDWR);
- }
-
- if (plock_device_fd < 0) {
- log_error("Failure to open plock device: %s", strerror(errno));
- return -1;
- }
-
- log_debug("plocks %d", plock_device_fd);
- log_debug("plock cpg message size: %u bytes",
- (unsigned int) (sizeof(struct dlm_header) +
- sizeof(struct dlm_plock_info)));
-
- return plock_device_fd;
-}
-
-void close_plocks(void)
-{
- if (plock_device_fd > 0)
- close(plock_device_fd);
-}
-
-static uint32_t mg_to_ls_id(uint32_t fsid)
-{
- struct lockspace *ls;
- int do_set = 1;
-
- retry:
- list_for_each_entry(ls, &lockspaces, list) {
- if (ls->associated_mg_id == fsid)
- return ls->global_id;
- }
-
- if (do_set) {
- do_set = 0;
- set_associated_id(fsid);
- goto retry;
- }
-
- return fsid;
-}
-
-/* FIXME: unify these two */
-
-static unsigned long time_diff_ms(struct timeval *begin, struct timeval *end)
-{
- struct timeval result;
- timersub(end, begin, &result);
- return (result.tv_sec * 1000) + (result.tv_usec / 1000);
-}
-
-static uint64_t dt_usec(struct timeval *start, struct timeval *stop)
-{
- uint64_t dt;
-
- dt = stop->tv_sec - start->tv_sec;
- dt *= 1000000;
- dt += stop->tv_usec - start->tv_usec;
- return dt;
-}
-
-static struct resource * rb_search_plock_resource(struct lockspace *ls, uint64_t number)
-{
- struct rb_node *n = ls->plock_resources_root.rb_node;
- struct resource *r;
-
- while (n) {
- r = rb_entry(n, struct resource, rb_node);
- if (number < r->number)
- n = n->rb_left;
- else if (number > r->number)
- n = n->rb_right;
- else
- return r;
- }
- return NULL;
-}
-
-static void rb_insert_plock_resource(struct lockspace *ls, struct resource *r)
-{
- struct resource *entry;
- struct rb_node **p;
- struct rb_node *parent = NULL;
-
- p = &ls->plock_resources_root.rb_node;
- while (*p) {
- parent = *p;
- entry = rb_entry(parent, struct resource, rb_node);
- if (r->number < entry->number)
- p = &parent->rb_left;
- else if (r->number > entry->number)
- p = &parent->rb_right;
- else
- return;
- }
- rb_link_node(&r->rb_node, parent, p);
- rb_insert_color(&r->rb_node, &ls->plock_resources_root);
-}
-
-static void rb_del_plock_resource(struct lockspace *ls, struct resource *r)
-{
- if (!RB_EMPTY_NODE(&r->rb_node)) {
- rb_erase(&r->rb_node, &ls->plock_resources_root);
- RB_CLEAR_NODE(&r->rb_node);
- }
-}
-
-static struct resource *search_resource(struct lockspace *ls, uint64_t number)
-{
- struct resource *r;
-
- list_for_each_entry(r, &ls->plock_resources, list) {
- if (r->number == number)
- return r;
- }
- return NULL;
-}
-
-static int find_resource(struct lockspace *ls, uint64_t number, int create,
- struct resource **r_out)
-{
- struct resource *r = NULL;
- int rv = 0;
-
- r = rb_search_plock_resource(ls, number);
- if (r)
- goto out;
-
- if (create == 0) {
- rv = -ENOENT;
- goto out;
- }
-
- r = malloc(sizeof(struct resource));
- if (!r) {
- log_elock(ls, "find_resource no memory %d", errno);
- rv = -ENOMEM;
- goto out;
- }
-
- memset(r, 0, sizeof(struct resource));
- r->number = number;
- INIT_LIST_HEAD(&r->locks);
- INIT_LIST_HEAD(&r->waiters);
- INIT_LIST_HEAD(&r->pending);
-
- if (cfgd_plock_ownership)
- r->owner = -1;
- else
- r->owner = 0;
-
- list_add_tail(&r->list, &ls->plock_resources);
- rb_insert_plock_resource(ls, r);
- out:
- if (r)
- gettimeofday(&r->last_access, NULL);
- *r_out = r;
- return rv;
-}
-
-static void put_resource(struct lockspace *ls, struct resource *r)
-{
- /* with ownership, resources are only freed via drop messages */
- if (cfgd_plock_ownership)
- return;
-
- if (list_empty(&r->locks) && list_empty(&r->waiters)) {
- rb_del_plock_resource(ls, r);
- list_del(&r->list);
- free(r);
- }
-}
-
-static inline int ranges_overlap(uint64_t start1, uint64_t end1,
- uint64_t start2, uint64_t end2)
-{
- if (end1 < start2 || start1 > end2)
- return 0;
- return 1;
-}
-
-/**
- * overlap_type - returns a value based on the type of overlap
- * @s1 - start of new lock range
- * @e1 - end of new lock range
- * @s2 - start of existing lock range
- * @e2 - end of existing lock range
- *
- */
-
-static int overlap_type(uint64_t s1, uint64_t e1, uint64_t s2, uint64_t e2)
-{
- int ret;
-
- /*
- * ---r1---
- * ---r2---
- */
-
- if (s1 == s2 && e1 == e2)
- ret = 0;
-
- /*
- * --r1--
- * ---r2---
- */
-
- else if (s1 == s2 && e1 < e2)
- ret = 1;
-
- /*
- * --r1--
- * ---r2---
- */
-
- else if (s1 > s2 && e1 == e2)
- ret = 1;
-
- /*
- * --r1--
- * ---r2---
- */
-
- else if (s1 > s2 && e1 < e2)
- ret = 2;
-
- /*
- * ---r1--- or ---r1--- or ---r1---
- * --r2-- --r2-- --r2--
- */
-
- else if (s1 <= s2 && e1 >= e2)
- ret = 3;
-
- /*
- * ---r1---
- * ---r2---
- */
-
- else if (s1 > s2 && e1 > e2)
- ret = 4;
-
- /*
- * ---r1---
- * ---r2---
- */
-
- else if (s1 < s2 && e1 < e2)
- ret = 4;
-
- else
- ret = -1;
-
- return ret;
-}
-
-/* shrink the range start2:end2 by the partially overlapping start:end */
-
-static int shrink_range2(uint64_t *start2, uint64_t *end2,
- uint64_t start, uint64_t end)
-{
- int error = 0;
-
- if (*start2 < start)
- *end2 = start - 1;
- else if (*end2 > end)
- *start2 = end + 1;
- else
- error = -1;
- return error;
-}
-
-static int shrink_range(struct posix_lock *po, uint64_t start, uint64_t end)
-{
- return shrink_range2(&po->start, &po->end, start, end);
-}
-
-static int is_conflict(struct resource *r, struct dlm_plock_info *in, int get)
-{
- struct posix_lock *po;
-
- list_for_each_entry(po, &r->locks, list) {
- if (po->nodeid == in->nodeid && po->owner == in->owner)
- continue;
- if (!ranges_overlap(po->start, po->end, in->start, in->end))
- continue;
-
- if (in->ex || po->ex) {
- if (get) {
- in->ex = po->ex;
- in->pid = po->pid;
- in->start = po->start;
- in->end = po->end;
- }
- return 1;
- }
- }
- return 0;
-}
-
-static int add_lock(struct resource *r, uint32_t nodeid, uint64_t owner,
- uint32_t pid, int ex, uint64_t start, uint64_t end)
-{
- struct posix_lock *po;
-
- po = malloc(sizeof(struct posix_lock));
- if (!po)
- return -ENOMEM;
- memset(po, 0, sizeof(struct posix_lock));
-
- po->start = start;
- po->end = end;
- po->nodeid = nodeid;
- po->owner = owner;
- po->pid = pid;
- po->ex = ex;
- list_add_tail(&po->list, &r->locks);
-
- return 0;
-}
-
-/* RN within RE (and starts or ends on RE boundary)
- 1. add new lock for non-overlap area of RE, orig mode
- 2. convert RE to RN range and mode */
-
-static int lock_case1(struct posix_lock *po, struct resource *r,
- struct dlm_plock_info *in)
-{
- uint64_t start2, end2;
- int rv;
-
- /* non-overlapping area start2:end2 */
- start2 = po->start;
- end2 = po->end;
- rv = shrink_range2(&start2, &end2, in->start, in->end);
- if (rv)
- goto out;
-
- po->start = in->start;
- po->end = in->end;
- po->ex = in->ex;
-
- rv = add_lock(r, in->nodeid, in->owner, in->pid, !in->ex, start2, end2);
- out:
- return rv;
-}
-
-/* RN within RE (RE overlaps RN on both sides)
- 1. add new lock for front fragment, orig mode
- 2. add new lock for back fragment, orig mode
- 3. convert RE to RN range and mode */
-
-static int lock_case2(struct posix_lock *po, struct resource *r,
- struct dlm_plock_info *in)
-
-{
- int rv;
-
- rv = add_lock(r, in->nodeid, in->owner, in->pid,
- !in->ex, po->start, in->start - 1);
- if (rv)
- goto out;
-
- rv = add_lock(r, in->nodeid, in->owner, in->pid,
- !in->ex, in->end + 1, po->end);
- if (rv)
- goto out;
-
- po->start = in->start;
- po->end = in->end;
- po->ex = in->ex;
- out:
- return rv;
-}
-
-static int lock_internal(struct lockspace *ls, struct resource *r,
- struct dlm_plock_info *in)
-{
- struct posix_lock *po, *safe;
- int rv = 0;
-
- list_for_each_entry_safe(po, safe, &r->locks, list) {
- if (po->nodeid != in->nodeid || po->owner != in->owner)
- continue;
- if (!ranges_overlap(po->start, po->end, in->start, in->end))
- continue;
-
- /* existing range (RE) overlaps new range (RN) */
-
- switch(overlap_type(in->start, in->end, po->start, po->end)) {
-
- case 0:
- if (po->ex == in->ex)
- goto out;
-
- /* ranges the same - just update the existing lock */
- po->ex = in->ex;
- goto out;
-
- case 1:
- if (po->ex == in->ex)
- goto out;
-
- rv = lock_case1(po, r, in);
- goto out;
-
- case 2:
- if (po->ex == in->ex)
- goto out;
-
- rv = lock_case2(po, r, in);
- goto out;
-
- case 3:
- list_del(&po->list);
- free(po);
- break;
-
- case 4:
- if (po->start < in->start)
- po->end = in->start - 1;
- else
- po->start = in->end + 1;
- break;
-
- default:
- rv = -1;
- goto out;
- }
- }
-
- rv = add_lock(r, in->nodeid, in->owner, in->pid,
- in->ex, in->start, in->end);
- out:
- return rv;
-
-}
-
-static int unlock_internal(struct lockspace *ls, struct resource *r,
- struct dlm_plock_info *in)
-{
- struct posix_lock *po, *safe;
- int rv = 0;
-
- list_for_each_entry_safe(po, safe, &r->locks, list) {
- if (po->nodeid != in->nodeid || po->owner != in->owner)
- continue;
- if (!ranges_overlap(po->start, po->end, in->start, in->end))
- continue;
-
- /* existing range (RE) overlaps new range (RN) */
-
- switch (overlap_type(in->start, in->end, po->start, po->end)) {
-
- case 0:
- /* ranges the same - just remove the existing lock */
-
- list_del(&po->list);
- free(po);
- goto out;
-
- case 1:
- /* RN within RE and starts or ends on RE boundary -
- * shrink and update RE */
-
- rv = shrink_range(po, in->start, in->end);
- goto out;
-
- case 2:
- /* RN within RE - shrink and update RE to be front
- * fragment, and add a new lock for back fragment */
-
- rv = add_lock(r, in->nodeid, in->owner, in->pid,
- po->ex, in->end + 1, po->end);
- po->end = in->start - 1;
- goto out;
-
- case 3:
- /* RE within RN - remove RE, then continue checking
- * because RN could cover other locks */
-
- list_del(&po->list);
- free(po);
- continue;
-
- case 4:
- /* front of RE in RN, or end of RE in RN - shrink and
- * update RE, then continue because RN could cover
- * other locks */
-
- rv = shrink_range(po, in->start, in->end);
- continue;
-
- default:
- rv = -1;
- goto out;
- }
- }
- out:
- return rv;
-}
-
-static void clear_waiters(struct lockspace *ls, struct resource *r,
- struct dlm_plock_info *in)
-{
- struct lock_waiter *w, *safe;
-
- list_for_each_entry_safe(w, safe, &r->waiters, list) {
- if (w->info.nodeid != in->nodeid || w->info.owner != in->owner)
- continue;
-
- list_del(&w->list);
-
- log_elock(ls, "clear waiter %llx %llx-%llx %d/%u/%llx",
- (unsigned long long)in->number,
- (unsigned long long)in->start,
- (unsigned long long)in->end,
- in->nodeid, in->pid,
- (unsigned long long)in->owner);
- free(w);
- }
-}
-
-static int add_waiter(struct lockspace *ls, struct resource *r,
- struct dlm_plock_info *in)
-
-{
- struct lock_waiter *w;
-
- w = malloc(sizeof(struct lock_waiter));
- if (!w)
- return -ENOMEM;
- memcpy(&w->info, in, sizeof(struct dlm_plock_info));
- list_add_tail(&w->list, &r->waiters);
- return 0;
-}
-
-static void write_result(struct lockspace *ls, struct dlm_plock_info *in,
- int rv)
-{
- if (need_fsid_translation)
- in->fsid = ls->associated_mg_id;
-
- in->rv = rv;
- write(plock_device_fd, in, sizeof(struct dlm_plock_info));
-}
-
-static void do_waiters(struct lockspace *ls, struct resource *r)
-{
- struct lock_waiter *w, *safe;
- struct dlm_plock_info *in;
- int rv;
-
- list_for_each_entry_safe(w, safe, &r->waiters, list) {
- in = &w->info;
-
- if (is_conflict(r, in, 0))
- continue;
-
- list_del(&w->list);
-
- /*
- log_group(ls, "take waiter %llx %llx-%llx %d/%u/%llx",
- in->number, in->start, in->end,
- in->nodeid, in->pid, in->owner);
- */
-
- rv = lock_internal(ls, r, in);
-
- if (in->nodeid == our_nodeid)
- write_result(ls, in, rv);
-
- free(w);
- }
-}
-
-static void do_lock(struct lockspace *ls, struct dlm_plock_info *in,
- struct resource *r)
-{
- int rv;
-
- if (is_conflict(r, in, 0)) {
- if (!in->wait)
- rv = -EAGAIN;
- else {
- rv = add_waiter(ls, r, in);
- if (rv)
- goto out;
- rv = -EINPROGRESS;
- }
- } else
- rv = lock_internal(ls, r, in);
-
- out:
- if (in->nodeid == our_nodeid && rv != -EINPROGRESS)
- write_result(ls, in, rv);
-
- do_waiters(ls, r);
- put_resource(ls, r);
-}
-
-static void do_unlock(struct lockspace *ls, struct dlm_plock_info *in,
- struct resource *r)
-{
- int rv;
-
- rv = unlock_internal(ls, r, in);
-
-#ifdef DLM_PLOCK_BUILD_WORKAROUND
- if (in->pad & DLM_PLOCK_FL_CLOSE) {
-#else
- if (in->flags & DLM_PLOCK_FL_CLOSE) {
-#endif
- clear_waiters(ls, r, in);
- /* no replies for unlock-close ops */
- goto skip_result;
- }
-
- if (in->nodeid == our_nodeid)
- write_result(ls, in, rv);
-
- skip_result:
- do_waiters(ls, r);
- put_resource(ls, r);
-}
-
-/* we don't even get to this function if the getlk isn't from us */
-
-static void do_get(struct lockspace *ls, struct dlm_plock_info *in,
- struct resource *r)
-{
- int rv;
-
- if (is_conflict(r, in, 1))
- rv = 1;
- else
- rv = 0;
-
- write_result(ls, in, rv);
- put_resource(ls, r);
-}
-
-static void save_message(struct lockspace *ls, struct dlm_header *hd, int len,
- int from, int type)
-{
- struct save_msg *sm;
-
- sm = malloc(sizeof(struct save_msg) + len);
- if (!sm)
- return;
- memset(sm, 0, sizeof(struct save_msg) + len);
-
- memcpy(&sm->buf, hd, len);
- sm->type = type;
- sm->len = len;
- sm->nodeid = from;
-
- log_plock(ls, "save %s from %d len %d", msg_name(type), from, len);
-
- list_add_tail(&sm->list, &ls->saved_messages);
-}
-
-static void __receive_plock(struct lockspace *ls, struct dlm_plock_info *in,
- int from, struct resource *r)
-{
- switch (in->optype) {
- case DLM_PLOCK_OP_LOCK:
- ls->last_plock_time = time(NULL);
- do_lock(ls, in, r);
- break;
- case DLM_PLOCK_OP_UNLOCK:
- ls->last_plock_time = time(NULL);
- do_unlock(ls, in, r);
- break;
- case DLM_PLOCK_OP_GET:
- do_get(ls, in, r);
- break;
- default:
- log_elock(ls, "receive_plock error from %d optype %d",
- from, in->optype);
- if (from == our_nodeid)
- write_result(ls, in, -EINVAL);
- }
-}
-
-/* When ls members receive our options message (for our mount), one of them
- saves all plock state received to that point in a checkpoint and then sends
- us our journals message. We know to retrieve the plock state from the
- checkpoint when we receive our journals message. Any plocks messages that
- arrive between seeing our options message and our journals message needs to
- be saved and processed after we synchronize our plock state from the
- checkpoint. Any plock message received while we're mounting but before we
- set save_plocks (when we see our options message) can be ignored because it
- should be reflected in the checkpointed state. */
-
-static void _receive_plock(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- struct dlm_plock_info info;
- struct resource *r = NULL;
- struct timeval now;
- uint64_t usec;
- int from = hd->nodeid;
- int rv, create;
-
- memcpy(&info, (char *)hd + sizeof(struct dlm_header), sizeof(info));
- info_bswap_in(&info);
-
- log_plock(ls, "receive plock %llx %s %s %llx-%llx %d/%u/%llx w %d",
- (unsigned long long)info.number,
- op_str(info.optype),
- ex_str(info.optype, info.ex),
- (unsigned long long)info.start, (unsigned long long)info.end,
- info.nodeid, info.pid, (unsigned long long)info.owner,
- info.wait);
-
- plock_recv_count++;
- if (!(plock_recv_count % 1000)) {
- gettimeofday(&now, NULL);
- usec = dt_usec(&plock_recv_time, &now);
- log_plock(ls, "plock_recv_count %u time %.3f s",
- plock_recv_count, usec * 1.e-6);
- plock_recv_time = now;
- }
-
- if (info.optype == DLM_PLOCK_OP_GET && from != our_nodeid)
- return;
-
- if (from != hd->nodeid || from != info.nodeid) {
- log_elock(ls, "receive_plock error from %d header %d info %d",
- from, hd->nodeid, info.nodeid);
- return;
- }
-
- create = !cfgd_plock_ownership;
-
- rv = find_resource(ls, info.number, create, &r);
-
- if (rv && cfgd_plock_ownership) {
- /* There must have been a race with a drop, so we need to
- ignore this plock op which will be resent. If we're the one
- who sent the plock, we need to send_own() and put it on the
- pending list to resend once the owner is established. */
-
- log_plock(ls, "receive_plock from %d no r %llx", from,
- (unsigned long long)info.number);
-
- if (from != our_nodeid)
- return;
-
- rv = find_resource(ls, info.number, 1, &r);
- if (rv)
- return;
- send_own(ls, r, our_nodeid);
- save_pending_plock(ls, r, &info);
- return;
- }
- if (rv) {
- /* r not found, rv is -ENOENT, this shouldn't happen because
- process_plocks() creates a resource for every op */
-
- log_elock(ls, "receive_plock error from %d no r %llx %d",
- from, (unsigned long long)info.number, rv);
- return;
- }
-
- /* The owner should almost always be 0 here, but other owners may
- be possible given odd combinations of races with drop. Odd races to
- worry about (some seem pretty improbable):
-
- - A sends drop, B sends plock, receive drop, receive plock.
- This is addressed above.
-
- - A sends drop, B sends plock, receive drop, B reads plock
- and sends own, receive plock, on B we find owner of -1.
-
- - A sends drop, B sends two plocks, receive drop, receive plocks.
- Receiving the first plock is the previous case, receiving the
- second plock will find r with owner of -1.
-
- - A sends drop, B sends two plocks, receive drop, C sends own,
- receive plock, B sends own, receive own (C), receive plock,
- receive own (B).
-
- Haven't tried to cook up a scenario that would lead to the
- last case below; receiving a plock from ourself and finding
- we're the owner of r. */
-
- if (!r->owner) {
- __receive_plock(ls, &info, from, r);
-
- } else if (r->owner == -1) {
- log_plock(ls, "receive_plock from %d r %llx owner %d", from,
- (unsigned long long)info.number, r->owner);
-
- if (from == our_nodeid)
- save_pending_plock(ls, r, &info);
-
- } else if (r->owner != our_nodeid) {
- log_plock(ls, "receive_plock from %d r %llx owner %d", from,
- (unsigned long long)info.number, r->owner);
-
- if (from == our_nodeid)
- save_pending_plock(ls, r, &info);
-
- } else if (r->owner == our_nodeid) {
- log_plock(ls, "receive_plock from %d r %llx owner %d", from,
- (unsigned long long)info.number, r->owner);
-
- if (from == our_nodeid)
- __receive_plock(ls, &info, from, r);
- }
-}
-
-void receive_plock(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- if (ls->save_plocks) {
- save_message(ls, hd, len, hd->nodeid, DLM_MSG_PLOCK);
- return;
- }
-
- _receive_plock(ls, hd, len);
-}
-
-static int send_struct_info(struct lockspace *ls, struct dlm_plock_info *in,
- int msg_type)
-{
- struct dlm_header *hd;
- int rv = 0, len;
- char *buf;
-
- len = sizeof(struct dlm_header) + sizeof(struct dlm_plock_info);
- buf = malloc(len);
- if (!buf) {
- rv = -ENOMEM;
- goto out;
- }
- memset(buf, 0, len);
-
- info_bswap_out(in);
-
- hd = (struct dlm_header *)buf;
- hd->type = msg_type;
-
- memcpy(buf + sizeof(struct dlm_header), in, sizeof(*in));
-
- dlm_send_message(ls, buf, len);
-
- free(buf);
- out:
- if (rv)
- log_elock(ls, "send_struct_info error %d", rv);
- return rv;
-}
-
-static void send_plock(struct lockspace *ls, struct resource *r,
- struct dlm_plock_info *in)
-{
- send_struct_info(ls, in, DLM_MSG_PLOCK);
-}
-
-static void send_own(struct lockspace *ls, struct resource *r, int owner)
-{
- struct dlm_plock_info info;
-
- /* if we've already sent an own message for this resource,
- (pending list is not empty), then we shouldn't send another */
-
- if (!list_empty(&r->pending)) {
- log_plock(ls, "send_own %llx already pending",
- (unsigned long long)r->number);
- return;
- }
-
- if (!owner)
- r->flags |= R_SEND_UNOWN;
- else
- r->flags |= R_SEND_OWN;
-
- memset(&info, 0, sizeof(info));
- info.number = r->number;
- info.nodeid = owner;
-
- send_struct_info(ls, &info, DLM_MSG_PLOCK_OWN);
-}
-
-static void send_syncs(struct lockspace *ls, struct resource *r)
-{
- struct dlm_plock_info info;
- struct posix_lock *po;
- struct lock_waiter *w;
- int rv;
-
- list_for_each_entry(po, &r->locks, list) {
- memset(&info, 0, sizeof(info));
- info.number = r->number;
- info.start = po->start;
- info.end = po->end;
- info.nodeid = po->nodeid;
- info.owner = po->owner;
- info.pid = po->pid;
- info.ex = po->ex;
-
- rv = send_struct_info(ls, &info, DLM_MSG_PLOCK_SYNC_LOCK);
- if (rv)
- goto out;
-
- po->flags |= P_SYNCING;
- }
-
- list_for_each_entry(w, &r->waiters, list) {
- memcpy(&info, &w->info, sizeof(info));
-
- rv = send_struct_info(ls, &info, DLM_MSG_PLOCK_SYNC_WAITER);
- if (rv)
- goto out;
-
- w->flags |= P_SYNCING;
- }
- out:
- return;
-}
-
-static void send_drop(struct lockspace *ls, struct resource *r)
-{
- struct dlm_plock_info info;
-
- memset(&info, 0, sizeof(info));
- info.number = r->number;
- r->flags |= R_SEND_DROP;
-
- send_struct_info(ls, &info, DLM_MSG_PLOCK_DROP);
-}
-
-/* plock op can't be handled until we know the owner value of the resource,
- so the op is saved on the pending list until the r owner is established */
-
-static void save_pending_plock(struct lockspace *ls, struct resource *r,
- struct dlm_plock_info *in)
-{
- struct lock_waiter *w;
-
- w = malloc(sizeof(struct lock_waiter));
- if (!w) {
- log_elock(ls, "save_pending_plock no mem");
- return;
- }
- memcpy(&w->info, in, sizeof(struct dlm_plock_info));
- list_add_tail(&w->list, &r->pending);
-}
-
-/* plock ops are on pending list waiting for ownership to be established.
- owner has now become us, so add these plocks to r */
-
-static void add_pending_plocks(struct lockspace *ls, struct resource *r)
-{
- struct lock_waiter *w, *safe;
-
- list_for_each_entry_safe(w, safe, &r->pending, list) {
- __receive_plock(ls, &w->info, our_nodeid, r);
- list_del(&w->list);
- free(w);
- }
-}
-
-/* plock ops are on pending list waiting for ownership to be established.
- owner has now become 0, so send these plocks to everyone */
-
-static void send_pending_plocks(struct lockspace *ls, struct resource *r)
-{
- struct lock_waiter *w, *safe;
-
- list_for_each_entry_safe(w, safe, &r->pending, list) {
- send_plock(ls, r, &w->info);
- list_del(&w->list);
- free(w);
- }
-}
-
-static void _receive_own(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- struct dlm_plock_info info;
- struct resource *r;
- int should_not_happen = 0;
- int from = hd->nodeid;
- int rv;
-
- ls->last_plock_time = time(NULL);
-
- memcpy(&info, (char *)hd + sizeof(struct dlm_header), sizeof(info));
- info_bswap_in(&info);
-
- log_plock(ls, "receive_own %llx from %u owner %u",
- (unsigned long long)info.number, hd->nodeid, info.nodeid);
-
- rv = find_resource(ls, info.number, 1, &r);
- if (rv)
- return;
-
- if (from == our_nodeid) {
- /*
- * received our own own message
- */
-
- if (info.nodeid == 0) {
- /* we are setting owner to 0 */
-
- if (r->owner == our_nodeid) {
- /* we set owner to 0 when we relinquish
- ownership */
- should_not_happen = 1;
- } else if (r->owner == 0) {
- /* this happens when we relinquish ownership */
- r->flags |= R_GOT_UNOWN;
- } else {
- should_not_happen = 1;
- }
-
- } else if (info.nodeid == our_nodeid) {
- /* we are setting owner to ourself */
-
- if (r->owner == -1) {
- /* we have gained ownership */
- r->owner = our_nodeid;
- add_pending_plocks(ls, r);
- } else if (r->owner == our_nodeid) {
- should_not_happen = 1;
- } else if (r->owner == 0) {
- send_pending_plocks(ls, r);
- } else {
- /* resource is owned by other node;
- they should set owner to 0 shortly */
- }
-
- } else {
- /* we should only ever set owner to 0 or ourself */
- should_not_happen = 1;
- }
- } else {
- /*
- * received own message from another node
- */
-
- if (info.nodeid == 0) {
- /* other node is setting owner to 0 */
-
- if (r->owner == -1) {
- /* we should have a record of the owner before
- it relinquishes */
- should_not_happen = 1;
- } else if (r->owner == our_nodeid) {
- /* only the owner should relinquish */
- should_not_happen = 1;
- } else if (r->owner == 0) {
- should_not_happen = 1;
- } else {
- r->owner = 0;
- r->flags |= R_GOT_UNOWN;
- send_pending_plocks(ls, r);
- }
-
- } else if (info.nodeid == from) {
- /* other node is setting owner to itself */
-
- if (r->owner == -1) {
- /* normal path for a node becoming owner */
- r->owner = from;
- } else if (r->owner == our_nodeid) {
- /* we relinquish our ownership: sync our local
- plocks to everyone, then set owner to 0 */
- send_syncs(ls, r);
- send_own(ls, r, 0);
- /* we need to set owner to 0 here because
- local ops may arrive before we receive
- our send_own message and can't be added
- locally */
- r->owner = 0;
- } else if (r->owner == 0) {
- /* can happen because we set owner to 0 before
- we receive our send_own sent just above */
- } else {
- /* do nothing, current owner should be
- relinquishing its ownership */
- }
-
- } else if (info.nodeid == our_nodeid) {
- /* no one else should try to set the owner to us */
- should_not_happen = 1;
- } else {
- /* a node should only ever set owner to 0 or itself */
- should_not_happen = 1;
- }
- }
-
- if (should_not_happen) {
- log_elock(ls, "receive_own error from %u %llx "
- "info nodeid %d r owner %d",
- from, (unsigned long long)r->number,
- info.nodeid, r->owner);
- }
-}
-
-void receive_own(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- if (ls->save_plocks) {
- save_message(ls, hd, len, hd->nodeid, DLM_MSG_PLOCK_OWN);
- return;
- }
-
- _receive_own(ls, hd, len);
-}
-
-static void clear_syncing_flag(struct lockspace *ls, struct resource *r,
- struct dlm_plock_info *in)
-{
- struct posix_lock *po;
- struct lock_waiter *w;
-
- list_for_each_entry(po, &r->locks, list) {
- if ((po->flags & P_SYNCING) &&
- in->start == po->start &&
- in->end == po->end &&
- in->nodeid == po->nodeid &&
- in->owner == po->owner &&
- in->pid == po->pid &&
- in->ex == po->ex) {
- po->flags &= ~P_SYNCING;
- return;
- }
- }
-
- list_for_each_entry(w, &r->waiters, list) {
- if ((w->flags & P_SYNCING) &&
- in->start == w->info.start &&
- in->end == w->info.end &&
- in->nodeid == w->info.nodeid &&
- in->owner == w->info.owner &&
- in->pid == w->info.pid &&
- in->ex == w->info.ex) {
- w->flags &= ~P_SYNCING;
- return;
- }
- }
-
- log_elock(ls, "clear_syncing error %llx no match %s %llx-%llx %d/%u/%llx",
- (unsigned long long)r->number,
- in->ex ? "WR" : "RD",
- (unsigned long long)in->start,
- (unsigned long long)in->end,
- in->nodeid, in->pid,
- (unsigned long long)in->owner);
-}
-
-static void _receive_sync(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- struct dlm_plock_info info;
- struct resource *r;
- int from = hd->nodeid;
- int rv;
-
- ls->last_plock_time = time(NULL);
-
- memcpy(&info, (char *)hd + sizeof(struct dlm_header), sizeof(info));
- info_bswap_in(&info);
-
- log_plock(ls, "receive sync %llx from %u %s %llx-%llx %d/%u/%llx",
- (unsigned long long)info.number, from, info.ex ? "WR" : "RD",
- (unsigned long long)info.start, (unsigned long long)info.end,
- info.nodeid, info.pid, (unsigned long long)info.owner);
-
- rv = find_resource(ls, info.number, 0, &r);
- if (rv) {
- log_elock(ls, "receive_sync error no r %llx from %d",
- info.number, from);
- return;
- }
-
- if (from == our_nodeid) {
- /* this plock now in sync on all nodes */
- clear_syncing_flag(ls, r, &info);
- return;
- }
-
- if (hd->type == DLM_MSG_PLOCK_SYNC_LOCK)
- add_lock(r, info.nodeid, info.owner, info.pid, info.ex,
- info.start, info.end);
- else if (hd->type == DLM_MSG_PLOCK_SYNC_WAITER)
- add_waiter(ls, r, &info);
-}
-
-void receive_sync(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- if (ls->save_plocks) {
- save_message(ls, hd, len, hd->nodeid, hd->type);
- return;
- }
-
- _receive_sync(ls, hd, len);
-}
-
-static void _receive_drop(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- struct dlm_plock_info info;
- struct resource *r;
- int from = hd->nodeid;
- int rv;
-
- ls->last_plock_time = time(NULL);
-
- memcpy(&info, (char *)hd + sizeof(struct dlm_header), sizeof(info));
- info_bswap_in(&info);
-
- log_plock(ls, "receive_drop %llx from %u",
- (unsigned long long)info.number, from);
-
- rv = find_resource(ls, info.number, 0, &r);
- if (rv) {
- /* we'll find no r if two nodes sent drop at once */
- log_plock(ls, "receive_drop from %d no r %llx", from,
- (unsigned long long)info.number);
- return;
- }
-
- if (r->owner != 0) {
- /* - A sent drop, B sent drop, receive drop A, C sent own,
- receive drop B (this warning on C, owner -1)
- - A sent drop, B sent drop, receive drop A, A sent own,
- receive own A, receive drop B (this warning on all,
- owner A) */
- log_plock(ls, "receive_drop from %d r %llx owner %d", from,
- (unsigned long long)r->number, r->owner);
- return;
- }
-
- if (!list_empty(&r->pending)) {
- /* shouldn't happen */
- log_elock(ls, "receive_drop error from %d r %llx pending op",
- from, (unsigned long long)r->number);
- return;
- }
-
- /* the decision to drop or not must be based on things that are
- guaranteed to be the same on all nodes */
-
- if (list_empty(&r->locks) && list_empty(&r->waiters)) {
- rb_del_plock_resource(ls, r);
- list_del(&r->list);
- free(r);
- } else {
- /* A sent drop, B sent a plock, receive plock, receive drop */
- log_plock(ls, "receive_drop from %d r %llx in use", from,
- (unsigned long long)r->number);
- }
-}
-
-void receive_drop(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- if (ls->save_plocks) {
- save_message(ls, hd, len, hd->nodeid, DLM_MSG_PLOCK_DROP);
- return;
- }
-
- _receive_drop(ls, hd, len);
-}
-
-/* We only drop resources from the unowned state to simplify things.
- If we want to drop a resource we own, we unown/relinquish it first. */
-
-/* FIXME: in the transition from owner = us, to owner = 0, to drop;
- we want the second period to be shorter than the first */
-
-static int drop_resources(struct lockspace *ls)
-{
- struct resource *r;
- struct timeval now;
- int count = 0;
-
- if (!cfgd_plock_ownership)
- return 0;
-
- if (list_empty(&ls->plock_resources))
- return 0;
-
- gettimeofday(&now, NULL);
-
- if (time_diff_ms(&ls->drop_resources_last, &now) <
- cfgd_drop_resources_time)
- return 1;
-
- ls->drop_resources_last = now;
-
- /* try to drop the oldest, unused resources */
-
- list_for_each_entry_reverse(r, &ls->plock_resources, list) {
- if (count >= cfgd_drop_resources_count)
- break;
- if (r->owner && r->owner != our_nodeid)
- continue;
- if (time_diff_ms(&r->last_access, &now) <
- cfgd_drop_resources_age)
- continue;
-
- if (list_empty(&r->locks) && list_empty(&r->waiters)) {
- if (r->owner == our_nodeid) {
- send_own(ls, r, 0);
- r->owner = 0;
- } else if (r->owner == 0 && got_unown(r)) {
- send_drop(ls, r);
- }
-
- count++;
- }
- }
-
- return 1;
-}
-
-void drop_resources_all(void)
-{
- struct lockspace *ls;
- int rv = 0;
-
- poll_drop_plock = 0;
-
- list_for_each_entry(ls, &lockspaces, list) {
- rv = drop_resources(ls);
- if (rv)
- poll_drop_plock = 1;
- }
-}
-
-int limit_plocks(void)
-{
- struct timeval now;
-
- /* Don't send more messages while the cpg message queue is backed up */
-
- if (message_flow_control_on) {
- update_flow_control_status();
- if (message_flow_control_on)
- return 1;
- }
-
- if (!cfgd_plock_rate_limit || !plock_read_count)
- return 0;
-
- gettimeofday(&now, NULL);
-
- /* Every time a plock op is read from the kernel, we increment
- plock_read_count. After every cfgd_plock_rate_limit (N) reads,
- we check the time it's taken to do those N; if the time is less than
- a second, then we delay reading any more until a second is up.
- This way we read a max of N ops from the kernel every second. */
-
- if (!(plock_read_count % cfgd_plock_rate_limit)) {
- if (time_diff_ms(&plock_rate_last, &now) < 1000) {
- plock_rate_delays++;
- return 2;
- }
- plock_rate_last = now;
- plock_read_count++;
- }
- return 0;
-}
-
-void process_plocks(int ci)
-{
- struct lockspace *ls;
- struct resource *r;
- struct dlm_plock_info info;
- struct timeval now;
- uint64_t usec;
- int create, rv;
-
- if (limit_plocks()) {
- poll_ignore_plock = 1;
- client_ignore(plock_ci, plock_fd);
- return;
- }
-
- gettimeofday(&now, NULL);
-
- memset(&info, 0, sizeof(info));
-
- rv = do_read(plock_device_fd, &info, sizeof(info));
- if (rv < 0) {
- log_debug("process_plocks: read error %d fd %d\n",
- errno, plock_device_fd);
- return;
- }
-
- /* kernel doesn't set the nodeid field */
- info.nodeid = our_nodeid;
-
- if (!cfgd_enable_plock) {
- rv = -ENOSYS;
- goto fail;
- }
-
- if (need_fsid_translation)
- info.fsid = mg_to_ls_id(info.fsid);
-
- ls = find_ls_id(info.fsid);
- if (!ls) {
- log_plock(ls, "process_plocks: no ls id %x", info.fsid);
- rv = -EEXIST;
- goto fail;
- }
-
- if (ls->disable_plock) {
- rv = -ENOSYS;
- goto fail;
- }
-
- log_plock(ls, "read plock %llx %s %s %llx-%llx %d/%u/%llx w %d",
- (unsigned long long)info.number,
- op_str(info.optype),
- ex_str(info.optype, info.ex),
- (unsigned long long)info.start, (unsigned long long)info.end,
- info.nodeid, info.pid, (unsigned long long)info.owner,
- info.wait);
-
- /* report plock rate and any delays since the last report */
- plock_read_count++;
- if (!(plock_read_count % 1000)) {
- usec = dt_usec(&plock_read_time, &now) ;
- log_plock(ls, "plock_read_count %u time %.3f s delays %u",
- plock_read_count, usec * 1.e-6, plock_rate_delays);
- plock_read_time = now;
- plock_rate_delays = 0;
- }
-
- create = (info.optype == DLM_PLOCK_OP_UNLOCK) ? 0 : 1;
-
- rv = find_resource(ls, info.number, create, &r);
- if (rv)
- goto fail;
-
- if (r->owner == 0) {
- /* plock state replicated on all nodes */
- send_plock(ls, r, &info);
-
- } else if (r->owner == our_nodeid) {
- /* we are the owner of r, so our plocks are local */
- __receive_plock(ls, &info, our_nodeid, r);
-
- } else {
- /* r owner is -1: r is new, try to become the owner;
- r owner > 0: tell other owner to give up ownership;
- both done with a message trying to set owner to ourself */
- send_own(ls, r, our_nodeid);
- save_pending_plock(ls, r, &info);
- }
-
- if (cfgd_plock_ownership && !list_empty(&ls->plock_resources))
- poll_drop_plock = 1;
- return;
-
- fail:
-#ifdef DLM_PLOCK_BUILD_WORKAROUND
- if (!(info.pad & DLM_PLOCK_FL_CLOSE)) {
-#else
- if (!(info.flags & DLM_PLOCK_FL_CLOSE)) {
-#endif
- info.rv = rv;
- rv = write(plock_device_fd, &info, sizeof(info));
- }
-}
-
-void process_saved_plocks(struct lockspace *ls)
-{
- struct save_msg *sm, *sm2;
- struct dlm_header *hd;
- int count = 0;
-
- log_dlock(ls, "process_saved_plocks begin");
-
- if (list_empty(&ls->saved_messages))
- goto out;
-
- list_for_each_entry_safe(sm, sm2, &ls->saved_messages, list) {
- hd = (struct dlm_header *)sm->buf;
-
- switch (sm->type) {
- case DLM_MSG_PLOCK:
- _receive_plock(ls, hd, sm->len);
- break;
- case DLM_MSG_PLOCK_OWN:
- _receive_own(ls, hd, sm->len);
- break;
- case DLM_MSG_PLOCK_DROP:
- _receive_drop(ls, hd, sm->len);
- break;
- case DLM_MSG_PLOCK_SYNC_LOCK:
- case DLM_MSG_PLOCK_SYNC_WAITER:
- _receive_sync(ls, hd, sm->len);
- break;
- default:
- continue;
- }
-
- list_del(&sm->list);
- free(sm);
- count++;
- }
- out:
- log_dlock(ls, "process_saved_plocks %d done", count);
-}
-
-/* locks still marked SYNCING should not go into the ckpt; the new node
- will get those locks by receiving PLOCK_SYNC messages */
-
-#define MAX_SEND_SIZE 1024 /* 1024 holds 24 plock_data */
-
-static char send_buf[MAX_SEND_SIZE];
-
-static int pack_send_buf(struct lockspace *ls, struct resource *r, int owner,
- int full, int *count_out, void **last)
-{
- struct resource_data *rd;
- struct plock_data *pp;
- struct posix_lock *po;
- struct lock_waiter *w;
- int count = 0;
- int find = 0;
- int len;
-
- /* N.B. owner not always equal to r->owner */
- rd = (struct resource_data *)(send_buf + sizeof(struct dlm_header));
- rd->number = cpu_to_le64(r->number);
- rd->owner = cpu_to_le32(owner);
-
- if (full) {
- rd->flags = RD_CONTINUE;
- find = 1;
- }
-
- /* plocks not replicated for owned resources */
- if (cfgd_plock_ownership && (owner == our_nodeid))
- goto done;
-
- len = sizeof(struct dlm_header) + sizeof(struct resource_data);
-
- pp = (struct plock_data *)(send_buf + sizeof(struct dlm_header) + sizeof(struct
resource_data));
-
- list_for_each_entry(po, &r->locks, list) {
- if (find && *last != po)
- continue;
- find = 0;
-
- if (po->flags & P_SYNCING)
- continue;
-
- if (len + sizeof(struct plock_data) > sizeof(send_buf)) {
- *last = po;
- goto full;
- }
- len += sizeof(struct plock_data);
-
- pp->start = cpu_to_le64(po->start);
- pp->end = cpu_to_le64(po->end);
- pp->owner = cpu_to_le64(po->owner);
- pp->pid = cpu_to_le32(po->pid);
- pp->nodeid = cpu_to_le32(po->nodeid);
- pp->ex = po->ex;
- pp->waiter = 0;
- pp++;
- count++;
- }
-
- list_for_each_entry(w, &r->waiters, list) {
- if (find && *last != w)
- continue;
- find = 0;
-
- if (w->flags & P_SYNCING)
- continue;
-
- if (len + sizeof(struct plock_data) > sizeof(send_buf)) {
- *last = w;
- goto full;
- }
- len += sizeof(struct plock_data);
-
- pp->start = cpu_to_le64(w->info.start);
- pp->end = cpu_to_le64(w->info.end);
- pp->owner = cpu_to_le64(w->info.owner);
- pp->pid = cpu_to_le32(w->info.pid);
- pp->nodeid = cpu_to_le32(w->info.nodeid);
- pp->ex = w->info.ex;
- pp->waiter = 1;
- pp++;
- count++;
- }
- done:
- rd->lock_count = cpu_to_le32(count);
- *count_out = count;
- *last = NULL;
- return 0;
-
- full:
- rd->lock_count = cpu_to_le32(count);
- *count_out = count;
- return 1;
-}
-
-/* Copy all plock state into a checkpoint so new node can retrieve it. The
- node creating the ckpt for the mounter needs to be the same node that's
- sending the mounter its journals message (i.e. the low nodeid). The new
- mounter knows the ckpt is ready to read only after it gets its journals
- message.
-
- If the mounter is becoming the new low nodeid in the group, the node doing
- the store closes the ckpt and the new node unlinks the ckpt after reading
- it. The ckpt should then disappear and the new node can create a new ckpt
- for the next mounter. */
-
-static int send_plocks_data(struct lockspace *ls, uint32_t seq, char *buf, int len)
-{
- struct dlm_header *hd;
-
- hd = (struct dlm_header *)buf;
- hd->type = DLM_MSG_PLOCKS_DATA;
- hd->msgdata = seq;
-
- dlm_send_message(ls, buf, len);
-
- return 0;
-}
-
-void send_all_plocks_data(struct lockspace *ls, uint32_t seq, uint32_t *plocks_data)
-{
- struct resource *r;
- void *last;
- int owner, count, len, full;
- uint32_t send_count = 0;
-
- if (!cfgd_enable_plock || ls->disable_plock)
- return;
-
- log_dlock(ls, "send_all_plocks_data %d:%u", our_nodeid, seq);
-
- /* - If r owner is -1, ckpt nothing.
- - If r owner is us, ckpt owner of us and no plocks.
- - If r owner is other, ckpt that owner and any plocks we have on r
- (they've just been synced but owner=0 msg not recved yet).
- - If r owner is 0 and !got_unown, then we've just unowned r;
- ckpt owner of us and any plocks that don't have SYNCING set
- (plocks with SYNCING will be handled by our sync messages).
- - If r owner is 0 and got_unown, then ckpt owner 0 and all plocks;
- (there should be no SYNCING plocks) */
-
- list_for_each_entry(r, &ls->plock_resources, list) {
- if (!cfgd_plock_ownership)
- owner = 0;
- else if (r->owner == -1)
- continue;
- else if (r->owner == our_nodeid)
- owner = our_nodeid;
- else if (r->owner)
- owner = r->owner;
- else if (!r->owner && !got_unown(r))
- owner = our_nodeid;
- else if (!r->owner)
- owner = 0;
- else {
- log_elock(ls, "send_all_plocks_data error owner %d r %llx",
- r->owner, (unsigned long long)r->number);
- continue;
- }
-
- memset(&send_buf, 0, sizeof(send_buf));
- count = 0;
- full = 0;
- last = NULL;
-
- do {
- full = pack_send_buf(ls, r, owner, full, &count, &last);
-
- len = sizeof(struct dlm_header) +
- sizeof(struct resource_data) +
- sizeof(struct plock_data) * count;
-
- log_plock(ls, "send_plocks_data %d:%u n %llu o %d locks %d len %d",
- our_nodeid, seq, (unsigned long long)r->number, r->owner,
- count, len);
-
- send_plocks_data(ls, seq, send_buf, len);
-
- send_count++;
-
- } while (full);
- }
-
- *plocks_data = send_count;
-
- log_dlock(ls, "send_all_plocks_data %d:%u %u done",
- our_nodeid, seq, send_count);
-}
-
-static void free_r_lists(struct resource *r)
-{
- struct posix_lock *po, *po2;
- struct lock_waiter *w, *w2;
-
- list_for_each_entry_safe(po, po2, &r->locks, list) {
- list_del(&po->list);
- free(po);
- }
-
- list_for_each_entry_safe(w, w2, &r->waiters, list) {
- list_del(&w->list);
- free(w);
- }
-}
-
-void receive_plocks_data(struct lockspace *ls, struct dlm_header *hd, int len)
-{
- struct resource_data *rd;
- struct plock_data *pp;
- struct posix_lock *po;
- struct lock_waiter *w;
- struct resource *r;
- uint64_t num;
- uint32_t count;
- uint32_t flags;
- int owner;
- int i;
-
- if (!cfgd_enable_plock || ls->disable_plock)
- return;
-
- if (!ls->need_plocks)
- return;
-
- if (!ls->save_plocks)
- return;
-
- ls->recv_plocks_data_count++;
-
- if (len < sizeof(struct dlm_header) + sizeof(struct resource_data)) {
- log_elock(ls, "recv_plocks_data %d:%u bad len %d",
- hd->nodeid, hd->msgdata, len);
- return;
- }
-
- rd = (struct resource_data *)((char *)hd + sizeof(struct dlm_header));
- num = le64_to_cpu(rd->number);
- owner = le32_to_cpu(rd->owner);
- count = le32_to_cpu(rd->lock_count);
- flags = le32_to_cpu(rd->flags);
-
- if (flags & RD_CONTINUE) {
- r = search_resource(ls, num);
- if (!r) {
- log_elock(ls, "recv_plocks_data %d:%u n %llu not found",
- hd->nodeid, hd->msgdata, (unsigned long long)num);
- return;
- }
- log_plock(ls, "recv_plocks_data %d:%u n %llu continue",
- hd->nodeid, hd->msgdata, (unsigned long long)num);
- goto unpack;
- }
-
- r = malloc(sizeof(struct resource));
- if (!r) {
- log_elock(ls, "recv_plocks_data %d:%u n %llu no mem",
- hd->nodeid, hd->msgdata, (unsigned long long)num);
- return;
- }
- memset(r, 0, sizeof(struct resource));
- INIT_LIST_HEAD(&r->locks);
- INIT_LIST_HEAD(&r->waiters);
- INIT_LIST_HEAD(&r->pending);
-
- if (!cfgd_plock_ownership) {
- if (owner) {
- log_elock(ls, "recv_plocks_data %d:%u n %llu bad owner %d",
- hd->nodeid, hd->msgdata, (unsigned long long)num,
- owner);
- goto fail_free;
- }
- } else {
- if (!owner)
- r->flags |= R_GOT_UNOWN;
-
- /* no locks should be included for owned resources */
-
- if (owner && count) {
- log_elock(ls, "recv_plocks_data %d:%u n %llu o %d bad count %u",
- (unsigned long long)num, owner, count);
- goto fail_free;
- }
- }
-
- r->number = num;
- r->owner = owner;
-
- unpack:
- if (len < sizeof(struct dlm_header) +
- sizeof(struct resource_data) +
- sizeof(struct plock_data) * count) {
- log_elock(ls, "recv_plocks_data %d:%u count %u bad len %d",
- hd->nodeid, hd->msgdata, count, len);
- goto fail_free;
- }
-
- pp = (struct plock_data *)((char *)rd + sizeof(struct resource_data));
-
- for (i = 0; i < count; i++) {
- if (!pp->waiter) {
- po = malloc(sizeof(struct posix_lock));
- if (!po)
- goto fail_free;
- po->start = le64_to_cpu(pp->start);
- po->end = le64_to_cpu(pp->end);
- po->owner = le64_to_cpu(pp->owner);
- po->pid = le32_to_cpu(pp->pid);
- po->nodeid = le32_to_cpu(pp->nodeid);
- po->ex = pp->ex;
- list_add_tail(&po->list, &r->locks);
- } else {
- w = malloc(sizeof(struct lock_waiter));
- if (!w)
- goto fail_free;
- w->info.start = le64_to_cpu(pp->start);
- w->info.end = le64_to_cpu(pp->end);
- w->info.owner = le64_to_cpu(pp->owner);
- w->info.pid = le32_to_cpu(pp->pid);
- w->info.nodeid = le32_to_cpu(pp->nodeid);
- w->info.ex = pp->ex;
- list_add_tail(&w->list, &r->waiters);
- }
- pp++;
- }
-
- log_plock(ls, "recv_plocks_data %d:%u n %llu o %d locks %d len %d",
- hd->nodeid, hd->msgdata, (unsigned long long)r->number,
- r->owner, count, len);
-
- if (!(flags & RD_CONTINUE)) {
- list_add_tail(&r->list, &ls->plock_resources);
- rb_insert_plock_resource(ls, r);
- }
- return;
-
- fail_free:
- if (!(flags & RD_CONTINUE)) {
- free_r_lists(r);
- free(r);
- }
- return;
-}
-
-void clear_plocks_data(struct lockspace *ls)
-{
- struct resource *r, *r2;
- uint32_t count = 0;
-
- if (!cfgd_enable_plock || ls->disable_plock)
- return;
-
- list_for_each_entry_safe(r, r2, &ls->plock_resources, list) {
- free_r_lists(r);
- rb_del_plock_resource(ls, r);
- list_del(&r->list);
- free(r);
- count++;
- }
-
- log_dlock(ls, "clear_plocks_data done %u recv_plocks_data_count %u",
- count, ls->recv_plocks_data_count);
-
- ls->recv_plocks_data_count = 0;
-}
-
-/* Called when a node has failed, or we're unmounting. For a node failure, we
- need to call this when the cpg confchg arrives so that we're guaranteed all
- nodes do this in the same sequence wrt other messages. */
-
-void purge_plocks(struct lockspace *ls, int nodeid, int unmount)
-{
- struct posix_lock *po, *po2;
- struct lock_waiter *w, *w2;
- struct resource *r, *r2;
- int purged = 0;
-
- if (!cfgd_enable_plock || ls->disable_plock)
- return;
-
- list_for_each_entry_safe(r, r2, &ls->plock_resources, list) {
- list_for_each_entry_safe(po, po2, &r->locks, list) {
- if (po->nodeid == nodeid || unmount) {
- list_del(&po->list);
- free(po);
- purged++;
- }
- }
-
- list_for_each_entry_safe(w, w2, &r->waiters, list) {
- if (w->info.nodeid == nodeid || unmount) {
- list_del(&w->list);
- free(w);
- purged++;
- }
- }
-
- /* TODO: haven't thought carefully about how this transition
- to owner 0 might interact with other owner messages in
- progress. */
-
- if (r->owner == nodeid) {
- r->owner = 0;
- r->flags |= R_GOT_UNOWN;
- r->flags |= R_PURGE_UNOWN;
- send_pending_plocks(ls, r);
- }
-
- if (!list_empty(&r->waiters))
- do_waiters(ls, r);
-
- if (!cfgd_plock_ownership &&
- list_empty(&r->locks) && list_empty(&r->waiters)) {
- rb_del_plock_resource(ls, r);
- list_del(&r->list);
- free(r);
- }
- }
-
- if (purged)
- ls->last_plock_time = time(NULL);
-
- log_dlock(ls, "purged %d plocks for %d", purged, nodeid);
-}
-
-int copy_plock_state(struct lockspace *ls, char *buf, int *len_out)
-{
- struct posix_lock *po;
- struct lock_waiter *w;
- struct resource *r;
- struct timeval now;
- int rv = 0;
- int len = DLMC_DUMP_SIZE, pos = 0, ret;
-
- gettimeofday(&now, NULL);
-
- list_for_each_entry(r, &ls->plock_resources, list) {
-
- if (list_empty(&r->locks) &&
- list_empty(&r->waiters) &&
- list_empty(&r->pending)) {
- ret = snprintf(buf + pos, len - pos,
- "%llu rown %d unused_ms %llu\n",
- (unsigned long long)r->number, r->owner,
- (unsigned long long)time_diff_ms(&r->last_access,
- &now));
- if (ret >= len - pos) {
- rv = -ENOSPC;
- goto out;
- }
- pos += ret;
- continue;
- }
-
- list_for_each_entry(po, &r->locks, list) {
- ret = snprintf(buf + pos, len - pos,
- "%llu %s %llu-%llu nodeid %d pid %u owner %llx rown %d\n",
- (unsigned long long)r->number,
- po->ex ? "WR" : "RD",
- (unsigned long long)po->start,
- (unsigned long long)po->end,
- po->nodeid, po->pid,
- (unsigned long long)po->owner, r->owner);
-
- if (ret >= len - pos) {
- rv = -ENOSPC;
- goto out;
- }
- pos += ret;
- }
-
- list_for_each_entry(w, &r->waiters, list) {
- ret = snprintf(buf + pos, len - pos,
- "%llu %s %llu-%llu nodeid %d pid %u owner %llx rown %d WAITING\n",
- (unsigned long long)r->number,
- w->info.ex ? "WR" : "RD",
- (unsigned long long)w->info.start,
- (unsigned long long)w->info.end,
- w->info.nodeid, w->info.pid,
- (unsigned long long)w->info.owner, r->owner);
-
- if (ret >= len - pos) {
- rv = -ENOSPC;
- goto out;
- }
- pos += ret;
- }
-
- list_for_each_entry(w, &r->pending, list) {
- ret = snprintf(buf + pos, len - pos,
- "%llu %s %llu-%llu nodeid %d pid %u owner %llx rown %d PENDING\n",
- (unsigned long long)r->number,
- w->info.ex ? "WR" : "RD",
- (unsigned long long)w->info.start,
- (unsigned long long)w->info.end,
- w->info.nodeid, w->info.pid,
- (unsigned long long)w->info.owner, r->owner);
-
- if (ret >= len - pos) {
- rv = -ENOSPC;
- goto out;
- }
- pos += ret;
- }
- }
- out:
- *len_out = pos;
- return rv;
-}
-
diff --git a/group/dlm_controld/rbtree.c b/group/dlm_controld/rbtree.c
deleted file mode 100644
index 430ccc1..0000000
--- a/group/dlm_controld/rbtree.c
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- Red Black Trees
- (C) 1999 Andrea Arcangeli <andrea(a)suse.de>
- (C) 2002 David Woodhouse <dwmw2(a)infradead.org>
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
- linux/lib/rbtree.c
-*/
-
-#include "rbtree.h"
-
-static void __rb_rotate_left(struct rb_node *node, struct rb_root *root)
-{
- struct rb_node *right = node->rb_right;
- struct rb_node *parent = rb_parent(node);
-
- if ((node->rb_right = right->rb_left))
- rb_set_parent(right->rb_left, node);
- right->rb_left = node;
-
- rb_set_parent(right, parent);
-
- if (parent)
- {
- if (node == parent->rb_left)
- parent->rb_left = right;
- else
- parent->rb_right = right;
- }
- else
- root->rb_node = right;
- rb_set_parent(node, right);
-}
-
-static void __rb_rotate_right(struct rb_node *node, struct rb_root *root)
-{
- struct rb_node *left = node->rb_left;
- struct rb_node *parent = rb_parent(node);
-
- if ((node->rb_left = left->rb_right))
- rb_set_parent(left->rb_right, node);
- left->rb_right = node;
-
- rb_set_parent(left, parent);
-
- if (parent)
- {
- if (node == parent->rb_right)
- parent->rb_right = left;
- else
- parent->rb_left = left;
- }
- else
- root->rb_node = left;
- rb_set_parent(node, left);
-}
-
-void rb_insert_color(struct rb_node *node, struct rb_root *root)
-{
- struct rb_node *parent, *gparent;
-
- while ((parent = rb_parent(node)) && rb_is_red(parent))
- {
- gparent = rb_parent(parent);
-
- if (parent == gparent->rb_left)
- {
- {
- register struct rb_node *uncle = gparent->rb_right;
- if (uncle && rb_is_red(uncle))
- {
- rb_set_black(uncle);
- rb_set_black(parent);
- rb_set_red(gparent);
- node = gparent;
- continue;
- }
- }
-
- if (parent->rb_right == node)
- {
- register struct rb_node *tmp;
- __rb_rotate_left(parent, root);
- tmp = parent;
- parent = node;
- node = tmp;
- }
-
- rb_set_black(parent);
- rb_set_red(gparent);
- __rb_rotate_right(gparent, root);
- } else {
- {
- register struct rb_node *uncle = gparent->rb_left;
- if (uncle && rb_is_red(uncle))
- {
- rb_set_black(uncle);
- rb_set_black(parent);
- rb_set_red(gparent);
- node = gparent;
- continue;
- }
- }
-
- if (parent->rb_left == node)
- {
- register struct rb_node *tmp;
- __rb_rotate_right(parent, root);
- tmp = parent;
- parent = node;
- node = tmp;
- }
-
- rb_set_black(parent);
- rb_set_red(gparent);
- __rb_rotate_left(gparent, root);
- }
- }
-
- rb_set_black(root->rb_node);
-}
-
-static void __rb_erase_color(struct rb_node *node, struct rb_node *parent,
- struct rb_root *root)
-{
- struct rb_node *other;
-
- while ((!node || rb_is_black(node)) && node != root->rb_node)
- {
- if (parent->rb_left == node)
- {
- other = parent->rb_right;
- if (rb_is_red(other))
- {
- rb_set_black(other);
- rb_set_red(parent);
- __rb_rotate_left(parent, root);
- other = parent->rb_right;
- }
- if ((!other->rb_left || rb_is_black(other->rb_left)) &&
- (!other->rb_right || rb_is_black(other->rb_right)))
- {
- rb_set_red(other);
- node = parent;
- parent = rb_parent(node);
- }
- else
- {
- if (!other->rb_right || rb_is_black(other->rb_right))
- {
- rb_set_black(other->rb_left);
- rb_set_red(other);
- __rb_rotate_right(other, root);
- other = parent->rb_right;
- }
- rb_set_color(other, rb_color(parent));
- rb_set_black(parent);
- rb_set_black(other->rb_right);
- __rb_rotate_left(parent, root);
- node = root->rb_node;
- break;
- }
- }
- else
- {
- other = parent->rb_left;
- if (rb_is_red(other))
- {
- rb_set_black(other);
- rb_set_red(parent);
- __rb_rotate_right(parent, root);
- other = parent->rb_left;
- }
- if ((!other->rb_left || rb_is_black(other->rb_left)) &&
- (!other->rb_right || rb_is_black(other->rb_right)))
- {
- rb_set_red(other);
- node = parent;
- parent = rb_parent(node);
- }
- else
- {
- if (!other->rb_left || rb_is_black(other->rb_left))
- {
- rb_set_black(other->rb_right);
- rb_set_red(other);
- __rb_rotate_left(other, root);
- other = parent->rb_left;
- }
- rb_set_color(other, rb_color(parent));
- rb_set_black(parent);
- rb_set_black(other->rb_left);
- __rb_rotate_right(parent, root);
- node = root->rb_node;
- break;
- }
- }
- }
- if (node)
- rb_set_black(node);
-}
-
-void rb_erase(struct rb_node *node, struct rb_root *root)
-{
- struct rb_node *child, *parent;
- int color;
-
- if (!node->rb_left)
- child = node->rb_right;
- else if (!node->rb_right)
- child = node->rb_left;
- else
- {
- struct rb_node *old = node, *left;
-
- node = node->rb_right;
- while ((left = node->rb_left) != NULL)
- node = left;
-
- if (rb_parent(old)) {
- if (rb_parent(old)->rb_left == old)
- rb_parent(old)->rb_left = node;
- else
- rb_parent(old)->rb_right = node;
- } else
- root->rb_node = node;
-
- child = node->rb_right;
- parent = rb_parent(node);
- color = rb_color(node);
-
- if (parent == old) {
- parent = node;
- } else {
- if (child)
- rb_set_parent(child, parent);
- parent->rb_left = child;
-
- node->rb_right = old->rb_right;
- rb_set_parent(old->rb_right, node);
- }
-
- node->rb_parent_color = old->rb_parent_color;
- node->rb_left = old->rb_left;
- rb_set_parent(old->rb_left, node);
-
- goto color;
- }
-
- parent = rb_parent(node);
- color = rb_color(node);
-
- if (child)
- rb_set_parent(child, parent);
- if (parent)
- {
- if (parent->rb_left == node)
- parent->rb_left = child;
- else
- parent->rb_right = child;
- }
- else
- root->rb_node = child;
-
- color:
- if (color == RB_BLACK)
- __rb_erase_color(child, parent, root);
-}
-
-/*
- * This function returns the first node (in sort order) of the tree.
- */
-struct rb_node *rb_first(const struct rb_root *root)
-{
- struct rb_node *n;
-
- n = root->rb_node;
- if (!n)
- return NULL;
- while (n->rb_left)
- n = n->rb_left;
- return n;
-}
-
-struct rb_node *rb_last(const struct rb_root *root)
-{
- struct rb_node *n;
-
- n = root->rb_node;
- if (!n)
- return NULL;
- while (n->rb_right)
- n = n->rb_right;
- return n;
-}
-
-struct rb_node *rb_next(const struct rb_node *node)
-{
- struct rb_node *parent;
-
- if (rb_parent(node) == node)
- return NULL;
-
- /* If we have a right-hand child, go down and then left as far
- as we can. */
- if (node->rb_right) {
- node = node->rb_right;
- while (node->rb_left)
- node=node->rb_left;
- return (struct rb_node *)node;
- }
-
- /* No right-hand children. Everything down and left is
- smaller than us, so any 'next' node must be in the general
- direction of our parent. Go up the tree; any time the
- ancestor is a right-hand child of its parent, keep going
- up. First time it's a left-hand child of its parent, said
- parent is our 'next' node. */
- while ((parent = rb_parent(node)) && node == parent->rb_right)
- node = parent;
-
- return parent;
-}
-
-struct rb_node *rb_prev(const struct rb_node *node)
-{
- struct rb_node *parent;
-
- if (rb_parent(node) == node)
- return NULL;
-
- /* If we have a left-hand child, go down and then right as far
- as we can. */
- if (node->rb_left) {
- node = node->rb_left;
- while (node->rb_right)
- node=node->rb_right;
- return (struct rb_node *)node;
- }
-
- /* No left-hand children. Go up till we find an ancestor which
- is a right-hand child of its parent */
- while ((parent = rb_parent(node)) && node == parent->rb_left)
- node = parent;
-
- return parent;
-}
-
-void rb_replace_node(struct rb_node *victim, struct rb_node *new,
- struct rb_root *root)
-{
- struct rb_node *parent = rb_parent(victim);
-
- /* Set the surrounding nodes to point to the replacement */
- if (parent) {
- if (victim == parent->rb_left)
- parent->rb_left = new;
- else
- parent->rb_right = new;
- } else {
- root->rb_node = new;
- }
- if (victim->rb_left)
- rb_set_parent(victim->rb_left, new);
- if (victim->rb_right)
- rb_set_parent(victim->rb_right, new);
-
- /* Copy the pointers/colour from the victim to the replacement */
- *new = *victim;
-}
diff --git a/group/include/linux_endian.h b/group/include/linux_endian.h
deleted file mode 100644
index 43089d2..0000000
--- a/group/include/linux_endian.h
+++ /dev/null
@@ -1,68 +0,0 @@
-#ifndef __LINUX_ENDIAN_DOT_H__
-#define __LINUX_ENDIAN_DOT_H__
-
-
-#include <endian.h>
-#include <byteswap.h>
-
-
-/* I'm not sure which versions of alpha glibc/gcc are broken,
- so fix all of them. */
-#ifdef __alpha__
-#undef bswap_64
-static __inline__ unsigned long bswap_64(unsigned long x)
-{
- unsigned int h = x >> 32;
- unsigned int l = x;
-
- h = bswap_32(h);
- l = bswap_32(l);
-
- return ((unsigned long)l << 32) | h;
-}
-#endif /* __alpha__ */
-
-
-#if __BYTE_ORDER == __BIG_ENDIAN
-
-#define be16_to_cpu(x) (x)
-#define be32_to_cpu(x) (x)
-#define be64_to_cpu(x) (x)
-
-#define cpu_to_be16(x) (x)
-#define cpu_to_be32(x) (x)
-#define cpu_to_be64(x) (x)
-
-#define le16_to_cpu(x) (bswap_16((x)))
-#define le32_to_cpu(x) (bswap_32((x)))
-#define le64_to_cpu(x) (bswap_64((x)))
-
-#define cpu_to_le16(x) (bswap_16((x)))
-#define cpu_to_le32(x) (bswap_32((x)))
-#define cpu_to_le64(x) (bswap_64((x)))
-
-#endif /* __BYTE_ORDER == __BIG_ENDIAN */
-
-
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-
-#define be16_to_cpu(x) (bswap_16((x)))
-#define be32_to_cpu(x) (bswap_32((x)))
-#define be64_to_cpu(x) (bswap_64((x)))
-
-#define cpu_to_be16(x) (bswap_16((x)))
-#define cpu_to_be32(x) (bswap_32((x)))
-#define cpu_to_be64(x) (bswap_64((x)))
-
-#define le16_to_cpu(x) (x)
-#define le32_to_cpu(x) (x)
-#define le64_to_cpu(x) (x)
-
-#define cpu_to_le16(x) (x)
-#define cpu_to_le32(x) (x)
-#define cpu_to_le64(x) (x)
-
-#endif /* __BYTE_ORDER == __LITTLE_ENDIAN */
-
-
-#endif /* __LINUX_ENDIAN_DOT_H__ */
diff --git a/group/include/list.h b/group/include/list.h
deleted file mode 100644
index 8100cbc..0000000
--- a/group/include/list.h
+++ /dev/null
@@ -1,336 +0,0 @@
-/* Copied from include/linux/list.h */
-
-#ifndef _LINUX_LIST_H
-#define _LINUX_LIST_H
-
-/**
- * container_of - cast a member of a structure out to the containing structure
- *
- * @ptr: the pointer to the member.
- * @type: the type of the container struct this is embedded in.
- * @member: the name of the member within the struct.
- *
- */
-#define container_of(ptr, type, member) ({ \
- const typeof( ((type *)0)->member ) *__mptr = (ptr); \
- (type *)( (char *)__mptr - offsetof(type,member) );})
-
-
-/*
- * These are non-NULL pointers that will result in page faults
- * under normal circumstances, used to verify that nobody uses
- * non-initialized list entries.
- */
-#define LIST_POISON1 ((void *) 0x00100100)
-#define LIST_POISON2 ((void *) 0x00200200)
-
-/*
- * Simple doubly linked list implementation.
- *
- * Some of the internal functions ("__xxx") are useful when
- * manipulating whole lists rather than single entries, as
- * sometimes we already know the next/prev entries and we can
- * generate better code by using them directly rather than
- * using the generic single-entry routines.
- */
-
-struct list_head {
- struct list_head *next, *prev;
-};
-
-#define LIST_HEAD_INIT(name) { &(name), &(name) }
-
-#define LIST_HEAD(name) \
- struct list_head name = LIST_HEAD_INIT(name)
-
-#define INIT_LIST_HEAD(ptr) do { \
- (ptr)->next = (ptr); (ptr)->prev = (ptr); \
-} while (0)
-
-/*
- * Insert a new entry between two known consecutive entries.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_add(struct list_head *new,
- struct list_head *prev,
- struct list_head *next)
-{
- next->prev = new;
- new->next = next;
- new->prev = prev;
- prev->next = new;
-}
-
-/**
- * list_add - add a new entry
- * @new: new entry to be added
- * @head: list head to add it after
- *
- * Insert a new entry after the specified head.
- * This is good for implementing stacks.
- */
-static inline void list_add(struct list_head *new, struct list_head *head)
-{
- __list_add(new, head, head->next);
-}
-
-/**
- * list_add_tail - add a new entry
- * @new: new entry to be added
- * @head: list head to add it before
- *
- * Insert a new entry before the specified head.
- * This is useful for implementing queues.
- */
-static inline void list_add_tail(struct list_head *new, struct list_head *head)
-{
- __list_add(new, head->prev, head);
-}
-
-/*
- * Delete a list entry by making the prev/next entries
- * point to each other.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_del(struct list_head * prev, struct list_head * next)
-{
- next->prev = prev;
- prev->next = next;
-}
-
-/**
- * list_del - deletes entry from list.
- * @entry: the element to delete from the list.
- * Note: list_empty on entry does not return true after this, the entry is
- * in an undefined state.
- */
-static inline void list_del(struct list_head *entry)
-{
- __list_del(entry->prev, entry->next);
- entry->next = LIST_POISON1;
- entry->prev = LIST_POISON2;
-}
-
-/**
- * list_del_init - deletes entry from list and reinitialize it.
- * @entry: the element to delete from the list.
- */
-static inline void list_del_init(struct list_head *entry)
-{
- __list_del(entry->prev, entry->next);
- INIT_LIST_HEAD(entry);
-}
-
-/**
- * list_move - delete from one list and add as another's head
- * @list: the entry to move
- * @head: the head that will precede our entry
- */
-static inline void list_move(struct list_head *list, struct list_head *head)
-{
- __list_del(list->prev, list->next);
- list_add(list, head);
-}
-
-/**
- * list_move_tail - delete from one list and add as another's tail
- * @list: the entry to move
- * @head: the head that will follow our entry
- */
-static inline void list_move_tail(struct list_head *list,
- struct list_head *head)
-{
- __list_del(list->prev, list->next);
- list_add_tail(list, head);
-}
-
-/**
- * list_empty - tests whether a list is empty
- * @head: the list to test.
- */
-static inline int list_empty(const struct list_head *head)
-{
- return head->next == head;
-}
-
-/**
- * list_empty_careful - tests whether a list is
- * empty _and_ checks that no other CPU might be
- * in the process of still modifying either member
- *
- * NOTE: using list_empty_careful() without synchronization
- * can only be safe if the only activity that can happen
- * to the list entry is list_del_init(). Eg. it cannot be used
- * if another CPU could re-list_add() it.
- *
- * @head: the list to test.
- */
-static inline int list_empty_careful(const struct list_head *head)
-{
- struct list_head *next = head->next;
- return (next == head) && (next == head->prev);
-}
-
-static inline void __list_splice(struct list_head *list,
- struct list_head *head)
-{
- struct list_head *first = list->next;
- struct list_head *last = list->prev;
- struct list_head *at = head->next;
-
- first->prev = head;
- head->next = first;
-
- last->next = at;
- at->prev = last;
-}
-
-/**
- * list_splice - join two lists
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice(struct list_head *list, struct list_head *head)
-{
- if (!list_empty(list))
- __list_splice(list, head);
-}
-
-/**
- * list_splice_init - join two lists and reinitialise the emptied list.
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- *
- * The list at @list is reinitialised
- */
-static inline void list_splice_init(struct list_head *list,
- struct list_head *head)
-{
- if (!list_empty(list)) {
- __list_splice(list, head);
- INIT_LIST_HEAD(list);
- }
-}
-
-/**
- * list_entry - get the struct for this entry
- * @ptr: the &struct list_head pointer.
- * @type: the type of the struct this is embedded in.
- * @member: the name of the list_struct within the struct.
- */
-#define list_entry(ptr, type, member) \
- container_of(ptr, type, member)
-
-/**
- * list_first_entry - get the first element from a list
- * @ptr: the list head to take the element from.
- * @type: the type of the struct this is embedded in.
- * @member: the name of the list_struct within the struct.
- *
- * Note, that list is expected to be not empty.
- */
-#define list_first_entry(ptr, type, member) \
- list_entry((ptr)->next, type, member)
-
-/**
- * list_for_each - iterate over a list
- * @pos: the &struct list_head to use as a loop counter.
- * @head: the head for your list.
- */
-#define list_for_each(pos, head) \
- for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/**
- * __list_for_each - iterate over a list
- * @pos: the &struct list_head to use as a loop counter.
- * @head: the head for your list.
- *
- * This variant differs from list_for_each() in that it's the
- * simplest possible list iteration code, no prefetching is done.
- * Use this for code that knows the list to be very short (empty
- * or 1 entry) most of the time.
- */
-#define __list_for_each(pos, head) \
- for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/**
- * list_for_each_prev - iterate over a list backwards
- * @pos: the &struct list_head to use as a loop counter.
- * @head: the head for your list.
- */
-#define list_for_each_prev(pos, head) \
- for (pos = (head)->prev; pos != (head); pos = pos->prev)
-
-/**
- * list_for_each_safe - iterate over a list safe against removal of list entry
- * @pos: the &struct list_head to use as a loop counter.
- * @n: another &struct list_head to use as temporary storage
- * @head: the head for your list.
- */
-#define list_for_each_safe(pos, n, head) \
- for (pos = (head)->next, n = pos->next; pos != (head); \
- pos = n, n = pos->next)
-
-/**
- * list_for_each_entry - iterate over list of given type
- * @pos: the type * to use as a loop counter.
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- */
-#define list_for_each_entry(pos, head, member) \
- for (pos = list_entry((head)->next, typeof(*pos), member); \
- &pos->member != (head); \
- pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_reverse - iterate backwards over list of given type.
- * @pos: the type * to use as a loop counter.
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- */
-#define list_for_each_entry_reverse(pos, head, member) \
- for (pos = list_entry((head)->prev, typeof(*pos), member); \
- &pos->member != (head); \
- pos = list_entry(pos->member.prev, typeof(*pos), member))
-
-/**
- * list_prepare_entry - prepare a pos entry for use as a start point in
- * list_for_each_entry_continue
- * @pos: the type * to use as a start point
- * @head: the head of the list
- * @member: the name of the list_struct within the struct.
- */
-#define list_prepare_entry(pos, head, member) \
- ((pos) ? : list_entry(head, typeof(*pos), member))
-
-/**
- * list_for_each_entry_continue - iterate over list of given type
- * continuing after existing point
- * @pos: the type * to use as a loop counter.
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- */
-#define list_for_each_entry_continue(pos, head, member) \
- for (pos = list_entry(pos->member.next, typeof(*pos), member); \
- &pos->member != (head); \
- pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_safe - iterate over list of given type safe against removal of
list entry
- * @pos: the type * to use as a loop counter.
- * @n: another type * to use as temporary storage
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe(pos, n, head, member) \
- for (pos = list_entry((head)->next, typeof(*pos), member), \
- n = list_entry(pos->member.next, typeof(*pos), member); \
- &pos->member != (head); \
- pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-
-#endif
diff --git a/group/include/rbtree.h b/group/include/rbtree.h
deleted file mode 100644
index e2b2409..0000000
--- a/group/include/rbtree.h
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- Red Black Trees
- (C) 1999 Andrea Arcangeli <andrea(a)suse.de>
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
- linux/include/linux/rbtree.h
-
- To use rbtrees you'll have to implement your own insert and search cores.
- This will avoid us to use callbacks and to drop drammatically performances.
- I know it's not the cleaner way, but in C (not in C++) to get
- performances and genericity...
-
- Some example of insert and search follows here. The search is a plain
- normal search over an ordered tree. The insert instead must be implemented
- int two steps: as first thing the code must insert the element in
- order as a red leaf in the tree, then the support library function
- rb_insert_color() must be called. Such function will do the
- not trivial work to rebalance the rbtree if necessary.
-
------------------------------------------------------------------------
-static inline struct page * rb_search_page_cache(struct inode * inode,
- unsigned long offset)
-{
- struct rb_node * n = inode->i_rb_page_cache.rb_node;
- struct page * page;
-
- while (n)
- {
- page = rb_entry(n, struct page, rb_page_cache);
-
- if (offset < page->offset)
- n = n->rb_left;
- else if (offset > page->offset)
- n = n->rb_right;
- else
- return page;
- }
- return NULL;
-}
-
-static inline struct page * __rb_insert_page_cache(struct inode * inode,
- unsigned long offset,
- struct rb_node * node)
-{
- struct rb_node ** p = &inode->i_rb_page_cache.rb_node;
- struct rb_node * parent = NULL;
- struct page * page;
-
- while (*p)
- {
- parent = *p;
- page = rb_entry(parent, struct page, rb_page_cache);
-
- if (offset < page->offset)
- p = &(*p)->rb_left;
- else if (offset > page->offset)
- p = &(*p)->rb_right;
- else
- return page;
- }
-
- rb_link_node(node, parent, p);
-
- return NULL;
-}
-
-static inline struct page * rb_insert_page_cache(struct inode * inode,
- unsigned long offset,
- struct rb_node * node)
-{
- struct page * ret;
- if ((ret = __rb_insert_page_cache(inode, offset, node)))
- goto out;
- rb_insert_color(node, &inode->i_rb_page_cache);
- out:
- return ret;
-}
------------------------------------------------------------------------
-*/
-
-#ifndef _LINUX_RBTREE_H
-#define _LINUX_RBTREE_H
-
-#include <linux/stddef.h>
-
-struct rb_node
-{
- unsigned long rb_parent_color;
-#define RB_RED 0
-#define RB_BLACK 1
- struct rb_node *rb_right;
- struct rb_node *rb_left;
-} __attribute__((aligned(sizeof(long))));
- /* The alignment might seem pointless, but allegedly CRIS needs it */
-
-struct rb_root
-{
- struct rb_node *rb_node;
-};
-
-
-#define rb_parent(r) ((struct rb_node *)((r)->rb_parent_color & ~3))
-#define rb_color(r) ((r)->rb_parent_color & 1)
-#define rb_is_red(r) (!rb_color(r))
-#define rb_is_black(r) rb_color(r)
-#define rb_set_red(r) do { (r)->rb_parent_color &= ~1; } while (0)
-#define rb_set_black(r) do { (r)->rb_parent_color |= 1; } while (0)
-
-static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p)
-{
- rb->rb_parent_color = (rb->rb_parent_color & 3) | (unsigned long)p;
-}
-static inline void rb_set_color(struct rb_node *rb, int color)
-{
- rb->rb_parent_color = (rb->rb_parent_color & ~1) | color;
-}
-
-#define RB_ROOT (struct rb_root) { NULL, }
-#define rb_entry(ptr, type, member) container_of(ptr, type, member)
-
-#define RB_EMPTY_ROOT(root) ((root)->rb_node == NULL)
-#define RB_EMPTY_NODE(node) (rb_parent(node) == node)
-#define RB_CLEAR_NODE(node) (rb_set_parent(node, node))
-
-extern void rb_insert_color(struct rb_node *, struct rb_root *);
-extern void rb_erase(struct rb_node *, struct rb_root *);
-
-/* Find logical next and previous nodes in a tree */
-extern struct rb_node *rb_next(const struct rb_node *);
-extern struct rb_node *rb_prev(const struct rb_node *);
-extern struct rb_node *rb_first(const struct rb_root *);
-extern struct rb_node *rb_last(const struct rb_root *);
-
-/* Fast replacement of a single node without remove/rebalance/add/rebalance */
-extern void rb_replace_node(struct rb_node *victim, struct rb_node *new,
- struct rb_root *root);
-
-static inline void rb_link_node(struct rb_node * node, struct rb_node * parent,
- struct rb_node ** rb_link)
-{
- node->rb_parent_color = (unsigned long )parent;
- node->rb_left = node->rb_right = NULL;
-
- *rb_link = node;
-}
-
-#endif /* _LINUX_RBTREE_H */
diff --git a/group/man/dlm_controld.8 b/group/man/dlm_controld.8
deleted file mode 100644
index 7100f0e..0000000
--- a/group/man/dlm_controld.8
+++ /dev/null
@@ -1,313 +0,0 @@
-.TH DLM_CONTROLD 8 2009-01-18 cluster cluster
-
-.SH NAME
-dlm_controld \- daemon that configures dlm according to cluster events
-
-.SH SYNOPSIS
-.B dlm_controld
-[OPTIONS]
-
-.SH DESCRIPTION
-The dlm lives in the kernel, and the cluster infrastructure (corosync
-membership and group management) lives in user space. The dlm in the
-kernel needs to adjust/recover for certain cluster events. It's the job
-of dlm_controld to receive these events and reconfigure the kernel dlm as
-needed. dlm_controld controls and configures the dlm through sysfs and
-configfs files that are considered dlm-internal interfaces.
-
-The cman init script usually starts the dlm_controld daemon.
-
-.SH OPTIONS
-Command line options override a corresponding setting in cluster.conf.
-
-.TP
-.B \-D
-Enable debugging to stderr and don't fork.
-.br
-See also
-.B dlm_tool dump
-in
-.BR dlm_tool (8).
-
-.TP
-.B \-L
-Enable debugging to log file.
-.br
-See also
-.B logging
-in
-.BR cluster.conf (5).
-
-.TP
-.B \-K
-Enable kernel dlm debugging messages.
-.br
-See also
-.B log_debug
-below.
-
-.TP
-.BI \-r " num"
-dlm kernel lowcomms protocol, 0 tcp, 1 sctp, 2 detect.
-2 selects tcp if corosync rrp_mode is "none", otherwise sctp.
-.br
-Default 2.
-
-.TP
-.BI \-g " num"
-groupd compatibility mode, 0 off, 1 on.
-.br
-Default 0.
-
-.TP
-.BI \-f " num"
-Enable (1) or disable (0) fencing recovery dependency.
-.br
-Default 1.
-
-.TP
-.BI \-q " num"
-Enable (1) or disable (0) quorum recovery dependency.
-.br
-Default 0.
-
-.TP
-.BI \-d " num"
-Enable (1) or disable (0) deadlock detection code.
-.br
-Default 0.
-
-.TP
-.BI \-p " num"
-Enable (1) or disable (0) plock code for cluster fs.
-.br
-Default 1.
-
-.TP
-.BI \-l " num"
-Limit the rate of plock operations, 0 for no limit.
-.br
-Default 0.
-
-.TP
-.BI \-o " num"
-Enable (1) or disable (0) plock ownership.
-.br
-Default 1.
-
-.TP
-.BI \-t " ms"
-Plock ownership drop resources time (milliseconds).
-.br
-Default 10000.
-
-.TP
-.BI \-c " num"
-Plock ownership drop resources count.
-.br
-Default 10.
-
-.TP
-.BI \-a " ms"
-Plock ownership drop resources age (milliseconds).
-.br
-Default 10000.
-
-.TP
-.B \-P
-Enable plock debugging messages (can produce excessive output).
-
-.TP
-.B \-h
-Print a help message describing available options, then exit.
-
-.TP
-.B \-V
-Print program version information, then exit.
-
-
-.SH FILES
-.BR cluster.conf (5)
-is usually located at /etc/cluster/cluster.conf. It is not read directly.
-Other cluster components load the contents into memory, and the values are
-accessed through the libccs library.
-
-Configuration options for dlm (kernel) and dlm_controld are added to the
-<dlm /> section of cluster.conf, within the top level <cluster> section.
-
-.SS Kernel options
-
-.TP
-.B protocol
-The network
-.B protocol
-can be set to tcp, sctp or detect which selects tcp or sctp based on
-the corosync rrp_mode configuration (redundant ring protocol).
-The rrp_mode "none" results in tcp. Default detect.
-
-<dlm protocol="detect"/>
-
-.TP
-.B timewarn
-After waiting
-.B timewarn
-centiseconds, the dlm will emit a warning via netlink. This only applies
-to lockspaces created with the DLM_LSFL_TIMEWARN flag, and is used for
-deadlock detection. Default 500 (5 seconds).
-
-<dlm timewarn="500"/>
-
-.TP
-.B log_debug
-DLM kernel debug messages can be enabled by setting
-.B log_debug
-to 1. Default 0.
-
-<dlm log_debug="0"/>
-
-.TP
-.B clusternode/weight
-The lock directory
-.B weight
-can be specified one the clusternode lines. Weights would usually be
-used in the lock server configurations shown below instead.
-
-<clusternode name="node01" nodeid="1" weight="1"/>
-
-.SS Daemon options
-
-.TP
-.B enable_fencing
-See command line description.
-
-<dlm enable_fencing="1"/>
-
-.TP
-.B enable_quorum
-See command line description.
-
-<dlm enable_quorum="0"/>
-
-.TP
-.B enable_deadlk
-See command line description.
-
-<dlm enable_deadlk="0"/>
-
-.TP
-.B enable_plock
-See command line description.
-
-<dlm enable_plock="1"/>
-
-.TP
-.B plock_rate_limit
-See command line description.
-
-<dlm plock_rate_limit="0"/>
-
-.TP
-.B plock_ownership
-See command line description.
-
-<dlm plock_ownership="1"/>
-
-.TP
-.B drop_resources_time
-See command line description.
-
-<dlm drop_resources_time="10000"/>
-
-.TP
-.B drop_resources_count
-See command line description.
-
-<dlm drop_resources_count="10"/>
-
-.TP
-.B drop_resources_age
-See command line description.
-
-<dlm drop_resources_age="10000"/>
-
-.TP
-.B plock_debug
-Enable (1) or disable (0) plock debugging messages (can produce excessive
-output). Default 0.
-
-<dlm plock_debug="0"/>
-
-
-.SS Disabling resource directory
-
-Lockspaces usually use a resource directory to keep track of which node is
-the master of each resource. The dlm can operate without the resource
-directory, though, by statically assigning the master of a resource using
-a hash of the resource name. To enable, set the per-lockspace
-.B nodir
-option to 1.
-
-.nf
-<dlm>
- <lockspace name="foo" nodir="1">
-</dlm>
-.fi
-
-.SS Lock-server configuration
-
-The nodir setting can be combined with node weights to create a
-configuration where select node(s) are the master of all resources/locks.
-These
-.B master
-nodes can be viewed as "lock servers" for the other nodes.
-
-.nf
-<dlm>
- <lockspace name="foo" nodir="1">
- <master name="node01"/>
- </lockspace>
-</dlm>
-
-or,
-
-<dlm>
- <lockspace name="foo" nodir="1">
- <master name="node01"/>
- <master name="node02"/>
- </lockspace>
-</dlm>
-.fi
-
-Lock management will be partitioned among the available masters. There
-can be any number of masters defined. The designated master nodes will
-master all resources/locks (according to the resource name hash). When no
-masters are members of the lockspace, then the nodes revert to the common
-fully-distributed configuration. Recovery is faster, with little
-disruption, when a non-master node joins/leaves.
-
-There is no special mode in the dlm for this lock server configuration,
-it's just a natural consequence of combining the "nodir" option with node
-weights. When a lockspace has master nodes defined, the master has a
-default weight of 1 and all non-master nodes have weight of 0. An explicit
-non-zero
-.B weight
-can also be assigned to master nodes, e.g.
-
-.nf
-<dlm>
- <lockspace name="foo" nodir="1">
- <master name="node01" weight="2"/>
- <master name="node02" weight="1"/>
- </lockspace>
-</dlm>
-.fi
-
-In which case node01 will master 2/3 of the total resources and node2 will
-master the other 1/3.
-
-.SH SEE ALSO
-.BR dlm_tool (8),
-.BR fenced (8),
-.BR cman (5),
-.BR cluster.conf (5)
-