dlm: master - dlm_controld: fencing and config
David Teigland
teigland at fedoraproject.org
Tue Mar 20 16:32:03 UTC 2012
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=d8d8a6de00f7d0f9ca1944913a26b805984851d5
Commit: d8d8a6de00f7d0f9ca1944913a26b805984851d5
Parent: eeb80cd73160856a16f73fc45d1ab48904aa55f0
Author: David Teigland <teigland at redhat.com>
AuthorDate: Mon Feb 27 13:10:25 2012 -0600
Committer: David Teigland <teigland at redhat.com>
CommitterDate: Tue Mar 20 11:24:17 2012 -0500
dlm_controld: fencing and config
New /etc/dlm/dlm.conf with same config options as
before, but using "key = val" format. Fencing
config can also be specified. Without an explicit
fencing config, default "dlm_stonith" agent is run
to let stonith handle fencing.
Signed-off-by: David Teigland <teigland at redhat.com>
---
dlm_controld/Makefile | 5 +-
dlm_controld/action.c | 4 +-
dlm_controld/config.c | 123 ++-
dlm_controld/cpg.c | 1082 +-----------------------
dlm_controld/daemon_cpg.c | 1921 ++++++++++++++++++++++++++++++++++++++++++
dlm_controld/dlm_controld.h | 1 +
dlm_controld/dlm_daemon.h | 85 ++-
dlm_controld/fence.c | 145 +++-
dlm_controld/fence_config.c | 403 +++++++++
dlm_controld/fence_config.h | 62 ++
dlm_controld/lib.c | 19 +
dlm_controld/libdlmcontrol.h | 1 +
dlm_controld/main.c | 90 ++-
dlm_controld/member.c | 51 +-
dlm_controld/plock.c | 10 -
dlm_tool/main.c | 17 +-
fence/Makefile | 56 ++
fence/stonith_helper.c | 96 +++
18 files changed, 3007 insertions(+), 1164 deletions(-)
diff --git a/dlm_controld/Makefile b/dlm_controld/Makefile
index a648ab1..b60b8df 100644
--- a/dlm_controld/Makefile
+++ b/dlm_controld/Makefile
@@ -19,7 +19,9 @@ LIB_TARGET = $(LIB_SO).$(LIB_MAJOR).$(LIB_MINOR)
BIN_SOURCE = action.c \
cpg.c \
+ daemon_cpg.c \
crc.c \
+ fence_config.c \
fence.c \
main.c \
plock.c \
@@ -51,13 +53,10 @@ BIN_CFLAGS += -D_GNU_SOURCE -g \
-fdiagnostics-show-option \
BIN_CFLAGS += -fPIE -DPIE
-BIN_CFLAGS += `xml2-config --cflags`
BIN_CFLAGS += -I../include -I../libdlm
BIN_LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
-BIN_LDFLAGS += `xml2-config --libs`
BIN_LDFLAGS += -lpthread -lrt -lcpg -lcmap -lcfg -lquorum
-#BIN_LDFLAGS += -lfenced
LIB_CFLAGS += $(BIN_CFLAGS)
LIB_LDFLAGS += -Wl,-z,relro -pie
diff --git a/dlm_controld/action.c b/dlm_controld/action.c
index d4bf11d..6cb34f2 100644
--- a/dlm_controld/action.c
+++ b/dlm_controld/action.c
@@ -34,10 +34,8 @@ static int detect_protocol(void)
}
rv = cmap_get_string(handle, "totem.rrp_mode", &str);
- if (rv != CS_OK) {
- log_error("cmap_get_string totem.rrp_mode error %d", rv);
+ if (rv != CS_OK)
goto out;
- }
log_debug("cmap totem.rrp_mode = '%s'", str);
diff --git a/dlm_controld/config.c b/dlm_controld/config.c
index 3bf02c0..f668b86 100644
--- a/dlm_controld/config.c
+++ b/dlm_controld/config.c
@@ -7,7 +7,6 @@
*/
#include "dlm_daemon.h"
-#include <libxml/tree.h>
/* TODO:
<dlm>
@@ -37,66 +36,100 @@ static void proto_val(char *str, int *val)
}
}
-static void set_val(xmlNode *root, const char *name, int *opt, int *val)
+static void set_val(char *line, int *val_out)
{
- xmlChar *str;
+ char key[PATH_MAX];
+ char val[PATH_MAX];
+ int rv;
- str = xmlGetProp(root, BAD_CAST name);
- if (str && !(*opt)) {
- *val = atoi((char *)str);
- log_debug("config %s = %d", name, *val);
- }
+ rv = sscanf(line, "%[^=]=%s", key, val);
+ if (rv != 2)
+ return;
+
+ *val_out = atoi(val);
+
+ log_debug("config %s=%d", key, *val_out);
+}
+
+static void get_val(char *line, char *val_out)
+{
+ char key[PATH_MAX];
+ char val[PATH_MAX];
+ int rv;
+
+ rv = sscanf(line, "%[^=]=%s", key, val);
+ if (rv != 2)
+ return;
+
+ strcpy(val_out, val);
}
void setup_config(int update)
{
- xmlDoc *doc;
- xmlNode *root;
- xmlChar *str;
+ FILE *file;
+ char line[PATH_MAX];
+ char str[PATH_MAX];
if (!path_exists(CONF_FILE_PATH))
return;
- doc = xmlParseFile(CONF_FILE_PATH);
- if (!doc) {
- log_error("xml parse error %d %s", errno, CONF_FILE_PATH);
+ file = fopen(CONF_FILE_PATH, "r");
+ if (!file)
return;
- }
- root = xmlDocGetRootElement(doc);
- if (!root) {
- log_error("xml root error %d %s", errno, CONF_FILE_PATH);
- xmlFreeDoc(doc);
- return;
- }
+ while (fgets(line, PATH_MAX, file)) {
+ if (line[0] == '#')
+ continue;
+ if (line[0] == '\n')
+ continue;
+
+ if (!optk_debug && !strncmp(line, "log_debug", strlen("log_debug")))
+ set_val(line, &cfgk_debug);
+
+ else if (!optk_timewarn && !strncmp(line, "timewarn", strlen("timewarn")) && !update)
+ set_val(line, &cfgk_timewarn);
+
+ else if (!optd_enable_fencing && !strncmp(line, "enable_fencing", strlen("enable_fencing")) && !update)
+ set_val(line, &cfgd_enable_fencing);
+
+ else if (!optd_enable_quorum_fencing && !strncmp(line, "enable_quorum_fencing", strlen("enable_quorum_fencing")) && !update)
+ set_val(line, &cfgd_enable_quorum_fencing);
+
+ else if (!optd_enable_quorum_lockspace && !strncmp(line, "enable_quorum_lockspace", strlen("enable_quorum_lockspace")) && !update)
+ set_val(line, &cfgd_enable_quorum_lockspace);
+
+ else if (!optd_enable_fscontrol && !strncmp(line, "enable_fscontrol", strlen("enable_fscontrol")) && !update)
+ set_val(line, &cfgd_enable_fscontrol);
+
+ else if (!optd_enable_plock && !strncmp(line, "enable_plock", strlen("enable_plock")) && !update)
+ set_val(line, &cfgd_enable_plock);
+
+ else if (!optd_plock_ownership && !strncmp(line, "plock_ownership", strlen("plock_ownership")) && !update)
+ set_val(line, &cfgd_plock_ownership);
+
+ else if (!optd_plock_debug && !strncmp(line, "plock_debug", strlen("plock_debug")))
+ set_val(line, &cfgd_plock_debug);
+
+ else if (!optd_plock_rate_limit && !strncmp(line, "plock_rate_limit", strlen("plock_rate_limit")))
+ set_val(line, &cfgd_plock_rate_limit);
+
+ else if (!optd_drop_resources_time && !strncmp(line, "drop_resources_time", strlen("drop_resources_time")))
+ set_val(line, &cfgd_drop_resources_time);
- if (update)
- goto do_update;
+ else if (!optd_drop_resources_count && !strncmp(line, "drop_resources_count", strlen("drop_resources_count")))
+ set_val(line, &cfgd_drop_resources_count);
- /* These config values are set from dlm.conf only if they haven't
- already been set on the command line. */
+ else if (!optd_drop_resources_age && !strncmp(line, "drop_resources_age", strlen("drop_resources_age")))
+ set_val(line, &cfgd_drop_resources_age);
- str = xmlGetProp(root, BAD_CAST "protocol");
- if (str && !optk_protocol) {
- proto_val((char *)str, &cfgk_protocol);
- log_debug("config protocol = %d", cfgk_protocol);
+ else if (!optk_protocol && !strncmp(line, "protocol", strlen("protocol")) && !update) {
+ memset(str, 0, sizeof(str));
+ get_val(line, str);
+ proto_val(str, &cfgk_protocol);
+ log_debug("config protocol = %d", cfgk_protocol);
+ }
}
- set_val(root, "log_debug", &optk_debug, &cfgk_debug);
- set_val(root, "timewarn", &optk_timewarn, &cfgk_timewarn);
- set_val(root, "enable_fencing", &optd_enable_fencing, &cfgd_enable_fencing);
- set_val(root, "enable_quorum", &optd_enable_quorum, &cfgd_enable_quorum);
- set_val(root, "enable_fscontrol", &optd_enable_fscontrol, &cfgd_enable_fscontrol);
- set_val(root, "enable_plock", &optd_enable_plock, &cfgd_enable_plock);
- set_val(root, "plock_ownership", &optd_plock_ownership, &cfgd_plock_ownership);
- do_update:
- /* The following can be changed while running */
- set_val(root, "plock_debug", &optd_plock_debug, &cfgd_plock_debug);
- set_val(root, "plock_rate_limit", &optd_plock_rate_limit, &cfgd_plock_rate_limit);
- set_val(root, "drop_resources_time", &optd_drop_resources_time, &cfgd_drop_resources_time);
- set_val(root, "drop_resources_count", &optd_drop_resources_count, &cfgd_drop_resources_count);
- set_val(root, "drop_resources_age", &optd_drop_resources_age, &cfgd_drop_resources_age);
-
- xmlFreeDoc(doc);
+ fclose(file);
}
diff --git a/dlm_controld/cpg.c b/dlm_controld/cpg.c
index 1360493..1fb5372 100644
--- a/dlm_controld/cpg.c
+++ b/dlm_controld/cpg.c
@@ -1,5 +1,5 @@
/*
- * Copyright 2004-2011 Red Hat, Inc.
+ * Copyright 2004-2012 Red Hat, Inc.
*
* This copyrighted material is made available to anyone wishing to use,
* modify, copy, or redistribute it subject to the terms and conditions
@@ -8,49 +8,6 @@
#include "dlm_daemon.h"
-#define PV_STATEFUL 0x0001
-
-struct protocol_version {
- uint16_t major;
- uint16_t minor;
- uint16_t patch;
- uint16_t flags;
-};
-
-struct protocol {
- union {
- struct protocol_version dm_ver;
- uint16_t daemon_max[4];
- };
- union {
- struct protocol_version km_ver;
- uint16_t kernel_max[4];
- };
- union {
- struct protocol_version dr_ver;
- uint16_t daemon_run[4];
- };
- union {
- struct protocol_version kr_ver;
- uint16_t kernel_run[4];
- };
-};
-
-/* per dlm_controld cpg: daemon_nodes */
-
-struct node_daemon {
- struct list_head list;
- int nodeid;
-
- uint64_t daemon_add_time;
- uint64_t daemon_rem_time;
- int daemon_member;
-
- int killed;
-
- struct protocol proto;
-};
-
/* per lockspace cpg: ls->node_history */
struct node {
@@ -71,11 +28,10 @@ struct node {
int check_fs;
int fs_notified;
- int request_fencing;
- int check_fencing;
- uint64_t fail_realtime;
- uint64_t fence_realtime;
+ int need_fencing;
uint32_t fence_queries; /* for debug */
+ uint64_t fail_walltime;
+ uint64_t fail_monotime;
};
/* per lockspace confchg: ls->changes */
@@ -127,93 +83,6 @@ struct id_info {
int nodeid;
};
-int message_flow_control_on;
-static cpg_handle_t cpg_handle_daemon;
-static int cpg_fd_daemon;
-static struct protocol our_protocol;
-static struct list_head daemon_nodes;
-static struct cpg_address daemon_member[MAX_NODES];
-static int daemon_member_count;
-
-static void log_config(const struct cpg_name *group_name,
- const struct cpg_address *member_list,
- size_t member_list_entries,
- const struct cpg_address *left_list,
- size_t left_list_entries,
- const struct cpg_address *joined_list,
- size_t joined_list_entries)
-{
- char m_buf[128];
- char j_buf[32];
- char l_buf[32];
- size_t i, len, pos;
- int ret;
-
- memset(m_buf, 0, sizeof(m_buf));
- memset(j_buf, 0, sizeof(j_buf));
- memset(l_buf, 0, sizeof(l_buf));
-
- len = sizeof(m_buf);
- pos = 0;
- for (i = 0; i < member_list_entries; i++) {
- ret = snprintf(m_buf + pos, len - pos, " %d",
- member_list[i].nodeid);
- if (ret >= len - pos)
- break;
- pos += ret;
- }
-
- len = sizeof(j_buf);
- pos = 0;
- for (i = 0; i < joined_list_entries; i++) {
- ret = snprintf(j_buf + pos, len - pos, " %d",
- joined_list[i].nodeid);
- if (ret >= len - pos)
- break;
- pos += ret;
- }
-
- len = sizeof(l_buf);
- pos = 0;
- for (i = 0; i < left_list_entries; i++) {
- ret = snprintf(l_buf + pos, len - pos, " %d",
- left_list[i].nodeid);
- if (ret >= len - pos)
- break;
- pos += ret;
- }
-
- log_debug("%s conf %zu %zu %zu memb%s join%s left%s", group_name->value,
- member_list_entries, joined_list_entries, left_list_entries,
- m_buf, j_buf, l_buf);
-}
-
-static void log_ringid(const char *name,
- struct cpg_ring_id *ringid,
- const uint32_t *member_list,
- size_t member_list_entries)
-{
- char m_buf[128];
- size_t i, len, pos;
- int ret;
-
- memset(m_buf, 0, sizeof(m_buf));
-
- len = sizeof(m_buf);
- pos = 0;
- for (i = 0; i < member_list_entries; i++) {
- ret = snprintf(m_buf + pos, len - pos, " %u",
- member_list[i]);
- if (ret >= len - pos)
- break;
- pos += ret;
- }
-
- log_debug("%s ring %u:%llu %zu memb%s",
- name, ringid->nodeid, (unsigned long long)ringid->seq,
- member_list_entries, m_buf);
-}
-
static void ls_info_in(struct ls_info *li)
{
li->ls_info_size = le32_to_cpu(li->ls_info_size);
@@ -243,93 +112,6 @@ static void ids_in(struct ls_info *li, struct id_info *ids)
}
}
-const char *msg_name(int type)
-{
- switch (type) {
- case DLM_MSG_PROTOCOL:
- return "protocol";
- case DLM_MSG_START:
- return "start";
- case DLM_MSG_PLOCK:
- return "plock";
- case DLM_MSG_PLOCK_OWN:
- return "plock_own";
- case DLM_MSG_PLOCK_DROP:
- return "plock_drop";
- case DLM_MSG_PLOCK_SYNC_LOCK:
- return "plock_sync_lock";
- case DLM_MSG_PLOCK_SYNC_WAITER:
- return "plock_sync_waiter";
- case DLM_MSG_PLOCKS_DATA:
- return "plocks_data";
- case DLM_MSG_PLOCKS_DONE:
- return "plocks_done";
- case DLM_MSG_DEADLK_CYCLE_START:
- return "deadlk_cycle_start";
- case DLM_MSG_DEADLK_CYCLE_END:
- return "deadlk_cycle_end";
- case DLM_MSG_DEADLK_CHECKPOINT_READY:
- return "deadlk_checkpoint_ready";
- case DLM_MSG_DEADLK_CANCEL_LOCK:
- return "deadlk_cancel_lock";
- default:
- return "unknown";
- }
-}
-
-static int _send_message(cpg_handle_t h, void *buf, int len, int type)
-{
- struct iovec iov;
- cs_error_t error;
- int retries = 0;
-
- iov.iov_base = buf;
- iov.iov_len = len;
-
- retry:
- error = cpg_mcast_joined(h, CPG_TYPE_AGREED, &iov, 1);
- if (error == CS_ERR_TRY_AGAIN) {
- retries++;
- usleep(1000);
- if (!(retries % 100))
- log_error("cpg_mcast_joined retry %d %s",
- retries, msg_name(type));
- goto retry;
- }
- if (error != CS_OK) {
- log_error("cpg_mcast_joined error %d handle %llx %s",
- error, (unsigned long long)h, msg_name(type));
- return -1;
- }
-
- if (retries)
- log_debug("cpg_mcast_joined retried %d %s",
- retries, msg_name(type));
-
- return 0;
-}
-
-/* header fields caller needs to set: type, to_nodeid, flags, msgdata */
-
-void dlm_send_message(struct lockspace *ls, char *buf, int len)
-{
- struct dlm_header *hd = (struct dlm_header *) buf;
- int type = hd->type;
-
- hd->version[0] = cpu_to_le16(our_protocol.daemon_run[0]);
- hd->version[1] = cpu_to_le16(our_protocol.daemon_run[1]);
- hd->version[2] = cpu_to_le16(our_protocol.daemon_run[2]);
- hd->type = cpu_to_le16(hd->type);
- hd->nodeid = cpu_to_le32(our_nodeid);
- hd->to_nodeid = cpu_to_le32(hd->to_nodeid);
- hd->global_id = cpu_to_le32(ls->global_id);
- hd->flags = cpu_to_le32(hd->flags);
- hd->msgdata = cpu_to_le32(hd->msgdata);
- hd->msgdata2 = cpu_to_le32(hd->msgdata2);
-
- _send_message(ls->cpg_handle, buf, len, type);
-}
-
static struct member *find_memb(struct change *cg, int nodeid)
{
struct member *memb;
@@ -424,7 +206,7 @@ static void free_ls(struct lockspace *ls)
For 1:
- node X fails
- we see node X fail and X has non-zero start_time,
- set check_fencing and record the fail time
+ set need_fencing and record the fail time
- wait for X to be removed from all dlm cpg's (probably not necessary)
- check that the fencing time is later than the recorded time above
@@ -442,8 +224,8 @@ static void free_ls(struct lockspace *ls)
when we see a node not in this list, add entry for it with zero start_time
record the time we get a good start message from the node, start_time
clear start_time if the node leaves
- if node fails with non-zero start_time, set check_fencing
- when a node is fenced, clear start_time and clear check_fencing
+ if node fails with non-zero start_time, set need_fencing
+ when a node is fenced, clear start_time and clear need_fencing
if a node remerges after this, no good start message, no new start_time set
if a node fails with zero start_time, it doesn't need fencing
if a node remerges before it's been fenced, no good start message, no new
@@ -526,9 +308,7 @@ static void node_history_lockspace_fail(struct lockspace *ls, int nodeid,
}
if (cfgd_enable_fencing && node->start_time) {
- node->request_fencing = 1;
- node->check_fencing = 1;
- node->fence_realtime = 0;
+ node->need_fencing = 1;
node->fence_queries = 0;
}
@@ -543,7 +323,9 @@ static void node_history_lockspace_fail(struct lockspace *ls, int nodeid,
node->lockspace_fail_time = now;
node->lockspace_fail_seq = node->lockspace_rem_seq;
node->lockspace_fail_reason = reason; /* for queries */
- node->fail_realtime = time(NULL);
+
+ node->fail_monotime = now;
+ node->fail_walltime = time(NULL);
}
static void node_history_start(struct lockspace *ls, int nodeid)
@@ -601,9 +383,9 @@ static int check_ringid_done(struct lockspace *ls)
static int check_fencing_done(struct lockspace *ls)
{
struct node *node;
- uint64_t last_fenced_time;
- int in_progress, wait_count = 0;
- int rv;
+ uint64_t fence_monotime;
+ int wait_count = 0;
+ int rv, in_progress;
if (!cfgd_enable_fencing) {
log_group(ls, "check_fencing disabled");
@@ -611,47 +393,35 @@ static int check_fencing_done(struct lockspace *ls)
}
list_for_each_entry(node, &ls->node_history, list) {
- if (!node->check_fencing)
+ if (!node->need_fencing)
continue;
- /* check with fenced to see if the node has been
- fenced since node->start_time */
-
- rv = fence_node_time(node->nodeid, &last_fenced_time);
+ rv = fence_node_time(node->nodeid, &fence_monotime);
if (rv < 0) {
log_error("fenced_node_time error %d", rv);
continue;
}
- /* fenced gives us real time */
-
- /* need >= not just > because in at least one case
- we've seen fenced_time within the same second as
- fail_time: with external fencing, e.g. fence_node */
-
- if (last_fenced_time >= node->fail_realtime) {
- log_group(ls, "check_fencing %d done "
- "start %llu fail %llu last %llu",
+ if (fence_monotime >= node->fail_monotime) {
+ log_group(ls, "check_fencing %d done start %llu fail %llu fence %llu",
node->nodeid,
(unsigned long long)node->start_time,
- (unsigned long long)node->fail_realtime,
- (unsigned long long)last_fenced_time);
- node->check_fencing = 0;
+ (unsigned long long)node->fail_monotime,
+ (unsigned long long)fence_monotime);
+
+ node->need_fencing = 0;
node->start_time = 0;
- node->fence_realtime = last_fenced_time;
+ continue;
} else {
- if (!node->fence_queries ||
- node->fence_realtime != last_fenced_time) {
- log_group(ls, "check_fencing %d wait "
- "start %llu fail %llu last %llu",
+ if (!node->fence_queries) {
+ log_group(ls, "check_fencing %d wait start %llu fail %llu",
node->nodeid,
(unsigned long long)node->start_time,
- (unsigned long long)node->fail_realtime,
- (unsigned long long)last_fenced_time);
+ (unsigned long long)node->fail_monotime);
node->fence_queries++;
- node->fence_realtime = last_fenced_time;
}
wait_count++;
+ continue;
}
}
@@ -679,39 +449,6 @@ static int check_fencing_done(struct lockspace *ls)
return 1;
}
-static int need_fencing(struct lockspace *ls)
-{
- struct node *node;
-
- list_for_each_entry(node, &ls->node_history, list) {
- if (node->check_fencing) {
- log_group(ls, "need_fencing %d", node->nodeid);
- return 1;
- }
- }
- return 0;
-}
-
-/* we don't need to ask fenced to initiate fencing; it does
- so itself when it sees a fence domain member fail. Without
- fenced we'll probably need to ask another daemon to initiate
- fencing, then check with it above, like we check libfenced. */
-
-static void request_fencing(struct lockspace *ls)
-{
- struct node *node;
- int rv;
-
- list_for_each_entry(node, &ls->node_history, list) {
- if (!node->request_fencing)
- continue;
- log_group(ls, "fence_request %d", node->nodeid);
- rv = fence_request(node->nodeid);
- if (!rv)
- node->request_fencing = 0;
- }
-}
-
/* wait for local fs_controld to ack each failed node */
static int check_fs_done(struct lockspace *ls)
@@ -839,43 +576,18 @@ static void stop_kernel(struct lockspace *ls, uint32_t seq)
/* we know that the cluster_quorate value here is consistent with the cpg events
because the ringid's are in sync per the check_ringid_done */
-/*
- enable_quorum = 0
- Never wait for quorum here.
-
- enable_quorum = 1
- Wait for quorum before calling request_fencing() in cases where
- fencing is needed. Reason for using this would be if fencing
- system doesn't wait for quorum before carrying out a fencing
- request, and we want to avoid having partitioned inquorate nodes
- fencing quorate nodes in another partition.
-
- enable_quorum = 2
- Always wait for quorum as a general condition here. This
- means that lockspace join/leave operations would block waiting
- for quorum. There's not any reason to do this per se, unless
- that's the behavior the application using the dlm wants.
-*/
-
static int wait_conditions_done(struct lockspace *ls)
{
if (!check_ringid_done(ls))
return 0;
- if ((cfgd_enable_quorum > 1) && !cluster_quorate) {
+ if ((cfgd_enable_quorum_lockspace > 1) && !cluster_quorate) {
log_group(ls, "wait for quorum");
return 0;
}
- if ((cfgd_enable_quorum > 0) && need_fencing(ls) && !cluster_quorate) {
- log_group(ls, "wait for quorum before fencing");
- return 0;
- }
-
- request_fencing(ls);
-
if (!check_fencing_done(ls)) {
- poll_fencing++;
+ poll_lockspaces++;
return 0;
}
@@ -1437,7 +1149,7 @@ static void apply_changes(struct lockspace *ls)
case CGST_WAIT_MESSAGES:
if (wait_messages_done(ls)) {
- our_protocol.dr_ver.flags |= PV_STATEFUL;
+ set_protocol_stateful();
start_kernel(ls);
prepare_plocks(ls);
cleanup_changes(ls);
@@ -1453,9 +1165,7 @@ void process_lockspace_changes(void)
{
struct lockspace *ls, *safe;
- poll_ringid = 0;
- poll_fencing = 0;
- poll_quorum = 0;
+ poll_lockspaces = 0;
poll_fs = 0;
list_for_each_entry_safe(ls, safe, &lockspaces, list) {
@@ -1464,24 +1174,6 @@ void process_lockspace_changes(void)
}
}
-static const char *reason_str(int reason)
-{
- switch (reason) {
- case CPG_REASON_JOIN:
- return "join";
- case CPG_REASON_LEAVE:
- return "leave";
- case CPG_REASON_NODEDOWN:
- return "nodedown";
- case CPG_REASON_NODEUP:
- return "nodeup";
- case CPG_REASON_PROCDOWN:
- return "procdown";
- default:
- return "unknown";
- };
-}
-
static int add_change(struct lockspace *ls,
const struct cpg_address *member_list,
size_t member_list_entries,
@@ -1665,20 +1357,6 @@ static void confchg_cb(cpg_handle_t handle,
#endif
}
-static void dlm_header_in(struct dlm_header *hd)
-{
- hd->version[0] = le16_to_cpu(hd->version[0]);
- hd->version[1] = le16_to_cpu(hd->version[1]);
- hd->version[2] = le16_to_cpu(hd->version[2]);
- hd->type = le16_to_cpu(hd->type);
- hd->nodeid = le32_to_cpu(hd->nodeid);
- hd->to_nodeid = le32_to_cpu(hd->to_nodeid);
- hd->global_id = le32_to_cpu(hd->global_id);
- hd->flags = le32_to_cpu(hd->flags);
- hd->msgdata = le32_to_cpu(hd->msgdata);
- hd->msgdata2 = le32_to_cpu(hd->msgdata2);
-}
-
/* after our join confchg, we want to ignore plock messages (see need_plocks
checks below) until the point in time where the ckpt_node saves plock
state (final start message received); at this time we want to shift from
@@ -1693,6 +1371,7 @@ static void deliver_cb(cpg_handle_t handle,
struct lockspace *ls;
struct dlm_header *hd;
int ignore_plock;
+ int rv;
ls = find_ls_handle(handle);
if (!ls) {
@@ -1700,7 +1379,7 @@ static void deliver_cb(cpg_handle_t handle,
return;
}
- if (len < sizeof(*hd)) {
+ if (len < sizeof(struct dlm_header)) {
log_error("deliver_cb short message %zd", len);
return;
}
@@ -1708,20 +1387,9 @@ static void deliver_cb(cpg_handle_t handle,
hd = (struct dlm_header *)data;
dlm_header_in(hd);
- if (hd->version[0] != our_protocol.daemon_run[0] ||
- hd->version[1] != our_protocol.daemon_run[1]) {
- log_error("reject message from %d version %u.%u.%u vs %u.%u.%u",
- nodeid, hd->version[0], hd->version[1],
- hd->version[2], our_protocol.daemon_run[0],
- our_protocol.daemon_run[1],
- our_protocol.daemon_run[2]);
+ rv = dlm_header_validate(hd, nodeid);
+ if (rv < 0)
return;
- }
-
- if (hd->nodeid != nodeid) {
- log_error("bad msg nodeid %d %d", hd->nodeid, nodeid);
- return;
- }
ignore_plock = 0;
@@ -1892,31 +1560,6 @@ static cpg_model_v1_data_t cpg_callbacks = {
.flags = CPG_MODEL_V1_DELIVER_INITIAL_TOTEM_CONF,
};
-void update_flow_control_status(void)
-{
- cpg_flow_control_state_t flow_control_state;
- cs_error_t error;
-
- error = cpg_flow_control_state_get(cpg_handle_daemon,
- &flow_control_state);
- if (error != CS_OK) {
- log_error("cpg_flow_control_state_get %d", error);
- return;
- }
-
- if (flow_control_state == CPG_FLOW_CONTROL_ENABLED) {
- if (message_flow_control_on == 0) {
- log_debug("flow control on");
- }
- message_flow_control_on = 1;
- } else {
- if (message_flow_control_on) {
- log_debug("flow control off");
- }
- message_flow_control_on = 0;
- }
-}
-
static void process_cpg_lockspace(int ci)
{
struct lockspace *ls;
@@ -1933,8 +1576,6 @@ static void process_cpg_lockspace(int ci)
log_error("cpg_dispatch error %d", error);
return;
}
-
- update_flow_control_status();
}
/* received an "online" uevent from dlm-kernel */
@@ -1945,14 +1586,6 @@ int dlm_join_lockspace(struct lockspace *ls)
cpg_handle_t h;
struct cpg_name name;
int i = 0, fd, ci, rv;
- int unused;
-
- rv = fence_in_progress(&unused);
- if (cfgd_enable_fencing && rv < 0) {
- log_error("dlm_join_lockspace no fence domain");
- rv = -1;
- goto fail_free;
- }
error = cpg_model_initialize(&h, CPG_MODEL_V1,
(cpg_model_data_t *)&cpg_callbacks, NULL);
@@ -2038,645 +1671,6 @@ int dlm_leave_lockspace(struct lockspace *ls)
return 0;
}
-static struct node_daemon *get_node_daemon(int nodeid)
-{
- struct node_daemon *node;
-
- list_for_each_entry(node, &daemon_nodes, list) {
- if (node->nodeid == nodeid)
- return node;
- }
- return NULL;
-}
-
-static void add_node_daemon(int nodeid)
-{
- struct node_daemon *node;
-
- if (get_node_daemon(nodeid))
- return;
-
- node = malloc(sizeof(struct node_daemon));
- if (!node) {
- log_error("add_node_daemon no mem");
- return;
- }
- memset(node, 0, sizeof(struct node_daemon));
- node->nodeid = nodeid;
- list_add_tail(&node->list, &daemon_nodes);
-}
-
-static void pv_in(struct protocol_version *pv)
-{
- pv->major = le16_to_cpu(pv->major);
- pv->minor = le16_to_cpu(pv->minor);
- pv->patch = le16_to_cpu(pv->patch);
- pv->flags = le16_to_cpu(pv->flags);
-}
-
-static void pv_out(struct protocol_version *pv)
-{
- pv->major = cpu_to_le16(pv->major);
- pv->minor = cpu_to_le16(pv->minor);
- pv->patch = cpu_to_le16(pv->patch);
- pv->flags = cpu_to_le16(pv->flags);
-}
-
-static void protocol_in(struct protocol *proto)
-{
- pv_in(&proto->dm_ver);
- pv_in(&proto->km_ver);
- pv_in(&proto->dr_ver);
- pv_in(&proto->kr_ver);
-}
-
-static void protocol_out(struct protocol *proto)
-{
- pv_out(&proto->dm_ver);
- pv_out(&proto->km_ver);
- pv_out(&proto->dr_ver);
- pv_out(&proto->kr_ver);
-}
-
-/* go through member list saved in last confchg, see if we have received a
- proto message from each */
-
-static int all_protocol_messages(void)
-{
- struct node_daemon *node;
- int i;
-
- if (!daemon_member_count)
- return 0;
-
- for (i = 0; i < daemon_member_count; i++) {
- node = get_node_daemon(daemon_member[i].nodeid);
- if (!node) {
- log_error("all_protocol_messages no node %d",
- daemon_member[i].nodeid);
- return 0;
- }
-
- if (!node->proto.daemon_max[0])
- return 0;
- }
- return 1;
-}
-
-static int pick_min_protocol(struct protocol *proto)
-{
- uint16_t mind[4];
- uint16_t mink[4];
- struct node_daemon *node;
- int i;
-
- memset(&mind, 0, sizeof(mind));
- memset(&mink, 0, sizeof(mink));
-
- /* first choose the minimum major */
-
- for (i = 0; i < daemon_member_count; i++) {
- node = get_node_daemon(daemon_member[i].nodeid);
- if (!node) {
- log_error("pick_min_protocol no node %d",
- daemon_member[i].nodeid);
- return -1;
- }
-
- if (!mind[0] || node->proto.daemon_max[0] < mind[0])
- mind[0] = node->proto.daemon_max[0];
-
- if (!mink[0] || node->proto.kernel_max[0] < mink[0])
- mink[0] = node->proto.kernel_max[0];
- }
-
- if (!mind[0] || !mink[0]) {
- log_error("pick_min_protocol zero major number");
- return -1;
- }
-
- /* second pick the minimum minor with the chosen major */
-
- for (i = 0; i < daemon_member_count; i++) {
- node = get_node_daemon(daemon_member[i].nodeid);
- if (!node)
- continue;
-
- if (mind[0] == node->proto.daemon_max[0]) {
- if (!mind[1] || node->proto.daemon_max[1] < mind[1])
- mind[1] = node->proto.daemon_max[1];
- }
-
- if (mink[0] == node->proto.kernel_max[0]) {
- if (!mink[1] || node->proto.kernel_max[1] < mink[1])
- mink[1] = node->proto.kernel_max[1];
- }
- }
-
- if (!mind[1] || !mink[1]) {
- log_error("pick_min_protocol zero minor number");
- return -1;
- }
-
- /* third pick the minimum patch with the chosen major.minor */
-
- for (i = 0; i < daemon_member_count; i++) {
- node = get_node_daemon(daemon_member[i].nodeid);
- if (!node)
- continue;
-
- if (mind[0] == node->proto.daemon_max[0] &&
- mind[1] == node->proto.daemon_max[1]) {
- if (!mind[2] || node->proto.daemon_max[2] < mind[2])
- mind[2] = node->proto.daemon_max[2];
- }
-
- if (mink[0] == node->proto.kernel_max[0] &&
- mink[1] == node->proto.kernel_max[1]) {
- if (!mink[2] || node->proto.kernel_max[2] < mink[2])
- mink[2] = node->proto.kernel_max[2];
- }
- }
-
- if (!mind[2] || !mink[2]) {
- log_error("pick_min_protocol zero patch number");
- return -1;
- }
-
- memcpy(&proto->daemon_run, &mind, sizeof(mind));
- memcpy(&proto->kernel_run, &mink, sizeof(mink));
- return 0;
-}
-
-static void receive_protocol(struct dlm_header *hd, int len)
-{
- struct protocol *p;
- struct node_daemon *node;
-
- p = (struct protocol *)((char *)hd + sizeof(struct dlm_header));
- protocol_in(p);
-
- if (len < sizeof(struct dlm_header) + sizeof(struct protocol)) {
- log_error("receive_protocol invalid len %d from %d",
- len, hd->nodeid);
- return;
- }
-
- /* zero is an invalid version value */
-
- if (!p->daemon_max[0] || !p->daemon_max[1] || !p->daemon_max[2] ||
- !p->kernel_max[0] || !p->kernel_max[1] || !p->kernel_max[2]) {
- log_error("receive_protocol invalid max value from %d "
- "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
- p->daemon_max[0], p->daemon_max[1], p->daemon_max[2],
- p->kernel_max[0], p->kernel_max[1], p->kernel_max[2]);
- return;
- }
-
- /* the run values will be zero until a version is set, after
- which none of the run values can be zero */
-
- if (p->daemon_run[0] && (!p->daemon_run[1] || !p->daemon_run[2] ||
- !p->kernel_run[0] || !p->kernel_run[1] || !p->kernel_run[2])) {
- log_error("receive_protocol invalid run value from %d "
- "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
- p->daemon_run[0], p->daemon_run[1], p->daemon_run[2],
- p->kernel_run[0], p->kernel_run[1], p->kernel_run[2]);
- return;
- }
-
- /* save this node's proto so we can tell when we've got all, and
- use it to select a minimum protocol from all */
-
- node = get_node_daemon(hd->nodeid);
- if (!node) {
- log_error("receive_protocol no node %d", hd->nodeid);
- return;
- }
-
- if (!node->daemon_member) {
- log_error("receive_protocol node %d not member", hd->nodeid);
- return;
- }
-
- log_debug("receive_protocol from %d max %u.%u.%u.%x run %u.%u.%u.%x",
- hd->nodeid,
- p->daemon_max[0], p->daemon_max[1],
- p->daemon_max[2], p->daemon_max[3],
- p->daemon_run[0], p->daemon_run[1],
- p->daemon_run[2], p->daemon_run[3]);
- log_debug("daemon node %d max %u.%u.%u.%x run %u.%u.%u.%x",
- hd->nodeid,
- node->proto.daemon_max[0], node->proto.daemon_max[1],
- node->proto.daemon_max[2], node->proto.daemon_max[3],
- node->proto.daemon_run[0], node->proto.daemon_run[1],
- node->proto.daemon_run[2], node->proto.daemon_run[3]);
- log_debug("daemon node %d join %llu left %llu local quorum %llu",
- hd->nodeid,
- (unsigned long long)node->daemon_add_time,
- (unsigned long long)node->daemon_rem_time,
- (unsigned long long)quorate_time);
-
- /* checking zero node->daemon_max[0] is a way to tell if we've received
- an acceptable (non-stateful) proto message from the node since we
- saw it join the daemon cpg */
-
- if (hd->nodeid != our_nodeid &&
- !node->proto.daemon_max[0] &&
- (p->dr_ver.flags & PV_STATEFUL) &&
- (our_protocol.dr_ver.flags & PV_STATEFUL)) {
-
- log_debug("daemon node %d stateful merge", hd->nodeid);
-
- if (cluster_quorate && node->daemon_rem_time &&
- quorate_time < node->daemon_rem_time) {
- log_debug("daemon node %d kill due to stateful merge", hd->nodeid);
- if (!node->killed)
- kick_node_from_cluster(hd->nodeid);
- node->killed = 1;
- }
-
- /* don't save p->proto into node->proto; we need to come
- through here based on zero daemon_max[0] for other proto
- messages like this one from the same node */
-
- return;
- }
-
- memcpy(&node->proto, p, sizeof(struct protocol));
-
- /* if we have zero run values, and this msg has non-zero run values,
- then adopt them as ours; otherwise save this proto message */
-
- if (our_protocol.daemon_run[0])
- return;
-
- if (p->daemon_run[0]) {
- our_protocol.daemon_run[0] = p->daemon_run[0];
- our_protocol.daemon_run[1] = p->daemon_run[1];
- our_protocol.daemon_run[2] = p->daemon_run[2];
-
- our_protocol.kernel_run[0] = p->kernel_run[0];
- our_protocol.kernel_run[1] = p->kernel_run[1];
- our_protocol.kernel_run[2] = p->kernel_run[2];
-
- log_debug("run protocol from nodeid %d", hd->nodeid);
- }
-}
-
-static void send_protocol(struct protocol *proto)
-{
- struct dlm_header *hd;
- struct protocol *pr;
- char *buf;
- int len;
-
- len = sizeof(struct dlm_header) + sizeof(struct protocol);
- buf = malloc(len);
- if (!buf) {
- log_error("send_protocol no mem %d", len);
- return;
- }
- memset(buf, 0, len);
-
- hd = (struct dlm_header *)buf;
- pr = (struct protocol *)(buf + sizeof(*hd));
-
- hd->type = cpu_to_le16(DLM_MSG_PROTOCOL);
- hd->nodeid = cpu_to_le32(our_nodeid);
-
- memcpy(pr, proto, sizeof(struct protocol));
- protocol_out(pr);
-
- _send_message(cpg_handle_daemon, buf, len, DLM_MSG_PROTOCOL);
-}
-
-int set_protocol(void)
-{
- struct protocol proto;
- struct pollfd pollfd;
- int sent_proposal = 0;
- int rv;
-
- memset(&pollfd, 0, sizeof(pollfd));
- pollfd.fd = cpg_fd_daemon;
- pollfd.events = POLLIN;
-
- while (1) {
- if (our_protocol.daemon_run[0])
- break;
-
- if (!sent_proposal && all_protocol_messages()) {
- /* propose a protocol; look through info from all
- nodes and pick the min for both daemon and kernel,
- and propose that */
-
- sent_proposal = 1;
-
- /* copy our max values */
- memcpy(&proto, &our_protocol, sizeof(struct protocol));
-
- rv = pick_min_protocol(&proto);
- if (rv < 0)
- return rv;
-
- log_debug("set_protocol member_count %d propose "
- "daemon %u.%u.%u kernel %u.%u.%u",
- daemon_member_count,
- proto.daemon_run[0], proto.daemon_run[1],
- proto.daemon_run[2], proto.kernel_run[0],
- proto.kernel_run[1], proto.kernel_run[2]);
-
- send_protocol(&proto);
- }
-
- /* only process messages/events from daemon cpg until protocol
- is established */
-
- rv = poll(&pollfd, 1, -1);
- if (rv == -1 && errno == EINTR) {
- if (daemon_quit)
- return -1;
- continue;
- }
- if (rv < 0) {
- log_error("set_protocol poll errno %d", errno);
- return -1;
- }
-
- if (pollfd.revents & POLLIN)
- process_cpg_daemon(0);
- if (pollfd.revents & (POLLERR | POLLHUP | POLLNVAL)) {
- log_error("set_protocol poll revents %u",
- pollfd.revents);
- return -1;
- }
- }
-
- if (our_protocol.daemon_run[0] != our_protocol.daemon_max[0] ||
- our_protocol.daemon_run[1] > our_protocol.daemon_max[1]) {
- log_error("incompatible daemon protocol run %u.%u.%u max %u.%u.%u",
- our_protocol.daemon_run[0],
- our_protocol.daemon_run[1],
- our_protocol.daemon_run[2],
- our_protocol.daemon_max[0],
- our_protocol.daemon_max[1],
- our_protocol.daemon_max[2]);
- return -1;
- }
-
- if (our_protocol.kernel_run[0] != our_protocol.kernel_max[0] ||
- our_protocol.kernel_run[1] > our_protocol.kernel_max[1]) {
- log_error("incompatible kernel protocol run %u.%u.%u max %u.%u.%u",
- our_protocol.kernel_run[0],
- our_protocol.kernel_run[1],
- our_protocol.kernel_run[2],
- our_protocol.kernel_max[0],
- our_protocol.kernel_max[1],
- our_protocol.kernel_max[2]);
- return -1;
- }
-
- log_debug("daemon run %u.%u.%u max %u.%u.%u "
- "kernel run %u.%u.%u max %u.%u.%u",
- our_protocol.daemon_run[0],
- our_protocol.daemon_run[1],
- our_protocol.daemon_run[2],
- our_protocol.daemon_max[0],
- our_protocol.daemon_max[1],
- our_protocol.daemon_max[2],
- our_protocol.kernel_run[0],
- our_protocol.kernel_run[1],
- our_protocol.kernel_run[2],
- our_protocol.kernel_max[0],
- our_protocol.kernel_max[1],
- our_protocol.kernel_max[2]);
-
- send_protocol(&our_protocol);
- return 0;
-}
-
-static void deliver_cb_daemon(cpg_handle_t handle,
- const struct cpg_name *group_name,
- uint32_t nodeid, uint32_t pid,
- void *data, size_t len)
-{
- struct dlm_header *hd;
-
- if (len < sizeof(*hd)) {
- log_error("deliver_cb short message %zd", len);
- return;
- }
-
- hd = (struct dlm_header *)data;
- dlm_header_in(hd);
-
- switch (hd->type) {
- case DLM_MSG_PROTOCOL:
- receive_protocol(hd, len);
- break;
- default:
- log_error("deliver_cb_daemon unknown msg type %d", hd->type);
- }
-}
-
-static int in_daemon_member_list(int nodeid)
-{
- int i;
-
- for (i = 0; i < daemon_member_count; i++) {
- if (daemon_member[i].nodeid == nodeid)
- return 1;
- }
- return 0;
-}
-
-static void confchg_cb_daemon(cpg_handle_t handle,
- const struct cpg_name *group_name,
- const struct cpg_address *member_list,
- size_t member_list_entries,
- const struct cpg_address *left_list,
- size_t left_list_entries,
- const struct cpg_address *joined_list,
- size_t joined_list_entries)
-{
- struct node_daemon *node;
- uint64_t now = monotime();
- int nodedown = 0, procdown = 0, leave = 0;
- int i;
-
- log_config(group_name, member_list, member_list_entries,
- left_list, left_list_entries,
- joined_list, joined_list_entries);
-
- for (i = 0; i < left_list_entries; i++) {
- if (left_list[i].reason == CPG_REASON_NODEDOWN)
- nodedown++;
- else if (left_list[i].reason == CPG_REASON_PROCDOWN)
- procdown++;
- else if (left_list[i].reason == CPG_REASON_LEAVE)
- leave++;
- }
-
- if (nodedown || procdown || leave)
- log_debug("%s left nodedown %d procdown %d leave %d",
- group_name->value, nodedown, procdown, leave);
-
- if (joined_list_entries)
- send_protocol(&our_protocol);
-
- memset(&daemon_member, 0, sizeof(daemon_member));
- daemon_member_count = member_list_entries;
-
- for (i = 0; i < member_list_entries; i++) {
- daemon_member[i] = member_list[i];
- /* add struct for nodes we've not seen before */
- add_node_daemon(member_list[i].nodeid);
- }
-
- list_for_each_entry(node, &daemon_nodes, list) {
- if (in_daemon_member_list(node->nodeid)) {
- if (node->daemon_member)
- continue;
-
- /* node joined daemon cpg */
- node->daemon_member = 1;
- node->daemon_add_time = now;
- } else {
- if (!node->daemon_member)
- continue;
-
- /* node left daemon cpg */
- node->daemon_member = 0;
- node->killed = 0;
- memset(&node->proto, 0, sizeof(struct protocol));
- node->daemon_rem_time = now;
- }
- }
-}
-
-static void totem_cb_daemon(cpg_handle_t handle,
- struct cpg_ring_id ring_id,
- uint32_t member_list_entries,
- const uint32_t *member_list)
-{
- log_ringid("dlm:controld", &ring_id, member_list, member_list_entries);
-}
-
-static cpg_model_v1_data_t cpg_callbacks_daemon = {
- .cpg_deliver_fn = deliver_cb_daemon,
- .cpg_confchg_fn = confchg_cb_daemon,
- .cpg_totem_confchg_fn = totem_cb_daemon,
- .flags = CPG_MODEL_V1_DELIVER_INITIAL_TOTEM_CONF,
-};
-
-void process_cpg_daemon(int ci)
-{
- cs_error_t error;
-
- error = cpg_dispatch(cpg_handle_daemon, CS_DISPATCH_ALL);
- if (error != CS_OK)
- log_error("daemon cpg_dispatch error %d", error);
-}
-
-int setup_cpg_daemon(void)
-{
- cs_error_t error;
- struct cpg_name name;
- int i = 0;
-
- INIT_LIST_HEAD(&daemon_nodes);
-
- /* daemon 1.1.1 was cluster3/STABLE3/RHEL6 which is incompatible
- with cluster4/RHEL7 */
-
- memset(&our_protocol, 0, sizeof(our_protocol));
-
- if (cfgd_enable_fscontrol)
- our_protocol.daemon_max[0] = 2;
- else
- our_protocol.daemon_max[0] = 3;
-
- our_protocol.daemon_max[1] = 1;
- our_protocol.daemon_max[2] = 1;
- our_protocol.kernel_max[0] = 1;
- our_protocol.kernel_max[1] = 1;
- our_protocol.kernel_max[2] = 1;
-
- error = cpg_model_initialize(&cpg_handle_daemon, CPG_MODEL_V1,
- (cpg_model_data_t *)&cpg_callbacks_daemon,
- NULL);
- if (error != CS_OK) {
- log_error("daemon cpg_initialize error %d", error);
- return -1;
- }
-
- cpg_fd_get(cpg_handle_daemon, &cpg_fd_daemon);
-
- memset(&name, 0, sizeof(name));
- sprintf(name.value, "dlm:controld");
- name.length = strlen(name.value) + 1;
-
- log_debug("cpg_join %s ...", name.value);
- retry:
- error = cpg_join(cpg_handle_daemon, &name);
- if (error == CS_ERR_TRY_AGAIN) {
- sleep(1);
- if (!(++i % 10))
- log_error("daemon cpg_join error retrying");
- goto retry;
- }
- if (error != CS_OK) {
- log_error("daemon cpg_join error %d", error);
- goto fail;
- }
-
- log_debug("setup_cpg_daemon %d", cpg_fd_daemon);
- return cpg_fd_daemon;
-
- fail:
- cpg_finalize(cpg_handle_daemon);
- return -1;
-}
-
-void close_cpg_daemon(void)
-{
- struct lockspace *ls;
- cs_error_t error;
- struct cpg_name name;
- int i = 0;
-
- if (!cpg_handle_daemon)
- return;
- if (cluster_down)
- goto fin;
-
- memset(&name, 0, sizeof(name));
- sprintf(name.value, "dlm:controld");
- name.length = strlen(name.value) + 1;
-
- log_debug("cpg_leave %s ...", name.value);
- retry:
- error = cpg_leave(cpg_handle_daemon, &name);
- if (error == CS_ERR_TRY_AGAIN) {
- sleep(1);
- if (!(++i % 10))
- log_error("daemon cpg_leave error retrying");
- goto retry;
- }
- if (error != CS_OK)
- log_error("daemon cpg_leave error %d", error);
- fin:
- list_for_each_entry(ls, &lockspaces, list) {
- if (ls->cpg_handle)
- cpg_finalize(ls->cpg_handle);
- }
- cpg_finalize(cpg_handle_daemon);
-}
-
-/* fs_controld has seen nodedown for nodeid; it's now ok for dlm to do
- recovery for the failed node */
-
int set_fs_notified(struct lockspace *ls, int nodeid)
{
struct node *node;
@@ -2755,12 +1749,8 @@ int set_lockspace_info(struct lockspace *ls, struct dlmc_lockspace *lockspace)
if (cg->state == CGST_WAIT_CONDITIONS)
lockspace->cg_next.wait_condition = 5;
- if (poll_ringid)
- lockspace->cg_next.wait_condition = 1;
else if (poll_fencing)
lockspace->cg_next.wait_condition = 2;
- else if (poll_quorum)
- lockspace->cg_next.wait_condition = 3;
else if (poll_fs)
lockspace->cg_next.wait_condition = 4;
@@ -2795,7 +1785,7 @@ static int _set_node_info(struct lockspace *ls, struct change *cg, int nodeid,
if (!n)
goto out;
- if (n->check_fencing)
+ if (n->need_fencing)
node->flags |= DLMC_NF_CHECK_FENCING;
if (n->check_fs)
node->flags |= DLMC_NF_CHECK_FS;
diff --git a/dlm_controld/daemon_cpg.c b/dlm_controld/daemon_cpg.c
new file mode 100644
index 0000000..e5e9270
--- /dev/null
+++ b/dlm_controld/daemon_cpg.c
@@ -0,0 +1,1921 @@
+/*
+ * Copyright 2004-2012 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+#include "dlm_daemon.h"
+
+/* protocol_version flags */
+#define PV_STATEFUL 0x0001
+
+struct protocol_version {
+ uint16_t major;
+ uint16_t minor;
+ uint16_t patch;
+ uint16_t flags;
+};
+
+struct protocol {
+ union {
+ struct protocol_version dm_ver;
+ uint16_t daemon_max[4];
+ };
+ union {
+ struct protocol_version km_ver;
+ uint16_t kernel_max[4];
+ };
+ union {
+ struct protocol_version dr_ver;
+ uint16_t daemon_run[4];
+ };
+ union {
+ struct protocol_version kr_ver;
+ uint16_t kernel_run[4];
+ };
+};
+
+/* fence_result flags */
+#define FR_FIPU 0x00000001
+#define FR_CLEAR_STARTUP 0x00000002
+#define FR_CLEAR_FIPU 0x00000004
+
+struct fence_result {
+ uint32_t version;
+ uint32_t flags;
+ uint32_t nodeid;
+ uint32_t result;
+ uint64_t fence_walltime;
+ char unused[1000];
+};
+
+struct node_daemon {
+ struct list_head list;
+ int nodeid;
+
+ uint64_t daemon_add_time;
+ uint64_t daemon_rem_time;
+ int daemon_member;
+
+ int killed;
+
+ struct protocol proto;
+
+ struct fence_config fence_config;
+
+ int fence_in_progress_unknown;
+ int left_reason;
+ int recover_setup;
+ int need_fence_clear;
+ int need_fencing;
+ int fence_pid;
+ int fence_pid_wait;
+ int fence_actor_done;
+ int fence_actors[MAX_NODES];
+ uint64_t fail_walltime;
+ uint64_t fail_monotime;
+ uint64_t fence_request_time;
+ uint64_t fence_walltime;
+ uint64_t fence_monotime;
+};
+
+#define REASON_STARTUP_FENCING -1
+
+static cpg_handle_t cpg_handle_daemon;
+static int cpg_fd_daemon;
+static struct protocol our_protocol;
+static struct list_head daemon_nodes;
+static struct list_head startup_nodes;
+static struct cpg_address daemon_member[MAX_NODES];
+static struct cpg_address daemon_joined[MAX_NODES];
+static struct cpg_address daemon_remove[MAX_NODES];
+static int daemon_member_count;
+static int daemon_joined_count;
+static int daemon_remove_count;
+static int daemon_ringid_wait;
+static struct cpg_ring_id daemon_ringid;
+static int daemon_clear_nodeid;
+static int daemon_clear_pid;
+static uint64_t last_join_monotime;
+static uint32_t last_join_seq;
+static uint32_t send_fipu_seq;
+static int fence_in_progress_unknown = 1;
+
+static void send_fence_result(int nodeid, int result, uint32_t flags, uint64_t walltime);
+static void send_fence_clear(int nodeid, int result, uint32_t flags, uint64_t walltime);
+
+void log_config(const struct cpg_name *group_name,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries)
+{
+ char m_buf[128];
+ char j_buf[32];
+ char l_buf[32];
+ size_t i, len, pos;
+ int ret;
+
+ memset(m_buf, 0, sizeof(m_buf));
+ memset(j_buf, 0, sizeof(j_buf));
+ memset(l_buf, 0, sizeof(l_buf));
+
+ len = sizeof(m_buf);
+ pos = 0;
+ for (i = 0; i < member_list_entries; i++) {
+ ret = snprintf(m_buf + pos, len - pos, " %d",
+ member_list[i].nodeid);
+ if (ret >= len - pos)
+ break;
+ pos += ret;
+ }
+
+ len = sizeof(j_buf);
+ pos = 0;
+ for (i = 0; i < joined_list_entries; i++) {
+ ret = snprintf(j_buf + pos, len - pos, " %d",
+ joined_list[i].nodeid);
+ if (ret >= len - pos)
+ break;
+ pos += ret;
+ }
+
+ len = sizeof(l_buf);
+ pos = 0;
+ for (i = 0; i < left_list_entries; i++) {
+ ret = snprintf(l_buf + pos, len - pos, " %d",
+ left_list[i].nodeid);
+ if (ret >= len - pos)
+ break;
+ pos += ret;
+ }
+
+ log_debug("%s conf %zu %zu %zu memb%s join%s left%s", group_name->value,
+ member_list_entries, joined_list_entries, left_list_entries,
+ m_buf, j_buf, l_buf);
+}
+
+void log_ringid(const char *name,
+ struct cpg_ring_id *ringid,
+ const uint32_t *member_list,
+ size_t member_list_entries)
+{
+ char m_buf[128];
+ size_t i, len, pos;
+ int ret;
+
+ memset(m_buf, 0, sizeof(m_buf));
+
+ len = sizeof(m_buf);
+ pos = 0;
+ for (i = 0; i < member_list_entries; i++) {
+ ret = snprintf(m_buf + pos, len - pos, " %u",
+ member_list[i]);
+ if (ret >= len - pos)
+ break;
+ pos += ret;
+ }
+
+ log_debug("%s ring %u:%llu %zu memb%s",
+ name, ringid->nodeid, (unsigned long long)ringid->seq,
+ member_list_entries, m_buf);
+}
+
+const char *reason_str(int reason)
+{
+ switch (reason) {
+ case CPG_REASON_JOIN:
+ return "join";
+ case CPG_REASON_LEAVE:
+ return "leave";
+ case CPG_REASON_NODEDOWN:
+ return "nodedown";
+ case CPG_REASON_NODEUP:
+ return "nodeup";
+ case CPG_REASON_PROCDOWN:
+ return "procdown";
+ default:
+ return "unknown";
+ };
+}
+
+const char *msg_name(int type)
+{
+ switch (type) {
+ case DLM_MSG_PROTOCOL:
+ return "protocol";
+ case DLM_MSG_FENCE_RESULT:
+ return "fence_result";
+ case DLM_MSG_FENCE_CLEAR:
+ return "fence_clear";
+
+ case DLM_MSG_START:
+ return "start";
+ case DLM_MSG_PLOCK:
+ return "plock";
+ case DLM_MSG_PLOCK_OWN:
+ return "plock_own";
+ case DLM_MSG_PLOCK_DROP:
+ return "plock_drop";
+ case DLM_MSG_PLOCK_SYNC_LOCK:
+ return "plock_sync_lock";
+ case DLM_MSG_PLOCK_SYNC_WAITER:
+ return "plock_sync_waiter";
+ case DLM_MSG_PLOCKS_DATA:
+ return "plocks_data";
+ case DLM_MSG_PLOCKS_DONE:
+ return "plocks_done";
+ case DLM_MSG_DEADLK_CYCLE_START:
+ return "deadlk_cycle_start";
+ case DLM_MSG_DEADLK_CYCLE_END:
+ return "deadlk_cycle_end";
+ case DLM_MSG_DEADLK_CHECKPOINT_READY:
+ return "deadlk_checkpoint_ready";
+ case DLM_MSG_DEADLK_CANCEL_LOCK:
+ return "deadlk_cancel_lock";
+ default:
+ return "unknown";
+ }
+}
+
+static int _send_message(cpg_handle_t h, void *buf, int len, int type)
+{
+ struct iovec iov;
+ cs_error_t error;
+ int retries = 0;
+
+ iov.iov_base = buf;
+ iov.iov_len = len;
+
+ retry:
+ error = cpg_mcast_joined(h, CPG_TYPE_AGREED, &iov, 1);
+ if (error == CS_ERR_TRY_AGAIN) {
+ retries++;
+ usleep(1000);
+ if (!(retries % 100))
+ log_error("cpg_mcast_joined retry %d %s",
+ retries, msg_name(type));
+ goto retry;
+ }
+ if (error != CS_OK) {
+ log_error("cpg_mcast_joined error %d handle %llx %s",
+ error, (unsigned long long)h, msg_name(type));
+ return -1;
+ }
+
+ if (retries)
+ log_debug("cpg_mcast_joined retried %d %s",
+ retries, msg_name(type));
+
+ return 0;
+}
+
+/* header fields caller needs to set: type, to_nodeid, flags, msgdata */
+
+void dlm_send_message(struct lockspace *ls, char *buf, int len)
+{
+ struct dlm_header *hd = (struct dlm_header *) buf;
+ int type = hd->type;
+
+ hd->version[0] = cpu_to_le16(our_protocol.daemon_run[0]);
+ hd->version[1] = cpu_to_le16(our_protocol.daemon_run[1]);
+ hd->version[2] = cpu_to_le16(our_protocol.daemon_run[2]);
+ hd->type = cpu_to_le16(hd->type);
+ hd->nodeid = cpu_to_le32(our_nodeid);
+ hd->to_nodeid = cpu_to_le32(hd->to_nodeid);
+ hd->global_id = cpu_to_le32(ls->global_id);
+ hd->flags = cpu_to_le32(hd->flags);
+ hd->msgdata = cpu_to_le32(hd->msgdata);
+ hd->msgdata2 = cpu_to_le32(hd->msgdata2);
+
+ _send_message(ls->cpg_handle, buf, len, type);
+}
+
+void dlm_header_in(struct dlm_header *hd)
+{
+ hd->version[0] = le16_to_cpu(hd->version[0]);
+ hd->version[1] = le16_to_cpu(hd->version[1]);
+ hd->version[2] = le16_to_cpu(hd->version[2]);
+ hd->type = le16_to_cpu(hd->type);
+ hd->nodeid = le32_to_cpu(hd->nodeid);
+ hd->to_nodeid = le32_to_cpu(hd->to_nodeid);
+ hd->global_id = le32_to_cpu(hd->global_id);
+ hd->flags = le32_to_cpu(hd->flags);
+ hd->msgdata = le32_to_cpu(hd->msgdata);
+ hd->msgdata2 = le32_to_cpu(hd->msgdata2);
+}
+
+int dlm_header_validate(struct dlm_header *hd, int nodeid)
+{
+ if (hd->version[0] != our_protocol.daemon_run[0] ||
+ hd->version[1] != our_protocol.daemon_run[1]) {
+ log_error("reject message from %d version %u.%u.%u vs %u.%u.%u",
+ nodeid, hd->version[0], hd->version[1],
+ hd->version[2], our_protocol.daemon_run[0],
+ our_protocol.daemon_run[1],
+ our_protocol.daemon_run[2]);
+ return -1;
+ }
+
+ if (hd->nodeid != nodeid) {
+ log_error("bad msg nodeid %d %d", hd->nodeid, nodeid);
+ return -1;
+ }
+
+ return 0;
+}
+
+static struct node_daemon *get_node_daemon(int nodeid)
+{
+ struct node_daemon *node;
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (node->nodeid == nodeid)
+ return node;
+ }
+ return NULL;
+}
+
+static int nodes_need_fencing(void)
+{
+ struct node_daemon *node;
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (node->need_fencing)
+ return 1;
+ }
+ return 0;
+}
+
+static int all_daemon_members_fipu(void)
+{
+ struct node_daemon *node;
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (!node->daemon_member)
+ continue;
+ if (!node->fence_in_progress_unknown)
+ return 0;
+ }
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (!node->daemon_member)
+ continue;
+ node->fence_in_progress_unknown = 0;
+ }
+
+ return 1;
+}
+
+int fence_node_time(int nodeid, uint64_t *last_fenced)
+{
+ struct node_daemon *node;
+
+ node = get_node_daemon(nodeid);
+ if (!node)
+ return -1;
+
+ *last_fenced = node->fence_monotime;
+ return 0;
+}
+
+int fence_in_progress(int *in_progress)
+{
+ if (fence_in_progress_unknown) {
+ *in_progress = 1;
+ } else if (!list_empty(&startup_nodes)) {
+ *in_progress = 2;
+ } else if (nodes_need_fencing()) {
+ *in_progress = 3;
+ } else {
+ *in_progress = 0;
+ }
+ return 0;
+}
+
+void add_startup_node(int nodeid)
+{
+ struct node_daemon *node;
+
+ node = malloc(sizeof(struct node_daemon));
+ if (!node) {
+ log_error("add_startup_node no mem");
+ return;
+ }
+ memset(node, 0, sizeof(struct node_daemon));
+ node->nodeid = nodeid;
+ list_add_tail(&node->list, &startup_nodes);
+}
+
+static int clear_startup_node(int nodeid, int all)
+{
+ struct node_daemon *node, *safe;
+ int count = 0;
+
+ list_for_each_entry_safe(node, safe, &startup_nodes, list) {
+ if (all || node->nodeid == nodeid) {
+ list_del(&node->list);
+ free(node);
+ count++;
+ }
+ }
+ return count;
+}
+
+static struct node_daemon *add_node_daemon(int nodeid)
+{
+ struct node_daemon *node;
+ struct fence_config *fc;
+
+ node = get_node_daemon(nodeid);
+ if (node)
+ return node;
+
+ node = malloc(sizeof(struct node_daemon));
+ if (!node) {
+ log_error("add_node_daemon no mem");
+ return NULL;
+ }
+ memset(node, 0, sizeof(struct node_daemon));
+ node->nodeid = nodeid;
+ list_add_tail(&node->list, &daemon_nodes);
+
+ /* TODO: allow the config to be reread */
+
+ fc = &node->fence_config;
+ fc->nodeid = nodeid;
+
+ /* explicit command line arg has first priority */
+
+ if (optd_fence_all_agent) {
+ fc->dev[0] = &fence_all_device;
+ goto out;
+ }
+
+ /* explicit config file setting has second priority */
+
+ fence_config_init(fc, (unsigned int)nodeid, (char *)CONF_FILE_PATH);
+
+ /* no command line, no config file, use default, third priority */
+
+ if (!fc->dev[0] && fence_all_agent[0])
+ fc->dev[0] = &fence_all_device;
+ out:
+ return node;
+}
+
+/* A clean daemon member is a node that has joined the daemon cpg
+ from a "clean state", i.e. not a stateful merge. If would not
+ have joined the daemon cpg if it found uncontrolled dlm kernel
+ state (check_uncontrolled_lockspaces). We would not have
+ accepted and saved its protocol in node->proto.daemon if it
+ was a stateful merge. */
+
+static int is_clean_daemon_member(int nodeid)
+{
+ struct node_daemon *node;
+
+ node = get_node_daemon(nodeid);
+ if (node && node->daemon_member && node->proto.daemon_max[0])
+ return 1;
+ return 0;
+}
+
+static int in_daemon_list(int nodeid, struct cpg_address *daemon_list, int count)
+{
+ int i;
+
+ for (i = 0; i < count; i++) {
+ if (daemon_list[i].nodeid == nodeid)
+ return 1;
+ }
+ return 0;
+}
+
+/* save in node->fence_actors[] any nodeid present when the node
+ failed which therefore saw it fail, knows it needs fencing, and
+ can request fencing for it if it becomes the low actor. A node
+ added in the same change with the removed node does not qualify. */
+
+static int set_fence_actors(struct node_daemon *node, int all_memb)
+{
+ int i, nodeid, count = 0, low = 0;
+
+ memset(node->fence_actors, 0, sizeof(node->fence_actors));
+
+ for (i = 0; i < daemon_member_count; i++) {
+ nodeid = daemon_member[i].nodeid;
+
+ if (!all_memb && in_daemon_list(nodeid, daemon_joined, daemon_joined_count))
+ continue;
+
+ node->fence_actors[count++] = nodeid;
+
+ if (!low || nodeid < low)
+ low = nodeid;
+ }
+
+ log_debug("set_fence_actors for %d low %d count %d",
+ node->nodeid, low, count);
+ return low;
+}
+
+static int get_fence_actor(struct node_daemon *node)
+{
+ int i, low, low_i;
+
+ retry:
+ low = 0;
+
+ for (i = 0; i < MAX_NODES; i++) {
+ if (!node->fence_actors[i])
+ continue;
+
+ if (!low || node->fence_actors[i] < low) {
+ low = node->fence_actors[i];
+ low_i = i;
+ }
+ }
+
+ if (low && !in_daemon_list(low, daemon_member, daemon_member_count)) {
+ log_debug("get_fence_actor for %d low actor %d is gone",
+ node->nodeid, low);
+
+ node->fence_actors[low_i] = 0;
+ goto retry;
+ }
+
+ return low;
+}
+
+/* if an actor fails to fence, it will send that result, and others
+ will clear it from the actors, which will cause the next lowest
+ actor to try */
+
+static void clear_fence_actor(int nodeid, int actor)
+{
+ struct node_daemon *node;
+ int i;
+
+ node = get_node_daemon(nodeid);
+ if (!node)
+ return;
+
+ for (i = 0; i < MAX_NODES; i++) {
+ if (node->fence_actors[i] == actor) {
+ node->fence_actors[i] = 0;
+ return;
+ }
+ }
+}
+
+/* TODO: handle delayed cleanup of more than one pid */
+
+static void fence_pid_cancel(int nodeid, int pid)
+{
+ struct node_daemon *node;
+ int rv, result;
+
+ log_debug("fence_pid_cancel nodeid %d pid %d", nodeid, pid);
+
+ kill(pid, SIGKILL);
+ usleep(500000);
+
+ rv = fence_result(nodeid, pid, &result);
+ if (rv == -EAGAIN) {
+ /* Try again later */
+ daemon_clear_nodeid = nodeid;
+ daemon_clear_pid = pid;
+ } else {
+ log_debug("fence_pid_cancel nodeid %d pid %d done %d",
+ nodeid, pid, result);
+
+ daemon_clear_nodeid = 0;
+ daemon_clear_pid = 0;
+
+ node = get_node_daemon(nodeid);
+ if (node && node->fence_pid == pid) {
+ node->fence_pid_wait = 0;
+ node->fence_pid = 0;
+ }
+ }
+}
+
+/*
+ * fence_in_progress_unknown (fipu)
+ *
+ * If current daemon members are fencing someone, and a new node
+ * joins, that new node needs to wait for the previous members to
+ * finish any fencing they're doing before it can start a lockspace.
+ *
+ * The previous members may be fencing the last node that was using
+ * the lockspace the new node is going to use, so if it doesn't wait,
+ * it could start using a lockspace with an unfenced user.
+ *
+ * So, the daemon starts with fence_in_progress_unknown set to
+ * indicate that other nodes may be fencing someone, and it won't
+ * start any lockspaces until it is clear.
+ *
+ * A node starts with fence_in_progress_unknown set and won't
+ * start any lockspaces until it's clear.
+ *
+ * When using startup_fencing:
+ *
+ * . When all nodes start up together, all have fipu set,
+ * and will go through startup fencing, which will eventually
+ * result in all nodes either being clean daemon members or fenced,
+ * so everyone will clear fipu by seeing that.
+ *
+ * . The more common case is when a new node joins other previously
+ * running nodes. The new node needs to be told that the others
+ * have no outstanding fencing ops before it can clear fipu.
+ * A previous member does send_fence_clear(0) to a new node once
+ * all fencing is complete. The two flags in send_fence_clear are
+ * usually sent together but may sometimes may be in separate messages:
+ * send_fence_clear(0, CLEAR_STARTUP) to clear startup_nodes right away
+ * send_fence_clear(0, CLEAR_FIPU) to clear fipu once all fencing is done
+ *
+ * When not using startup_fencing:
+ *
+ * . When all nodes start up together, all have fipu set, and all
+ * will be waiting to receive_fence_clear from a previous node
+ * in order to clear it. The nodes need to detect this situation,
+ * and when they do, they will know that everyone is in startup,
+ * so there can be no pending fencing on a previous node, so all
+ * can clear fipu. To detect this case, when a node starts up
+ * with !startup_fence, it sends a special send_fence_clear(-ENODATA, FIPU)
+ * message about itself to indicate it has fipu set and needs it cleared.
+ * After sending this, it checks to see if all present nodes have sent
+ * this same message about themselves. If so, then this startup
+ * case has been detected, an all will clear fipu.
+ *
+ * . New nodes that join after this startup initialization will be
+ * handled the same as when startup_fencing is set (above).
+ *
+ *
+ * startup_fencing
+ * ---------------
+ *
+ * case A
+ * all nodes start up,
+ * all have fipu set,
+ * all wait for startup_nodes to be empty, (joined or moved to need_fencing)
+ * all wait for no daemon_nodes to need_fencing, (joined or were fenced)
+ * all clear fipu
+ *
+ * later,
+ *
+ * case B
+ * new node starts,
+ * new node has fipu set,
+ * cur node sees need_fence_clear on new node
+ * cur node sees no pending fencing ops,
+ * cur node send_fence_clear(0) to new node,
+ * new node clears startup_nodes and fipu
+ *
+ * !startup_fencing
+ * ----------------
+ *
+ * case C
+ * all nodes start up,
+ * all have fipu set,
+ * all send_fence_clear(-ENODATA,FIPU),
+ * all receive_fence_clear(-ENODATA,FIPU) from everyone,
+ * all_daemon_members_fipu() is 1,
+ * all clear fipu
+ *
+ * later same as case B above
+ */
+
+/*
+ * TODO: limit to one agent running at once, in case both
+ * instances need to log into the same switch, for example.
+ */
+
+static void daemon_fence_work(void)
+{
+ struct node_daemon *node, *safe;
+ int rv, nodeid, pid, need, low, actor, result;
+ uint32_t flags;
+
+ if (daemon_ringid_wait) {
+ /* We've seen a nodedown confchg callback, but not the
+ corresponding ringid callback. */
+ log_debug("fence work wait for cpg ringid");
+ return;
+ }
+
+ if (cluster_ringid_seq != daemon_ringid.seq) {
+ /* wait for ringids to be in sync */
+ log_debug("fence work wait for cluster ringid");
+ return;
+ }
+
+ /* poll_fencing++; */
+
+ if (cfgd_enable_quorum_fencing && !cluster_quorate) {
+ /* wait for quorum before doing any fencing, but if there
+ is none, send_fence_clear below can unblock new nodes */
+ log_debug("fence work wait for quorum");
+ goto out_fipu;
+ }
+
+ /*
+ * startup fencing
+ */
+
+ list_for_each_entry_safe(node, safe, &startup_nodes, list) {
+ if (is_clean_daemon_member(node->nodeid)) {
+ log_debug("fence startup %d member skip", node->nodeid);
+ list_del(&node->list);
+ free(node);
+ continue;
+ }
+
+ if (!cfgd_startup_fence)
+ continue;
+
+ if (monotime() - last_join_monotime < cfgd_startup_fence) {
+ log_debug("fence startup %d delay %d from %llu",
+ node->nodeid, cfgd_startup_fence,
+ (unsigned long long)last_join_monotime);
+ poll_fencing++;
+ continue;
+ }
+
+ /* clear this entry and create a daemon_nodes entry with
+ need_fencing and the fence loops below will handle it */
+
+ nodeid = node->nodeid;
+ list_del(&node->list);
+ free(node);
+
+ node = add_node_daemon(nodeid);
+ if (!node) {
+ log_debug("fence startup %d add failed", nodeid);
+ continue;
+ }
+ if (node->need_fencing) {
+ /* don't think this should happen? */
+ log_error("fence startup %d already set", nodeid);
+ continue;
+ }
+ node->need_fencing = 1;
+ node->fence_config.pos = 0;
+ node->left_reason = REASON_STARTUP_FENCING;
+ node->fail_monotime = cluster_joined_monotime - 1;
+ node->fail_walltime = cluster_joined_walltime - 1;
+ node->fence_monotime = 0;
+ node->fence_walltime = 0;
+ node->fence_request_time = 0;
+ low = set_fence_actors(node, 1);
+
+ log_debug("fence startup nodeid %d act %d", node->nodeid, low);
+ }
+
+ /*
+ * request fencing
+ */
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (!node->need_fencing)
+ continue;
+
+ if (node->fence_pid_wait)
+ continue;
+
+ if (is_clean_daemon_member(node->nodeid)) {
+ /* node rejoined cleanly, doesn't need fencing */
+ log_debug("fence request %d member skip", node->nodeid);
+ node->need_fencing = 0;
+ continue;
+ }
+
+ /* get_fence_actor picks the low nodeid that existed
+ when node failed and is still around. if the current
+ actor fails, get_fence_actor will not find it in the
+ members list, will clear it, and return the next actor */
+
+ actor = get_fence_actor(node);
+
+ if (!actor) {
+ log_error("fence request %d no actor", node->nodeid);
+ continue;
+ }
+
+ if (actor != our_nodeid) {
+ log_debug("fence request %d defer to %d",
+ node->nodeid, actor);
+ continue;
+ }
+
+ log_debug("fence request %d", node->nodeid);
+
+ rv = fence_request(node->nodeid,
+ node->fail_walltime, node->fail_monotime,
+ &node->fence_config, &pid);
+ if (rv < 0) {
+ send_fence_result(node->nodeid, rv, 0, time(NULL));
+ continue;
+ }
+
+ node->fence_pid_wait = 1;
+ node->fence_pid = pid;
+ node->fence_request_time = monotime();
+ }
+
+ /*
+ * check outstanding fence requests
+ */
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (!node->need_fencing)
+ continue;
+
+ if (!node->fence_pid_wait) {
+ /*
+ * another node is the actor, or we were actor,
+ * sent done msg and are waiting to recv it
+ */
+ log_debug("fence wait %d for done", node->nodeid);
+ continue;
+ }
+
+ if (is_clean_daemon_member(node->nodeid)) {
+ /*
+ * node has rejoined in clean state so we can
+ * abort outstanding fence op for it. all nodes
+ * will see and do this, so we don't need to send
+ * a fence result.
+ */
+ log_debug("fence wait %d member skip", node->nodeid);
+ node->need_fencing = 0;
+ node->fence_walltime = time(NULL);
+ node->fence_monotime = monotime();
+ fence_pid_cancel(node->nodeid, node->fence_pid);
+ continue;
+ }
+
+ poll_fencing++;
+
+ rv = fence_result(node->nodeid, node->fence_pid, &result);
+ if (rv == -EAGAIN) {
+ /* agent pid is still running */
+ log_debug("fence wait %d pid %d running",
+ node->nodeid, node->fence_pid);
+ continue;
+ }
+
+ if (rv < 0) {
+ /* shouldn't happen */
+ log_error("fence wait %d pid %d error %d",
+ node->nodeid, node->fence_pid_wait, rv);
+ node->fence_pid_wait = 0;
+ continue;
+ }
+
+ if (!result) {
+ /* agent exit 0, if there's another agent to run in
+ parallel, set it to run next, otherwise success */
+
+ log_debug("fence nodeid %d pid %d succeeded",
+ node->nodeid, node->fence_pid);
+
+ node->fence_pid_wait = 0;
+ node->fence_pid = 0;
+
+ rv = fence_config_next_parallel(&node->fence_config);
+ if (rv < 0)
+ send_fence_result(node->nodeid, 0, 0, time(NULL));
+ } else {
+ /* agent exit 1, if there's another agent to run at
+ next priority, set it to run next, otherwise fail */
+
+ log_debug("fence nodeid %d pid %d failed %d",
+ node->nodeid, node->fence_pid, result);
+
+ node->fence_pid_wait = 0;
+ node->fence_pid = 0;
+
+ rv = fence_config_next_priority(&node->fence_config);
+ if (rv < 0)
+ send_fence_result(node->nodeid, result, 0, time(NULL));
+ }
+ }
+
+ /*
+ * clear fence_in_progress_unknown
+ */
+ out_fipu:
+ need = nodes_need_fencing();
+
+ if (cfgd_startup_fence && fence_in_progress_unknown && !need && list_empty(&startup_nodes)) {
+ /*
+ * case A in comment above
+ * all nodes are starting and have fipu set, they all do
+ * startup fencing, and eventually see unknown nodes become
+ * members or get fenced, so all clear fipu for themselves.
+ */
+ fence_in_progress_unknown = 0;
+ log_debug("fence_in_progress_unknown 0 startup");
+ }
+
+ if (!fence_in_progress_unknown) {
+ /*
+ * case B in comment above
+ * some cur nodes have fipu clear, new nodes have fipu set.
+ * A current node needs to send_fence_clear to the new nodes
+ * once all fencing is done so they clear fipu.
+ */
+ low = 0;
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (!node->daemon_member || node->need_fence_clear)
+ continue;
+ if (!low || node->nodeid < low)
+ low = node->nodeid;
+ }
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (!node->daemon_member || !node->need_fence_clear)
+ continue;
+ if (node->nodeid == our_nodeid) {
+ node->need_fence_clear = 0;
+ continue;
+ }
+ if (low != our_nodeid)
+ continue;
+
+ flags = 0;
+
+ if (node->need_fence_clear & FR_CLEAR_STARTUP) {
+ flags |= FR_CLEAR_STARTUP;
+ node->need_fence_clear &= ~FR_CLEAR_STARTUP;
+ }
+
+ if ((node->need_fence_clear & FR_CLEAR_FIPU) && !need) {
+ flags |= FR_CLEAR_FIPU;
+ node->need_fence_clear &= ~FR_CLEAR_FIPU;
+ }
+
+ if (!flags)
+ continue;
+
+ send_fence_clear(node->nodeid, 0, flags, 0);
+ }
+ }
+
+ if (!cfgd_startup_fence && fence_in_progress_unknown) {
+ /*
+ * case C in comment above
+ * all nodes are starting and have fipu set. All expect a
+ * previous node to send_fence_clear so they can clear fipu.
+ * But there are no previous nodes. They need to detect this
+ * condition. Each node does send_fence_clear(ENODATA,FIPU).
+ * When all have received this from all, condition is
+ * detected and all clear fipu.
+ */
+ if (all_daemon_members_fipu()) {
+ fence_in_progress_unknown = 0;
+ log_debug("fence_in_progress_unknown 0 all_fipu");
+ } else if (last_join_seq > send_fipu_seq) {
+ /* the seq numbers keep us from spamming this msg */
+ send_fence_clear(our_nodeid, -ENODATA, FR_FIPU, 0);
+ log_debug("send_fence_clear %d fipu", our_nodeid);
+ send_fipu_seq = last_join_seq;
+ }
+ }
+
+ /*
+ * clean up a zombie pid from an agent we killed
+ */
+
+ if (daemon_clear_pid)
+ fence_pid_cancel(daemon_clear_nodeid, daemon_clear_pid);
+}
+
+void process_fencing_changes(void)
+{
+ poll_fencing = 0;
+ daemon_fence_work();
+}
+
+static void receive_fence_clear(struct dlm_header *hd, int len)
+{
+ struct fence_result *fr;
+ struct node_daemon *node;
+ int count;
+
+ fr = (struct fence_result *)((char *)hd + sizeof(struct dlm_header));
+
+ fr->flags = le32_to_cpu(fr->flags);
+ fr->nodeid = le32_to_cpu(fr->nodeid);
+ fr->result = le32_to_cpu(fr->result);
+ fr->fence_walltime = le64_to_cpu(fr->fence_walltime);
+
+ if (len < sizeof(struct dlm_header) + sizeof(struct fence_result)) {
+ log_error("receive_fence_clear invalid len %d from %d",
+ len, hd->nodeid);
+ return;
+ }
+
+ node = get_node_daemon(fr->nodeid);
+ if (!node) {
+ log_error("receive_fence_clear from %d no daemon node %d",
+ hd->nodeid, fr->nodeid);
+ return;
+ }
+
+ log_debug("receive_fence_clear from %d for %d result %d flags %x",
+ hd->nodeid, fr->nodeid, fr->result, fr->flags);
+
+ /*
+ * A node sends this message about itself indicating that it's in
+ * startup with fipu set. The only time we care about node->fipu
+ * is when all nodes are fipu in startup. node->need_fence_clear
+ * and node->fipu are not related, they address different cases.
+ */
+ if ((fr->result == -ENODATA) && (fr->flags & FR_FIPU)) {
+ if (!fence_in_progress_unknown)
+ return;
+
+ node->fence_in_progress_unknown = 1;
+ return;
+ }
+
+ /*
+ * An previous member sends this to new members to tell them that
+ * they can clear startup_nodes and clear fipu. These two flags
+ * may come in separate messages if there is a pending fencing op
+ * when the new member joins (CLEAR_STARTUP will come right away,
+ * but CLEAR_FIPU will come once the fencing op is done.)
+ */
+ if (!fr->result && (node->nodeid == our_nodeid)) {
+ if ((fr->flags & FR_CLEAR_STARTUP) && !list_empty(&startup_nodes)) {
+ count = clear_startup_node(0, 1);
+ log_debug("clear_startup_nodes %d", count);
+ }
+
+ if ((fr->flags & FR_CLEAR_FIPU) && fence_in_progress_unknown) {
+ fence_in_progress_unknown = 0;
+ log_debug("fence_in_progress_unknown 0 recv");
+ }
+ }
+
+ /* this node doesn't need these flags any more */
+ if (!fr->result) {
+ if (fr->flags & FR_CLEAR_STARTUP)
+ node->need_fence_clear &= ~FR_CLEAR_STARTUP;
+ if (fr->flags & FR_CLEAR_FIPU)
+ node->need_fence_clear &= ~FR_CLEAR_FIPU;
+ }
+}
+
+static void send_fence_clear(int nodeid, int result, uint32_t flags, uint64_t walltime)
+{
+ struct dlm_header *hd;
+ struct fence_result *fr;
+ char *buf;
+ int len;
+
+ len = sizeof(struct dlm_header) + sizeof(struct fence_result);
+ buf = malloc(len);
+ if (!buf) {
+ log_error("send_fence_clear no mem %d", len);
+ return;
+ }
+ memset(buf, 0, len);
+
+ hd = (struct dlm_header *)buf;
+ fr = (struct fence_result *)(buf + sizeof(*hd));
+
+ hd->type = cpu_to_le16(DLM_MSG_FENCE_CLEAR);
+ hd->nodeid = cpu_to_le32(our_nodeid);
+
+ fr->flags = cpu_to_le32(flags);
+ fr->nodeid = cpu_to_le32(nodeid);
+ fr->result = cpu_to_le32(result);
+ fr->fence_walltime = cpu_to_le64(walltime);
+
+ _send_message(cpg_handle_daemon, buf, len, DLM_MSG_FENCE_CLEAR);
+}
+
+static void receive_fence_result(struct dlm_header *hd, int len)
+{
+ struct fence_result *fr;
+ struct node_daemon *node;
+ int count;
+
+ fr = (struct fence_result *)((char *)hd + sizeof(struct dlm_header));
+
+ fr->flags = le32_to_cpu(fr->flags);
+ fr->nodeid = le32_to_cpu(fr->nodeid);
+ fr->result = le32_to_cpu(fr->result);
+ fr->fence_walltime = le64_to_cpu(fr->fence_walltime);
+
+ if (len < sizeof(struct dlm_header) + sizeof(struct fence_result)) {
+ log_error("receive_fence_result invalid len %d from %d",
+ len, hd->nodeid);
+ return;
+ }
+
+ count = clear_startup_node(fr->nodeid, 0);
+ if (count) {
+ log_debug("receive_fence_result from %d for %d clear startup",
+ hd->nodeid, fr->nodeid);
+ }
+
+ node = get_node_daemon(fr->nodeid);
+ if (!node) {
+ log_error("receive_fence_result from %d for %d no daemon node",
+ hd->nodeid, fr->nodeid);
+ return;
+ }
+
+ log_debug("receive_fence_result from %d for %d result %d walltime %llu",
+ hd->nodeid, fr->nodeid, fr->result,
+ (unsigned long long)fr->fence_walltime);
+
+ if (!node->need_fencing) {
+ /* should never happen */
+ log_error("receive_fence_result from %d for %d result %d no need_fencing",
+ hd->nodeid, fr->nodeid, fr->result);
+ return;
+ }
+
+ if (fr->result == -ECANCELED) {
+ /* if an agent pid is running, kill it and clean up */
+ if (node->fence_pid_wait && node->fence_pid)
+ fence_pid_cancel(node->nodeid, node->fence_pid);
+ fr->result = 0; /* force success below */
+ }
+
+ if (!fr->result) {
+ node->need_fencing = 0;
+ node->fence_walltime = fr->fence_walltime;
+ node->fence_monotime = monotime();
+ node->fence_actor_done = hd->nodeid;
+ } else {
+ clear_fence_actor(fr->nodeid, hd->nodeid);
+ }
+}
+
+static void send_fence_result(int nodeid, int result, uint32_t flags, uint64_t walltime)
+{
+ struct dlm_header *hd;
+ struct fence_result *fr;
+ char *buf;
+ int len;
+
+ len = sizeof(struct dlm_header) + sizeof(struct fence_result);
+ buf = malloc(len);
+ if (!buf) {
+ log_error("send_fence_result no mem %d", len);
+ return;
+ }
+ memset(buf, 0, len);
+
+ hd = (struct dlm_header *)buf;
+ fr = (struct fence_result *)(buf + sizeof(*hd));
+
+ hd->type = cpu_to_le16(DLM_MSG_FENCE_RESULT);
+ hd->nodeid = cpu_to_le32(our_nodeid);
+
+ fr->flags = cpu_to_le32(flags);
+ fr->nodeid = cpu_to_le32(nodeid);
+ fr->result = cpu_to_le32(result);
+ fr->fence_walltime = cpu_to_le64(walltime);
+
+ _send_message(cpg_handle_daemon, buf, len, DLM_MSG_FENCE_RESULT);
+}
+
+void fence_ack_node(int nodeid)
+{
+ send_fence_result(nodeid, -ECANCELED, 0, time(NULL));
+}
+
+void set_protocol_stateful(void)
+{
+ our_protocol.dr_ver.flags |= PV_STATEFUL;
+}
+
+static void pv_in(struct protocol_version *pv)
+{
+ pv->major = le16_to_cpu(pv->major);
+ pv->minor = le16_to_cpu(pv->minor);
+ pv->patch = le16_to_cpu(pv->patch);
+ pv->flags = le16_to_cpu(pv->flags);
+}
+
+static void pv_out(struct protocol_version *pv)
+{
+ pv->major = cpu_to_le16(pv->major);
+ pv->minor = cpu_to_le16(pv->minor);
+ pv->patch = cpu_to_le16(pv->patch);
+ pv->flags = cpu_to_le16(pv->flags);
+}
+
+static void protocol_in(struct protocol *proto)
+{
+ pv_in(&proto->dm_ver);
+ pv_in(&proto->km_ver);
+ pv_in(&proto->dr_ver);
+ pv_in(&proto->kr_ver);
+}
+
+static void protocol_out(struct protocol *proto)
+{
+ pv_out(&proto->dm_ver);
+ pv_out(&proto->km_ver);
+ pv_out(&proto->dr_ver);
+ pv_out(&proto->kr_ver);
+}
+
+/* go through member list saved in last confchg, see if we have received a
+ proto message from each */
+
+static int all_protocol_messages(void)
+{
+ struct node_daemon *node;
+ int i;
+
+ if (!daemon_member_count)
+ return 0;
+
+ for (i = 0; i < daemon_member_count; i++) {
+ node = get_node_daemon(daemon_member[i].nodeid);
+ if (!node) {
+ log_error("all_protocol_messages no node %d",
+ daemon_member[i].nodeid);
+ return 0;
+ }
+
+ if (!node->proto.daemon_max[0])
+ return 0;
+ }
+ return 1;
+}
+
+static int pick_min_protocol(struct protocol *proto)
+{
+ uint16_t mind[4];
+ uint16_t mink[4];
+ struct node_daemon *node;
+ int i;
+
+ memset(&mind, 0, sizeof(mind));
+ memset(&mink, 0, sizeof(mink));
+
+ /* first choose the minimum major */
+
+ for (i = 0; i < daemon_member_count; i++) {
+ node = get_node_daemon(daemon_member[i].nodeid);
+ if (!node) {
+ log_error("pick_min_protocol no node %d",
+ daemon_member[i].nodeid);
+ return -1;
+ }
+
+ if (!mind[0] || node->proto.daemon_max[0] < mind[0])
+ mind[0] = node->proto.daemon_max[0];
+
+ if (!mink[0] || node->proto.kernel_max[0] < mink[0])
+ mink[0] = node->proto.kernel_max[0];
+ }
+
+ if (!mind[0] || !mink[0]) {
+ log_error("pick_min_protocol zero major number");
+ return -1;
+ }
+
+ /* second pick the minimum minor with the chosen major */
+
+ for (i = 0; i < daemon_member_count; i++) {
+ node = get_node_daemon(daemon_member[i].nodeid);
+ if (!node)
+ continue;
+
+ if (mind[0] == node->proto.daemon_max[0]) {
+ if (!mind[1] || node->proto.daemon_max[1] < mind[1])
+ mind[1] = node->proto.daemon_max[1];
+ }
+
+ if (mink[0] == node->proto.kernel_max[0]) {
+ if (!mink[1] || node->proto.kernel_max[1] < mink[1])
+ mink[1] = node->proto.kernel_max[1];
+ }
+ }
+
+ if (!mind[1] || !mink[1]) {
+ log_error("pick_min_protocol zero minor number");
+ return -1;
+ }
+
+ /* third pick the minimum patch with the chosen major.minor */
+
+ for (i = 0; i < daemon_member_count; i++) {
+ node = get_node_daemon(daemon_member[i].nodeid);
+ if (!node)
+ continue;
+
+ if (mind[0] == node->proto.daemon_max[0] &&
+ mind[1] == node->proto.daemon_max[1]) {
+ if (!mind[2] || node->proto.daemon_max[2] < mind[2])
+ mind[2] = node->proto.daemon_max[2];
+ }
+
+ if (mink[0] == node->proto.kernel_max[0] &&
+ mink[1] == node->proto.kernel_max[1]) {
+ if (!mink[2] || node->proto.kernel_max[2] < mink[2])
+ mink[2] = node->proto.kernel_max[2];
+ }
+ }
+
+ if (!mind[2] || !mink[2]) {
+ log_error("pick_min_protocol zero patch number");
+ return -1;
+ }
+
+ memcpy(&proto->daemon_run, &mind, sizeof(mind));
+ memcpy(&proto->kernel_run, &mink, sizeof(mink));
+ return 0;
+}
+
+static void receive_protocol(struct dlm_header *hd, int len)
+{
+ struct protocol *p;
+ struct node_daemon *node;
+ int new = 0;
+
+ p = (struct protocol *)((char *)hd + sizeof(struct dlm_header));
+ protocol_in(p);
+
+ if (len < sizeof(struct dlm_header) + sizeof(struct protocol)) {
+ log_error("receive_protocol invalid len %d from %d",
+ len, hd->nodeid);
+ return;
+ }
+
+ /* zero is an invalid version value */
+
+ if (!p->daemon_max[0] || !p->daemon_max[1] || !p->daemon_max[2] ||
+ !p->kernel_max[0] || !p->kernel_max[1] || !p->kernel_max[2]) {
+ log_error("receive_protocol invalid max value from %d "
+ "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
+ p->daemon_max[0], p->daemon_max[1], p->daemon_max[2],
+ p->kernel_max[0], p->kernel_max[1], p->kernel_max[2]);
+ return;
+ }
+
+ /* the run values will be zero until a version is set, after
+ which none of the run values can be zero */
+
+ if (p->daemon_run[0] && (!p->daemon_run[1] || !p->daemon_run[2] ||
+ !p->kernel_run[0] || !p->kernel_run[1] || !p->kernel_run[2])) {
+ log_error("receive_protocol invalid run value from %d "
+ "daemon %u.%u.%u kernel %u.%u.%u", hd->nodeid,
+ p->daemon_run[0], p->daemon_run[1], p->daemon_run[2],
+ p->kernel_run[0], p->kernel_run[1], p->kernel_run[2]);
+ return;
+ }
+
+ /* save this node's proto so we can tell when we've got all, and
+ use it to select a minimum protocol from all */
+
+ node = get_node_daemon(hd->nodeid);
+ if (!node) {
+ log_error("receive_protocol no node %d", hd->nodeid);
+ return;
+ }
+
+ if (!node->daemon_member) {
+ log_error("receive_protocol node %d not member", hd->nodeid);
+ return;
+ }
+
+ log_debug("receive_protocol %d max %u.%u.%u.%x run %u.%u.%u.%x",
+ hd->nodeid,
+ p->daemon_max[0], p->daemon_max[1],
+ p->daemon_max[2], p->daemon_max[3],
+ p->daemon_run[0], p->daemon_run[1],
+ p->daemon_run[2], p->daemon_run[3]);
+
+ if (memcmp(&node->proto, p, sizeof(struct protocol))) {
+ log_debug("daemon node %d prot max %u.%u.%u.%x run %u.%u.%u.%x",
+ hd->nodeid,
+ node->proto.daemon_max[0], node->proto.daemon_max[1],
+ node->proto.daemon_max[2], node->proto.daemon_max[3],
+ node->proto.daemon_run[0], node->proto.daemon_run[1],
+ node->proto.daemon_run[2], node->proto.daemon_run[3]);
+ new = 1;
+ }
+
+ /* checking zero node->daemon_max[0] is a way to tell if we've received
+ an acceptable (non-stateful) proto message from the node since we
+ saw it join the daemon cpg */
+
+ if (hd->nodeid != our_nodeid &&
+ !node->proto.daemon_max[0] &&
+ (p->dr_ver.flags & PV_STATEFUL) &&
+ (our_protocol.dr_ver.flags & PV_STATEFUL)) {
+
+ log_debug("daemon node %d stateful merge", hd->nodeid);
+ log_debug("daemon node %d join %llu left %llu local quorum %llu",
+ hd->nodeid,
+ (unsigned long long)node->daemon_add_time,
+ (unsigned long long)node->daemon_rem_time,
+ (unsigned long long)cluster_quorate_monotime);
+
+ if (cluster_quorate && node->daemon_rem_time &&
+ cluster_quorate_monotime < node->daemon_rem_time) {
+ log_debug("daemon node %d kill due to stateful merge", hd->nodeid);
+ if (!node->killed)
+ kick_node_from_cluster(hd->nodeid);
+ node->killed = 1;
+ }
+
+ /* don't save p->proto into node->proto; we need to come
+ through here based on zero daemon_max[0] for other proto
+ messages like this one from the same node */
+
+ return;
+ }
+
+ if (new) {
+ memcpy(&node->proto, p, sizeof(struct protocol));
+
+ log_debug("daemon node %d save max %u.%u.%u.%x run %u.%u.%u.%x",
+ node->nodeid,
+ node->proto.daemon_max[0], node->proto.daemon_max[1],
+ node->proto.daemon_max[2], node->proto.daemon_max[3],
+ node->proto.daemon_run[0], node->proto.daemon_run[1],
+ node->proto.daemon_run[2], node->proto.daemon_run[3]);
+ }
+
+ /* if we have zero run values, and this msg has non-zero run values,
+ then adopt them as ours; otherwise save this proto message */
+
+ if (our_protocol.daemon_run[0])
+ return;
+
+ if (p->daemon_run[0]) {
+ our_protocol.daemon_run[0] = p->daemon_run[0];
+ our_protocol.daemon_run[1] = p->daemon_run[1];
+ our_protocol.daemon_run[2] = p->daemon_run[2];
+
+ our_protocol.kernel_run[0] = p->kernel_run[0];
+ our_protocol.kernel_run[1] = p->kernel_run[1];
+ our_protocol.kernel_run[2] = p->kernel_run[2];
+
+ log_debug("run protocol from nodeid %d", hd->nodeid);
+ }
+}
+
+static void send_protocol(struct protocol *proto)
+{
+ struct dlm_header *hd;
+ struct protocol *pr;
+ char *buf;
+ int len;
+
+ len = sizeof(struct dlm_header) + sizeof(struct protocol);
+ buf = malloc(len);
+ if (!buf) {
+ log_error("send_protocol no mem %d", len);
+ return;
+ }
+ memset(buf, 0, len);
+
+ hd = (struct dlm_header *)buf;
+ pr = (struct protocol *)(buf + sizeof(*hd));
+
+ hd->type = cpu_to_le16(DLM_MSG_PROTOCOL);
+ hd->nodeid = cpu_to_le32(our_nodeid);
+
+ memcpy(pr, proto, sizeof(struct protocol));
+ protocol_out(pr);
+
+ _send_message(cpg_handle_daemon, buf, len, DLM_MSG_PROTOCOL);
+}
+
+int set_protocol(void)
+{
+ struct protocol proto;
+ struct pollfd pollfd;
+ int sent_proposal = 0;
+ int rv;
+
+ memset(&pollfd, 0, sizeof(pollfd));
+ pollfd.fd = cpg_fd_daemon;
+ pollfd.events = POLLIN;
+
+ while (1) {
+ if (our_protocol.daemon_run[0])
+ break;
+
+ if (!sent_proposal && all_protocol_messages()) {
+ /* propose a protocol; look through info from all
+ nodes and pick the min for both daemon and kernel,
+ and propose that */
+
+ sent_proposal = 1;
+
+ /* copy our max values */
+ memcpy(&proto, &our_protocol, sizeof(struct protocol));
+
+ rv = pick_min_protocol(&proto);
+ if (rv < 0)
+ return rv;
+
+ log_debug("set_protocol member_count %d propose "
+ "daemon %u.%u.%u kernel %u.%u.%u",
+ daemon_member_count,
+ proto.daemon_run[0], proto.daemon_run[1],
+ proto.daemon_run[2], proto.kernel_run[0],
+ proto.kernel_run[1], proto.kernel_run[2]);
+
+ send_protocol(&proto);
+ }
+
+ /* only process messages/events from daemon cpg until protocol
+ is established */
+
+ rv = poll(&pollfd, 1, -1);
+ if (rv == -1 && errno == EINTR) {
+ if (daemon_quit)
+ return -1;
+ continue;
+ }
+ if (rv < 0) {
+ log_error("set_protocol poll errno %d", errno);
+ return -1;
+ }
+
+ if (pollfd.revents & POLLIN)
+ process_cpg_daemon(0);
+ if (pollfd.revents & (POLLERR | POLLHUP | POLLNVAL)) {
+ log_error("set_protocol poll revents %u",
+ pollfd.revents);
+ return -1;
+ }
+ }
+
+ if (our_protocol.daemon_run[0] != our_protocol.daemon_max[0] ||
+ our_protocol.daemon_run[1] > our_protocol.daemon_max[1]) {
+ log_error("incompatible daemon protocol run %u.%u.%u max %u.%u.%u",
+ our_protocol.daemon_run[0],
+ our_protocol.daemon_run[1],
+ our_protocol.daemon_run[2],
+ our_protocol.daemon_max[0],
+ our_protocol.daemon_max[1],
+ our_protocol.daemon_max[2]);
+ return -1;
+ }
+
+ if (our_protocol.kernel_run[0] != our_protocol.kernel_max[0] ||
+ our_protocol.kernel_run[1] > our_protocol.kernel_max[1]) {
+ log_error("incompatible kernel protocol run %u.%u.%u max %u.%u.%u",
+ our_protocol.kernel_run[0],
+ our_protocol.kernel_run[1],
+ our_protocol.kernel_run[2],
+ our_protocol.kernel_max[0],
+ our_protocol.kernel_max[1],
+ our_protocol.kernel_max[2]);
+ return -1;
+ }
+
+ log_debug("daemon run %u.%u.%u max %u.%u.%u "
+ "kernel run %u.%u.%u max %u.%u.%u",
+ our_protocol.daemon_run[0],
+ our_protocol.daemon_run[1],
+ our_protocol.daemon_run[2],
+ our_protocol.daemon_max[0],
+ our_protocol.daemon_max[1],
+ our_protocol.daemon_max[2],
+ our_protocol.kernel_run[0],
+ our_protocol.kernel_run[1],
+ our_protocol.kernel_run[2],
+ our_protocol.kernel_max[0],
+ our_protocol.kernel_max[1],
+ our_protocol.kernel_max[2]);
+
+ send_protocol(&our_protocol);
+ return 0;
+}
+
+static void deliver_cb_daemon(cpg_handle_t handle,
+ const struct cpg_name *group_name,
+ uint32_t nodeid, uint32_t pid,
+ void *data, size_t len)
+{
+ struct dlm_header *hd;
+
+ if (len < sizeof(*hd)) {
+ log_error("deliver_cb short message %zd", len);
+ return;
+ }
+
+ hd = (struct dlm_header *)data;
+ dlm_header_in(hd);
+
+ switch (hd->type) {
+ case DLM_MSG_PROTOCOL:
+ receive_protocol(hd, len);
+ break;
+ case DLM_MSG_FENCE_RESULT:
+ receive_fence_result(hd, len);
+ break;
+ case DLM_MSG_FENCE_CLEAR:
+ receive_fence_clear(hd, len);
+ break;
+ default:
+ log_error("deliver_cb_daemon unknown msg type %d", hd->type);
+ }
+
+ daemon_fence_work();
+}
+
+static void confchg_cb_daemon(cpg_handle_t handle,
+ const struct cpg_name *group_name,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries)
+{
+ struct node_daemon *node;
+ uint64_t now, now_wall;
+ int nodedown = 0, procdown = 0, leave = 0;
+ int we_joined = 0;
+ int i, reason, low;
+
+ now = monotime();
+ now_wall = time(NULL);
+
+ log_config(group_name, member_list, member_list_entries,
+ left_list, left_list_entries,
+ joined_list, joined_list_entries);
+
+ memset(&daemon_member, 0, sizeof(daemon_member));
+ daemon_member_count = member_list_entries;
+
+ for (i = 0; i < member_list_entries; i++) {
+ daemon_member[i] = member_list[i];
+ /* add struct for nodes we've not seen before */
+ add_node_daemon(member_list[i].nodeid);
+ }
+
+ memset(&daemon_joined, 0, sizeof(daemon_joined));
+ daemon_joined_count = joined_list_entries;
+
+ for (i = 0; i < joined_list_entries; i++) {
+ daemon_joined[i] = joined_list[i];
+ if (joined_list[i].nodeid == our_nodeid)
+ we_joined = 1;
+ }
+
+ memset(&daemon_remove, 0, sizeof(daemon_remove));
+ daemon_remove_count = left_list_entries;
+
+ for (i = 0; i < left_list_entries; i++) {
+ daemon_remove[i] = left_list[i];
+
+ if (left_list[i].reason == CPG_REASON_NODEDOWN)
+ nodedown++;
+ else if (left_list[i].reason == CPG_REASON_PROCDOWN)
+ procdown++;
+ else if (left_list[i].reason == CPG_REASON_LEAVE)
+ leave++;
+ }
+
+ if (nodedown || procdown || leave)
+ log_debug("%s left nodedown %d procdown %d leave %d",
+ group_name->value, nodedown, procdown, leave);
+
+ if (nodedown)
+ daemon_ringid_wait = 1;
+
+ if (joined_list_entries)
+ send_protocol(&our_protocol);
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (in_daemon_list(node->nodeid, daemon_member, daemon_member_count)) {
+ if (node->daemon_member)
+ continue;
+
+ /* node joined daemon cpg */
+ node->daemon_member = 1;
+ node->daemon_add_time = now;
+
+ last_join_monotime = now;
+ last_join_seq++;
+
+ /* a joining node shows prev members in joined list */
+ if (!we_joined)
+ node->need_fence_clear = FR_CLEAR_STARTUP|FR_CLEAR_FIPU;
+
+ if (node->need_fencing) {
+ /* need_fencing will be cleared if we accept a
+ valid proto from it */
+ log_error("daemon new nodeid %d needs fencing",
+ node->nodeid);
+ }
+
+ } else {
+ if (!node->daemon_member)
+ continue;
+
+ /* node left daemon cpg */
+ node->daemon_member = 0;
+ node->killed = 0;
+ memset(&node->proto, 0, sizeof(struct protocol));
+ node->daemon_rem_time = now;
+
+ /* tell loop below to look at this node */
+ node->recover_setup = 1;
+ }
+ }
+
+ /* set up recovery work for nodes that just failed */
+
+ /* TODO: limit to nodes with a valid proto?
+ * node_history_lockspace_fail() would only set
+ * need_fencing if node->start_time was non-zero. */
+
+ list_for_each_entry(node, &daemon_nodes, list) {
+ if (!node->recover_setup)
+ continue;
+
+ node->recover_setup = 0;
+ reason = 0;
+ low = 0;
+
+ if (!cfgd_enable_fencing)
+ continue;
+
+ if (node->need_fencing) {
+ log_error("daemon remove nodeid %d already needs fencing",
+ node->nodeid);
+ continue;
+ }
+
+ for (i = 0; i < left_list_entries; i++) {
+ if (left_list[i].nodeid != node->nodeid)
+ continue;
+ reason = left_list[i].reason;
+ break;
+ }
+
+ if (reason == CPG_REASON_NODEDOWN || reason == CPG_REASON_PROCDOWN) {
+ node->need_fencing = 1;
+ node->fence_config.pos = 0;
+ node->left_reason = reason;
+ node->fail_monotime = now;
+ node->fail_walltime = now_wall;
+ node->fence_monotime = 0;
+ node->fence_walltime = 0;
+ node->fence_request_time = 0;
+ low = set_fence_actors(node, 0);
+ }
+
+ log_debug("daemon remove %d %s need_fencing %d low %d",
+ node->nodeid, reason_str(reason), node->need_fencing, low);
+ }
+
+ daemon_fence_work();
+}
+
+static void totem_cb_daemon(cpg_handle_t handle,
+ struct cpg_ring_id ring_id,
+ uint32_t member_list_entries,
+ const uint32_t *member_list)
+{
+ daemon_ringid.nodeid = ring_id.nodeid;
+ daemon_ringid.seq = ring_id.seq;
+ daemon_ringid_wait = 0;
+
+ log_ringid("dlm:controld", &ring_id, member_list, member_list_entries);
+
+ daemon_fence_work();
+}
+
+static cpg_model_v1_data_t cpg_callbacks_daemon = {
+ .cpg_deliver_fn = deliver_cb_daemon,
+ .cpg_confchg_fn = confchg_cb_daemon,
+ .cpg_totem_confchg_fn = totem_cb_daemon,
+ .flags = CPG_MODEL_V1_DELIVER_INITIAL_TOTEM_CONF,
+};
+
+void process_cpg_daemon(int ci)
+{
+ cs_error_t error;
+
+ error = cpg_dispatch(cpg_handle_daemon, CS_DISPATCH_ALL);
+ if (error != CS_OK)
+ log_error("daemon cpg_dispatch error %d", error);
+}
+
+int setup_cpg_daemon(void)
+{
+ cs_error_t error;
+ struct cpg_name name;
+ int i = 0;
+
+ /* daemon 1.1.1 was cluster3/STABLE3/RHEL6 which is incompatible
+ with cluster4/RHEL7 */
+
+ memset(&our_protocol, 0, sizeof(our_protocol));
+
+ if (cfgd_enable_fscontrol)
+ our_protocol.daemon_max[0] = 2;
+ else
+ our_protocol.daemon_max[0] = 3;
+
+ our_protocol.daemon_max[1] = 1;
+ our_protocol.daemon_max[2] = 1;
+ our_protocol.kernel_max[0] = 1;
+ our_protocol.kernel_max[1] = 1;
+ our_protocol.kernel_max[2] = 1;
+
+ error = cpg_model_initialize(&cpg_handle_daemon, CPG_MODEL_V1,
+ (cpg_model_data_t *)&cpg_callbacks_daemon,
+ NULL);
+ if (error != CS_OK) {
+ log_error("daemon cpg_initialize error %d", error);
+ return -1;
+ }
+
+ cpg_fd_get(cpg_handle_daemon, &cpg_fd_daemon);
+
+ memset(&name, 0, sizeof(name));
+ sprintf(name.value, "dlm:controld");
+ name.length = strlen(name.value) + 1;
+
+ log_debug("cpg_join %s ...", name.value);
+ retry:
+ error = cpg_join(cpg_handle_daemon, &name);
+ if (error == CS_ERR_TRY_AGAIN) {
+ sleep(1);
+ if (!(++i % 10))
+ log_error("daemon cpg_join error retrying");
+ goto retry;
+ }
+ if (error != CS_OK) {
+ log_error("daemon cpg_join error %d", error);
+ goto fail;
+ }
+
+ log_debug("setup_cpg_daemon %d", cpg_fd_daemon);
+ return cpg_fd_daemon;
+
+ fail:
+ cpg_finalize(cpg_handle_daemon);
+ return -1;
+}
+
+void close_cpg_daemon(void)
+{
+ struct lockspace *ls;
+ cs_error_t error;
+ struct cpg_name name;
+ int i = 0;
+
+ if (!cpg_handle_daemon)
+ return;
+ if (cluster_down)
+ goto fin;
+
+ memset(&name, 0, sizeof(name));
+ sprintf(name.value, "dlm:controld");
+ name.length = strlen(name.value) + 1;
+
+ log_debug("cpg_leave %s ...", name.value);
+ retry:
+ error = cpg_leave(cpg_handle_daemon, &name);
+ if (error == CS_ERR_TRY_AGAIN) {
+ sleep(1);
+ if (!(++i % 10))
+ log_error("daemon cpg_leave error retrying");
+ goto retry;
+ }
+ if (error != CS_OK)
+ log_error("daemon cpg_leave error %d", error);
+ fin:
+ list_for_each_entry(ls, &lockspaces, list) {
+ if (ls->cpg_handle)
+ cpg_finalize(ls->cpg_handle);
+ }
+ cpg_finalize(cpg_handle_daemon);
+}
+
+void init_daemon(void)
+{
+ INIT_LIST_HEAD(&daemon_nodes);
+ INIT_LIST_HEAD(&startup_nodes);
+
+}
diff --git a/dlm_controld/dlm_controld.h b/dlm_controld/dlm_controld.h
index 638b0de..96b425c 100644
--- a/dlm_controld/dlm_controld.h
+++ b/dlm_controld/dlm_controld.h
@@ -29,6 +29,7 @@
#define DLMC_CMD_FS_NOTIFIED 9
#define DLMC_CMD_DEADLOCK_CHECK 10
#define DLMC_CMD_DUMP_LOG_PLOCK 11
+#define DLMC_CMD_FENCE_ACK 12
struct dlmc_header {
unsigned int magic;
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 5fca041..5e83db3 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -9,16 +9,18 @@
#ifndef __DLM_DAEMON_DOT_H__
#define __DLM_DAEMON_DOT_H__
-#include <sys/types.h>
#include <asm/types.h>
+#include <sys/types.h>
#include <sys/uio.h>
-#include <netinet/in.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/utsname.h>
#include <sys/poll.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <netinet/in.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <net/if.h>
@@ -37,7 +39,6 @@
#include <syslog.h>
#include <sched.h>
#include <signal.h>
-#include <sys/time.h>
#include <dirent.h>
#include <corosync/cpg.h>
@@ -45,6 +46,7 @@
#include <linux/dlmconstants.h>
#include "libdlmcontrol.h"
#include "dlm_controld.h"
+#include "fence_config.h"
#include "list.h"
#include "rbtree.h"
#include "linux_endian.h"
@@ -60,9 +62,9 @@
/* TODO: get CONFDIR, LOGDIR, RUNDIR from build */
-#define RUNDIR "/var/run/cluster"
-#define LOGDIR "/var/log/cluster"
-#define CONFDIR "/etc"
+#define RUNDIR "/var/run/dlm"
+#define LOGDIR "/var/log/dlm"
+#define CONFDIR "/etc/dlm"
#define RUN_FILE_NAME "dlm_controld.pid"
#define LOG_FILE_NAME "dlm_controld.log"
@@ -80,7 +82,8 @@
#define DEFAULT_DEBUG_LOGFILE 0
#define DEFAULT_ENABLE_FENCING 1
-#define DEFAULT_ENABLE_QUORUM 0
+#define DEFAULT_ENABLE_QUORUM_FENCING 1
+#define DEFAULT_ENABLE_QUORUM_LOCKSPACE 0
#define DEFAULT_ENABLE_FSCONTROL 0
#define DEFAULT_ENABLE_PLOCK 1
#define DEFAULT_PLOCK_DEBUG 0
@@ -89,6 +92,8 @@
#define DEFAULT_DROP_RESOURCES_TIME 10000 /* 10 sec */
#define DEFAULT_DROP_RESOURCES_COUNT 10
#define DEFAULT_DROP_RESOURCES_AGE 10000 /* 10 sec */
+#define DEFAULT_FENCE_ALL_AGENT "dlm_stonith"
+#define DEFAULT_STARTUP_FENCE 30
/* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h.
Copied in libdlm.h so apps don't need to include the kernel header.
@@ -118,9 +123,8 @@
EXTERN int daemon_debug_opt;
EXTERN int daemon_quit;
EXTERN int cluster_down;
-EXTERN int poll_ringid;
+EXTERN int poll_lockspaces;
EXTERN int poll_fencing;
-EXTERN int poll_quorum;
EXTERN int poll_fs;
EXTERN int poll_ignore_plock;
EXTERN int poll_drop_plock;
@@ -128,20 +132,24 @@ EXTERN int plock_fd;
EXTERN int plock_ci;
EXTERN struct list_head lockspaces;
EXTERN int cluster_quorate;
-EXTERN uint64_t quorate_time;
+EXTERN uint64_t cluster_quorate_monotime;
+EXTERN uint64_t cluster_joined_monotime;
+EXTERN uint64_t cluster_joined_walltime;
EXTERN uint32_t cluster_ringid_seq;
EXTERN char cluster_name[DLM_LOCKSPACE_LEN+1];
EXTERN int our_nodeid;
EXTERN uint32_t control_minor;
EXTERN uint32_t monitor_minor;
EXTERN uint32_t plock_minor;
+EXTERN struct fence_device fence_all_device;
EXTERN int optk_debug;
EXTERN int optk_timewarn;
EXTERN int optk_protocol;
EXTERN int optd_debug_logfile;
EXTERN int optd_enable_fencing;
-EXTERN int optd_enable_quorum;
+EXTERN int optd_enable_quorum_fencing;
+EXTERN int optd_enable_quorum_lockspace;
EXTERN int optd_enable_fscontrol;
EXTERN int optd_enable_plock;
EXTERN int optd_plock_debug;
@@ -150,13 +158,16 @@ EXTERN int optd_plock_ownership;
EXTERN int optd_drop_resources_time;
EXTERN int optd_drop_resources_count;
EXTERN int optd_drop_resources_age;
+EXTERN int optd_startup_fence;
+EXTERN int optd_fence_all_agent;
EXTERN int cfgk_debug;
EXTERN int cfgk_timewarn;
EXTERN int cfgk_protocol;
EXTERN int cfgd_debug_logfile;
EXTERN int cfgd_enable_fencing;
-EXTERN int cfgd_enable_quorum;
+EXTERN int cfgd_enable_quorum_fencing;
+EXTERN int cfgd_enable_quorum_lockspace;
EXTERN int cfgd_enable_fscontrol;
EXTERN int cfgd_enable_plock;
EXTERN int cfgd_plock_debug;
@@ -165,6 +176,8 @@ EXTERN int cfgd_plock_ownership;
EXTERN int cfgd_drop_resources_time;
EXTERN int cfgd_drop_resources_count;
EXTERN int cfgd_drop_resources_age;
+EXTERN int cfgd_startup_fence;
+EXTERN char fence_all_agent[PATH_MAX];
#define LOG_DUMP_SIZE DLMC_DUMP_SIZE
@@ -194,7 +207,9 @@ enum {
DLM_MSG_DEADLK_CYCLE_START,
DLM_MSG_DEADLK_CYCLE_END,
DLM_MSG_DEADLK_CHECKPOINT_READY,
- DLM_MSG_DEADLK_CANCEL_LOCK
+ DLM_MSG_DEADLK_CANCEL_LOCK,
+ DLM_MSG_FENCE_RESULT,
+ DLM_MSG_FENCE_CLEAR,
};
/* dlm_header flags */
@@ -288,15 +303,10 @@ void setup_config(int update);
int get_weight(int nodeid, char *lockspace);
/* cpg.c */
-int setup_cpg_daemon(void);
-void close_cpg_daemon(void);
-void process_cpg_daemon(int ci);
-int set_protocol(void);
void process_lockspace_changes(void);
-void dlm_send_message(struct lockspace *ls, char *buf, int len);
+void process_fencing_changes(void);
int dlm_join_lockspace(struct lockspace *ls);
int dlm_leave_lockspace(struct lockspace *ls);
-const char *msg_name(int type);
void update_flow_control_status(void);
int set_node_info(struct lockspace *ls, int nodeid, struct dlmc_node *node);
int set_lockspace_info(struct lockspace *ls, struct dlmc_lockspace *lockspace);
@@ -305,6 +315,36 @@ int set_lockspace_nodes(struct lockspace *ls, int option, int *node_count,
struct dlmc_node **nodes_out);
int set_fs_notified(struct lockspace *ls, int nodeid);
+/* daemon_cpg.c */
+void init_daemon(void);
+void fence_ack_node(int nodeid);
+void add_startup_node(int nodeid);
+const char *reason_str(int reason);
+const char *msg_name(int type);
+void dlm_send_message(struct lockspace *ls, char *buf, int len);
+void dlm_header_in(struct dlm_header *hd);
+int dlm_header_validate(struct dlm_header *hd, int nodeid);
+int fence_node_time(int nodeid, uint64_t *last_fenced);
+int fence_in_progress(int *in_progress);
+int setup_cpg_daemon(void);
+void close_cpg_daemon(void);
+void process_cpg_daemon(int ci);
+void set_protocol_stateful(void);
+int set_protocol(void);
+
+void log_config(const struct cpg_name *group_name,
+ const struct cpg_address *member_list,
+ size_t member_list_entries,
+ const struct cpg_address *left_list,
+ size_t left_list_entries,
+ const struct cpg_address *joined_list,
+ size_t joined_list_entries);
+
+void log_ringid(const char *name,
+ struct cpg_ring_id *ringid,
+ const uint32_t *member_list,
+ size_t member_list_entries);
+
/* deadlock.c */
void setup_deadlock(void);
void send_cycle_start(struct lockspace *ls);
@@ -346,11 +386,12 @@ int setup_cluster_cfg(void);
void close_cluster_cfg(void);
void process_cluster_cfg(int ci);
void kick_node_from_cluster(int nodeid);
+int setup_node_config(void);
/* fence.c */
-int fence_request(int nodeid);
-int fence_node_time(int nodeid, uint64_t *last_fenced_time);
-int fence_in_progress(int *count);
+int fence_request(int nodeid, uint64_t fail_walltime, uint64_t fail_monotime,
+ struct fence_config *fc, int *pid_out);
+int fence_result(int nodeid, int pid, int *result);
/* netlink.c */
int setup_netlink(void);
diff --git a/dlm_controld/fence.c b/dlm_controld/fence.c
index 50b7d4d..91c5e67 100644
--- a/dlm_controld/fence.c
+++ b/dlm_controld/fence.c
@@ -7,28 +7,147 @@
*/
#include "dlm_daemon.h"
-#include <pacemaker/crm/stonith-ng.h>
-int fence_request(int nodeid)
+static int run_agent(char *agent, char *args, int *pid_out)
{
- int rv;
- rv = stonith_api_kick_helper(nodeid, 300, 1);
- if (rv) {
- log_error("stonith_api_kick_helper %d error %d", nodeid, rv);
- return rv;
+ int pid, len;
+ int pw_fd = -1; /* parent write file descriptor */
+ int cr_fd = -1; /* child read file descriptor */
+ int pfd[2];
+
+ len = strlen(args);
+
+ if (pipe(pfd))
+ return -errno;
+
+ cr_fd = pfd[0];
+ pw_fd = pfd[1];
+
+ pid = fork();
+ if (pid < 0) {
+ close(cr_fd);
+ close(pw_fd);
+ return -errno;
}
- return 0;
+
+ if (pid) {
+ /* parent */
+ int ret;
+
+ do {
+ ret = write(pw_fd, args, len);
+ } while (ret < 0 && errno == EINTR);
+
+ if (ret != len)
+ goto fail;
+
+ close(cr_fd);
+ close(pw_fd);
+
+ *pid_out = pid;
+ return 0;
+ } else {
+ /* child */
+ int c_stdout, c_stderr;
+
+ /* redirect agent stdout/stderr to /dev/null */
+ close(1);
+ c_stdout = open("/dev/null", O_WRONLY);
+ if (c_stdout < 0)
+ goto fail;
+ close(2);
+ c_stderr = open("/dev/null", O_WRONLY);
+ if (c_stderr < 0)
+ goto fail;
+
+ /* redirect agent stdin from parent */
+ close(0);
+ if (dup(cr_fd) < 0)
+ goto fail;
+
+ close(cr_fd);
+ close(pw_fd);
+
+ execlp(agent, agent, NULL);
+ exit(EXIT_FAILURE);
+ }
+ fail:
+ close(cr_fd);
+ close(pw_fd);
+ return -1;
}
-int fence_node_time(int nodeid, uint64_t *last_fenced_time)
+int fence_request(int nodeid, uint64_t fail_walltime, uint64_t fail_monotime,
+ struct fence_config *fc, int *pid_out)
{
- *last_fenced_time = stonith_api_time_helper(nodeid, 0);
+ struct fence_device *dev;
+ char args[FENCE_CONFIG_ARGS_MAX];
+ char extra[FENCE_CONFIG_NAME_MAX];
+ int rv, pid = -1;
+
+ memset(args, 0, sizeof(args));
+
+ memset(extra, 0, sizeof(extra));
+ snprintf(extra, sizeof(extra)-1, "fail_time=%llu\n", (unsigned long long)fail_walltime);
+
+ dev = fc->dev[fc->pos];
+ if (!dev)
+ return -1;
+
+ rv = fence_config_agent_args(fc, extra, args);
+ if (rv < 0) {
+ log_error("fence_request %d args error %d", nodeid, rv);
+ return rv;
+ }
+
+ rv = run_agent(dev->agent, args, &pid);
+ if (rv < 0) {
+ log_error("fence_request %d agent %s pid %d run error %d",
+ nodeid, dev->agent, pid, rv);
+ return rv;
+ }
+
+ log_debug("fence_request %d pos %d agent %s pid %d running",
+ nodeid, fc->pos, dev->agent, pid);
+
+ *pid_out = pid;
return 0;
}
-int fence_in_progress(int *count)
+/*
+ * if pid has exited, return 0 with exit code in result
+ * if pid is running, return -EAGAIN
+ * other error, return -EXXX
+ */
+
+int fence_result(int nodeid, int pid, int *result)
{
- *count = 0;
- return 0;
+ int status, rv;
+
+ rv = waitpid(pid, &status, WNOHANG);
+
+ if (rv < 0) {
+ /* shouldn't happen */
+ log_error("agent pid %d nodeid %d errno %d",
+ pid, nodeid, errno);
+ return rv;
+
+ } else if (!rv) {
+ /* pid still running */
+ return -EAGAIN;
+
+ } else if (WIFEXITED(status)) {
+ /* pid exited */
+
+ *result = WEXITSTATUS(status);
+
+ log_error("agent pid %d nodeid %d result %d",
+ pid, nodeid, *result);
+ return 0;
+
+ } else {
+ /* pid state changed but still running */
+ return -EAGAIN;
+ }
}
diff --git a/dlm_controld/fence_config.c b/dlm_controld/fence_config.c
new file mode 100644
index 0000000..9f72e36
--- /dev/null
+++ b/dlm_controld/fence_config.c
@@ -0,0 +1,403 @@
+/*
+ * Copyright 2012 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+
+#include "fence_config.h"
+
+#if 0
+
+Empty new line separates the config for each fence device.
+
+-
+
+fence_all fence_foo key=val ...
+
+Special fence config format that applies to all nodes, allows
+no per node config parameters, and cannot be used with any
+other fence device configuration.
+
+-
+
+device <dev_name> <agent> <dev_args>
+connect <dev_name> node=<nodeid> <con_args>
+
+General fence config format, allowing per node config
+parameters.
+
+-
+
+device foo fence_foo ipaddr=1.1.1.1 login=x password=y
+connect foo node=1 port=1
+connect foo node=2 port=2
+connect foo node=3 port=3
+
+Simple example of nodes connected to switch ports.
+If fencing with the device fails, the next device
+listed for the node, if any, will be tried.
+
+-
+
+device foo:1 fence_foo ipaddr=1.1.1.1 login=x password=y
+connect foo:1 node=1 port=1
+connect foo:1 node=2 port=2
+connect foo:1 node=3 port=3
+
+device foo:2 fence_foo ipaddr=2.2.2.2 login=x password=y
+connect foo:2 node=1 port=1
+connect foo:2 node=2 port=2
+connect foo:2 node=3 port=3
+
+Associate two parallel path/power devices that must both
+succeed for fencing to succeed. Devices have same base
+name with :1 :2 suffix.
+
+-
+
+device foo fence_foo ipaddr=1.1.1.1 login=x password=y
+connect foo node=1 port=1
+connect foo node=2 port=2
+connect foo node=3 port=3
+unfence foo
+
+Add unfence line to indicate nodes connected to the device
+should be unfenced.
+
+#endif
+
+#define MAX_LINE (FENCE_CONFIG_ARGS_MAX + (3 * FENCE_CONFIG_NAME_MAX))
+
+static unsigned int con_args_nodeid(char *args)
+{
+ char *k;
+ unsigned int v;
+ int rv;
+
+ k = strstr(args, "node=");
+
+ rv = sscanf(k, "node=%u", &v);
+ if (rv != 1)
+ return 0;
+ return v;
+}
+
+static int read_config_section(unsigned int nodeid, FILE *file, char *dev_line,
+ struct fence_device **dev_out,
+ struct fence_connect **con_out)
+{
+ struct fence_device *dev;
+ struct fence_connect *con;
+ char line[MAX_LINE];
+ char unused[FENCE_CONFIG_NAME_MAX];
+ char agent[FENCE_CONFIG_NAME_MAX];
+ char dev_name[FENCE_CONFIG_NAME_MAX];
+ char con_name[FENCE_CONFIG_NAME_MAX];
+ char dev_args[FENCE_CONFIG_ARGS_MAX];
+ char con_args[FENCE_CONFIG_ARGS_MAX];
+ int rv, unfence = 0;
+
+ if (strlen(dev_line) > MAX_LINE)
+ return -1;
+
+ memset(dev_name, 0, sizeof(dev_name));
+ memset(agent, 0, sizeof(agent));
+ memset(dev_args, 0, sizeof(dev_args));
+
+ rv = sscanf(dev_line, "%s %s %s %[^\n]s\n", unused, dev_name, agent, dev_args);
+ if (rv < 3)
+ return -1;
+
+ while (fgets(line, MAX_LINE, file)) {
+ if (line[0] == '\n')
+ break;
+ if (line[0] == '#')
+ continue;
+
+ if (!strncmp(line, "unfence", strlen("unfence"))) {
+ /* TODO: verify dev_name matches */
+ unfence = 1;
+ continue;
+ }
+
+ /* invalid config */
+ if (strncmp(line, "connect", strlen("connect")))
+ return -1;
+
+ memset(con_name, 0, sizeof(con_name));
+ memset(con_args, 0, sizeof(con_args));
+
+ rv = sscanf(line, "%s %s %[^\n]s", unused, con_name, con_args);
+ if (rv < 3)
+ return -1;
+
+ /* invalid config */
+ if (strncmp(dev_name, con_name, FENCE_CONFIG_NAME_MAX))
+ return -1;
+
+ /* skip connection for another node */
+ if (con_args_nodeid(con_args) != nodeid)
+ continue;
+
+ dev = malloc(sizeof(struct fence_device));
+ if (!dev)
+ return -ENOMEM;
+
+ con = malloc(sizeof(struct fence_connect));
+ if (!con) {
+ free(dev);
+ return -ENOMEM;
+ }
+
+ memset(dev, 0, sizeof(struct fence_device));
+ memset(con, 0, sizeof(struct fence_connect));
+
+ strncpy(dev->name, dev_name, FENCE_CONFIG_NAME_MAX-1);
+ strncpy(dev->agent, agent, FENCE_CONFIG_NAME_MAX-1);
+ strncpy(dev->args, dev_args, FENCE_CONFIG_ARGS_MAX-1);
+ strncpy(con->name, con_name, FENCE_CONFIG_NAME_MAX-1);
+ strncpy(con->args, con_args, FENCE_CONFIG_ARGS_MAX-1);
+ dev->unfence = unfence;
+
+ *dev_out = dev;
+ *con_out = con;
+ return 0;
+ }
+
+ return -1;
+}
+
+void fence_config_free(struct fence_config *fc)
+{
+ struct fence_device *dev;
+ struct fence_connect *con;
+ int i;
+
+ for (i = 0; i < FENCE_CONFIG_DEVS_MAX; i++) {
+ dev = fc->dev[i];
+ con = fc->con[i];
+ if (dev)
+ free(dev);
+ if (con)
+ free(con);
+ }
+
+ memset(fc, 0, sizeof(struct fence_config));
+}
+
+int fence_config_init(struct fence_config *fc, unsigned int nodeid, char *path)
+{
+ char line[MAX_LINE];
+ struct fence_device *dev;
+ struct fence_connect *con;
+ FILE *file;
+ int pos = 0;
+ int rv;
+
+ fc->nodeid = nodeid;
+
+ file = fopen(path, "r");
+ if (!file)
+ return -1;
+
+ while (fgets(line, MAX_LINE, file)) {
+ if (line[0] == '#')
+ continue;
+ if (line[0] == '\n')
+ continue;
+
+ if (!strncmp(line, "fence_all", strlen("fence_all"))) {
+ /* fence_all cannot be used with other fence devices */
+ if (pos) {
+ rv = -EINVAL;
+ goto out;
+ }
+
+ dev = malloc(sizeof(struct fence_device));
+ if (!dev) {
+ rv = -ENOMEM;
+ goto out;
+ }
+
+ rv = sscanf(line, "%s %s %[^\n]s\n", dev->name, dev->agent, dev->args);
+ if (rv < 2) {
+ rv = -EINVAL;
+ goto out;
+ }
+
+ fc->dev[0] = dev;
+ fc->pos = 0;
+ rv = 0;
+ goto out;
+ }
+
+ if (strncmp(line, "device", strlen("device")))
+ continue;
+
+ /* read connect and unfence lines following a device line */
+
+ rv = read_config_section(nodeid, file, line, &dev, &con);
+ if (rv < 0)
+ continue;
+
+ fc->dev[pos] = dev;
+ fc->con[pos] = con;
+ pos++;
+ }
+
+ if (pos)
+ rv = 0;
+ out:
+ fclose(file);
+ return rv;
+}
+
+static int same_base_name(struct fence_device *a,
+ struct fence_device *b)
+{
+ int len, i;
+
+ len = strlen(a->name);
+ if (len > strlen(b->name))
+ len = strlen(b->name);
+
+ for (i = 0; i < len; i++) {
+ if (a->name[i] == ':' && b->name[i] == ':')
+ return 1;
+ if (a->name[i] == b->name[i])
+ continue;
+ return 0;
+ }
+ return 0;
+}
+
+/*
+ * if next dev is in parallel with last one,
+ * set d,c return 0, else -1
+ *
+ * two consecutive devs with same basename are parallel
+ */
+
+int fence_config_next_parallel(struct fence_config *fc)
+{
+ struct fence_device *prev, *next;
+ int d = fc->pos;
+
+ if (d >= FENCE_CONFIG_DEVS_MAX)
+ return -1;
+
+ prev = fc->dev[d];
+ next = fc->dev[d+1];
+
+ if (!next)
+ return -1;
+
+ if (same_base_name(prev, next)) {
+ fc->pos = d+1;
+ return 0;
+ }
+ return -1;
+}
+
+/*
+ * if there's a dev with the next priority,
+ * set d,c return 0, else -1
+ *
+ * look for another dev with a non-matching basename
+ */
+
+int fence_config_next_priority(struct fence_config *fc)
+{
+ struct fence_device *prev, *next;
+ int d = fc->pos;
+ int i;
+
+ if (d >= FENCE_CONFIG_DEVS_MAX)
+ return -1;
+
+ prev = fc->dev[d];
+
+ for (i = d+1; i < FENCE_CONFIG_DEVS_MAX; i++) {
+ next = fc->dev[i];
+
+ if (!next)
+ return -1;
+
+ if (same_base_name(prev, next))
+ continue;
+
+ fc->pos = d+1;
+ return 0;
+ }
+ return -1;
+}
+
+int fence_config_agent_args(struct fence_config *fc, char *extra, char *args)
+{
+ struct fence_device *dev;
+ struct fence_connect *con;
+ char node[FENCE_CONFIG_NAME_MAX];
+ char *p;
+ int n = 0;
+ int i, len;
+
+ dev = fc->dev[fc->pos];
+ con = fc->con[fc->pos];
+
+ memset(node, 0, sizeof(node));
+ snprintf(node, FENCE_CONFIG_NAME_MAX-1, "node=%u\n", fc->nodeid);
+ len = strlen(node);
+
+ if (dev)
+ len += strlen(dev->args) + 1; /* +1 for \n */
+ if (con)
+ len += strlen(con->args) + 1;
+ if (extra)
+ len += strlen(extra) + 1;
+
+ if (len > FENCE_CONFIG_ARGS_MAX - 1)
+ return -1;
+
+ if (dev && dev->args[0]) {
+ p = dev->args;
+
+ for (i = 0; i < strlen(dev->args); i++) {
+ if (*p == ' ')
+ args[n++] = '\n';
+ else
+ args[n++] = *p;
+ p++;
+ }
+ args[n++] = '\n';
+ }
+
+ if (con && con->args[0]) {
+ p = con->args;
+
+ for (i = 0; i < strlen(con->args); i++) {
+ if (*p == ' ')
+ args[n++] = '\n';
+ else
+ args[n++] = *p;
+ p++;
+ }
+ args[n++] = '\n';
+ }
+
+ if (!strstr(args, "node="))
+ strcat(args, node);
+ if (extra)
+ strcat(args, extra);
+
+ return 0;
+}
+
diff --git a/dlm_controld/fence_config.h b/dlm_controld/fence_config.h
new file mode 100644
index 0000000..47e7bda
--- /dev/null
+++ b/dlm_controld/fence_config.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2012 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+#ifndef _FENCE_CONFIG_H_
+#define _FENCE_CONFIG_H_
+
+#define FENCE_CONFIG_DEVS_MAX 4 /* max devs per node */
+#define FENCE_CONFIG_NAME_MAX 256 /* including terminating \0 */
+#define FENCE_CONFIG_ARGS_MAX 4096 /* including terminating \0 */
+
+struct fence_device {
+ char name[FENCE_CONFIG_NAME_MAX];
+ char agent[FENCE_CONFIG_NAME_MAX];
+ char args[FENCE_CONFIG_ARGS_MAX];
+ int unfence;
+};
+
+struct fence_connect {
+ char name[FENCE_CONFIG_NAME_MAX];
+ char args[FENCE_CONFIG_ARGS_MAX];
+};
+
+/* describes fence config for one node */
+
+struct fence_config {
+ struct fence_device *dev[FENCE_CONFIG_DEVS_MAX];
+ struct fence_connect *con[FENCE_CONFIG_DEVS_MAX];
+ unsigned int nodeid;
+ int pos;
+};
+
+
+int fence_config_init(struct fence_config *fc, unsigned int nodeid, char *path);
+void fence_config_free(struct fence_config *fc);
+
+/*
+ * Iterate through fence_config, sets pos to indicate next to try.
+ * Based on two rules:
+ *
+ * - next_parallel is the next device with the same base name
+ * as the current device (base name is name preceding ":")
+ *
+ * - next_priority is the next device without the same base name
+ * as the current device
+ */
+
+int fence_config_next_parallel(struct fence_config *fc);
+int fence_config_next_priority(struct fence_config *fc);
+
+/*
+ * Combine dev->args and con->args, replacing ' ' with '\n'.
+ * Also add "node=nodeid" if "node=" does not already exist.
+ */
+
+int fence_config_agent_args(struct fence_config *fc, char *extra, char *args);
+
+#endif
diff --git a/dlm_controld/lib.c b/dlm_controld/lib.c
index 09f84cf..228f037 100644
--- a/dlm_controld/lib.c
+++ b/dlm_controld/lib.c
@@ -425,3 +425,22 @@ int dlmc_deadlock_check(char *name)
return rv;
}
+int dlmc_fence_ack(char *name)
+{
+ struct dlmc_header h;
+ int fd, rv;
+
+ init_header(&h, DLMC_CMD_FENCE_ACK, name, 0);
+
+ fd = do_connect(DLMC_SOCK_PATH);
+ if (fd < 0) {
+ rv = fd;
+ goto out;
+ }
+
+ rv = do_write(fd, &h, sizeof(h));
+ close(fd);
+ out:
+ return rv;
+}
+
diff --git a/dlm_controld/libdlmcontrol.h b/dlm_controld/libdlmcontrol.h
index 609372d..4034372 100644
--- a/dlm_controld/libdlmcontrol.h
+++ b/dlm_controld/libdlmcontrol.h
@@ -93,6 +93,7 @@ int dlmc_fs_notified(int fd, char *name, int nodeid);
int dlmc_fs_result(int fd, char *name, int *type, int *nodeid, int *result);
int dlmc_deadlock_check(char *name);
+int dlmc_fence_ack(char *name);
#endif
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index e0420b6..e623a6f 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -163,6 +163,10 @@ static void sigterm_handler(int sig)
daemon_quit = 1;
}
+static void sigchld_handler(int sig)
+{
+}
+
static struct lockspace *create_ls(char *name)
{
struct lockspace *ls;
@@ -649,6 +653,10 @@ static void process_connection(int ci)
}
switch (h.command) {
+ case DLMC_CMD_FENCE_ACK:
+ fence_ack_node(atoi(h.name));
+ break;
+
case DLMC_CMD_FS_REGISTER:
if (cfgd_enable_fscontrol) {
rv = fs_register_add(h.name);
@@ -869,6 +877,8 @@ static void loop(void)
void (*workfn) (int ci);
void (*deadfn) (int ci);
+ setup_config(0);
+
rv = setup_queries();
if (rv < 0)
goto out;
@@ -884,13 +894,15 @@ static void loop(void)
if (rv > 0)
client_add(rv, process_cluster_cfg, cluster_dead);
+ rv = setup_node_config();
+ if (rv < 0)
+ goto out;
+
rv = setup_cluster();
if (rv < 0)
goto out;
client_add(rv, process_cluster, cluster_dead);
- setup_config(0);
-
rv = check_uncontrolled_lockspaces();
if (rv < 0)
goto out;
@@ -945,8 +957,10 @@ static void loop(void)
if (rv == -1 && errno == EINTR) {
if (daemon_quit && list_empty(&lockspaces))
goto out;
- log_error("shutdown ignored, active lockspaces");
- daemon_quit = 0;
+ if (daemon_quit) {
+ log_error("shutdown ignored, active lockspaces");
+ daemon_quit = 0;
+ }
continue;
}
if (rv < 0) {
@@ -977,7 +991,12 @@ static void loop(void)
poll_timeout = -1;
- if (poll_fencing || poll_fs) {
+ if (poll_fencing) {
+ process_fencing_changes();
+ poll_timeout = 1000;
+ }
+
+ if (poll_lockspaces || poll_fs) {
process_lockspace_changes();
poll_timeout = 1000;
}
@@ -1097,7 +1116,7 @@ static void print_usage(void)
printf(" -f <num> Enable (1) or disable (0) fencing recovery dependency\n");
printf(" Default is %d\n", DEFAULT_ENABLE_FENCING);
printf(" -q <num> Enable (1) or disable (0) quorum recovery dependency\n");
- printf(" Default is %d\n", DEFAULT_ENABLE_QUORUM);
+ printf(" Default is %d\n", DEFAULT_ENABLE_QUORUM_FENCING);
printf(" -s <num> Enable (1) or disable (0) fs_controld recovery coordination\n");
printf(" Default is %d\n", DEFAULT_ENABLE_FSCONTROL);
#if 0
@@ -1121,7 +1140,7 @@ static void print_usage(void)
printf(" -V Print program version information, then exit\n");
}
-#define OPTION_STRING "LDKf:q:p:Pl:o:t:c:a:hVr:s:"
+#define OPTION_STRING "LDKf:q:p:Pl:o:t:c:a:hVr:s:e:d:"
static void read_arguments(int argc, char **argv)
{
@@ -1151,21 +1170,35 @@ static void read_arguments(int argc, char **argv)
cfgk_protocol = atoi(optarg);
break;
+ case 's':
+ optd_enable_fscontrol = 1;
+ cfgd_enable_fscontrol = atoi(optarg);
+ break;
+
+ /* fencing options */
+
case 'f':
optd_enable_fencing = 1;
cfgd_enable_fencing = atoi(optarg);
break;
case 'q':
- optd_enable_quorum = 1;
- cfgd_enable_quorum = atoi(optarg);
+ optd_enable_quorum_fencing = 1;
+ cfgd_enable_quorum_fencing = atoi(optarg);
break;
- case 's':
- optd_enable_fscontrol = 1;
- cfgd_enable_fscontrol = atoi(optarg);
+ case 'e':
+ optd_fence_all_agent = 1;
+ strcpy(fence_all_agent, optarg);
break;
+ case 'd':
+ optd_startup_fence = 1;
+ cfgd_startup_fence = atoi(optarg);
+
+
+ /* plock options */
+
case 'p':
optd_enable_plock = 1;
cfgd_enable_plock = atoi(optarg);
@@ -1256,15 +1289,15 @@ static void set_scheduler(void)
int main(int argc, char **argv)
{
- int fd;
+ struct sigaction act;
+ int fd, rv;
cfgk_debug = -1;
cfgk_timewarn = -1;
cfgk_protocol = PROTO_DETECT;
cfgd_debug_logfile = DEFAULT_DEBUG_LOGFILE;
- cfgd_enable_fencing = DEFAULT_ENABLE_FENCING;
- cfgd_enable_quorum = DEFAULT_ENABLE_QUORUM;
- cfgd_enable_quorum = DEFAULT_ENABLE_FSCONTROL;
+ cfgd_enable_fscontrol = DEFAULT_ENABLE_FSCONTROL;
+
cfgd_enable_plock = DEFAULT_ENABLE_PLOCK;
cfgd_plock_debug = DEFAULT_PLOCK_DEBUG;
cfgd_plock_rate_limit = DEFAULT_PLOCK_RATE_LIMIT;
@@ -1273,8 +1306,20 @@ int main(int argc, char **argv)
cfgd_drop_resources_count = DEFAULT_DROP_RESOURCES_COUNT;
cfgd_drop_resources_age = DEFAULT_DROP_RESOURCES_AGE;
+ cfgd_enable_fencing = DEFAULT_ENABLE_FENCING;
+ cfgd_enable_quorum_lockspace= DEFAULT_ENABLE_QUORUM_LOCKSPACE;
+ cfgd_enable_quorum_fencing = DEFAULT_ENABLE_QUORUM_FENCING;
+ cfgd_startup_fence = DEFAULT_STARTUP_FENCE;
+
+ strcpy(fence_all_agent, DEFAULT_FENCE_ALL_AGENT);
+ memset(&fence_all_device, 0, sizeof(struct fence_device));
+ strcpy(fence_all_device.name, "fence_all");
+ strcpy(fence_all_device.agent, fence_all_agent);
+
INIT_LIST_HEAD(&lockspaces);
INIT_LIST_HEAD(&fs_register_list);
+
+ init_daemon();
read_arguments(argc, argv);
@@ -1293,7 +1338,18 @@ int main(int argc, char **argv)
log_level(NULL, LOG_INFO, "dlm_controld %s started", RELEASE_VERSION);
- signal(SIGTERM, sigterm_handler);
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = sigterm_handler;
+ rv = sigaction(SIGTERM, &act, NULL);
+ if (rv < 0)
+ return -rv;
+
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = sigchld_handler;
+ act.sa_flags = SA_NOCLDSTOP;
+ rv = sigaction(SIGCHLD, &act, NULL);
+ if (rv < 0)
+ return -rv;
set_scheduler();
diff --git a/dlm_controld/member.c b/dlm_controld/member.c
index 7af581c..841e5e3 100644
--- a/dlm_controld/member.c
+++ b/dlm_controld/member.c
@@ -9,6 +9,7 @@
#include "dlm_daemon.h"
#include <corosync/corotypes.h>
#include <corosync/cfg.h>
+#include <corosync/cmap.h>
#include <corosync/quorum.h>
static corosync_cfg_handle_t ch;
@@ -111,8 +112,13 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,
int i, j, num_addrs;
uint64_t now = monotime();
+ if (!cluster_joined_monotime) {
+ cluster_joined_monotime = now;
+ cluster_joined_walltime = time(NULL);
+ }
+
if (!cluster_quorate && quorate)
- quorate_time = now;
+ cluster_quorate_monotime = now;
cluster_quorate = quorate;
cluster_ringid_seq = (uint32_t)ring_seq;
@@ -174,8 +180,10 @@ void process_cluster(int ci)
cs_error_t err;
err = quorum_dispatch(qh, CS_DISPATCH_ALL);
- if (err != CS_OK)
+ if (err != CS_OK) {
+ log_error("process_cluster quorum_dispatch %d", err);
cluster_dead(0);
+ }
}
/* Force re-read of quorum nodes */
@@ -184,8 +192,10 @@ void update_cluster(void)
cs_error_t err;
err = quorum_dispatch(qh, CS_DISPATCH_ONE);
- if (err != CS_OK)
+ if (err != CS_OK) {
+ log_error("update_cluster quorum_dispatch %d", err);
cluster_dead(0);
+ }
}
int setup_cluster(void)
@@ -272,8 +282,10 @@ void process_cluster_cfg(int ci)
cs_error_t err;
err = corosync_cfg_dispatch(ch, CS_DISPATCH_ALL);
- if (err != CS_OK)
+ if (err != CS_OK) {
+ log_error("process_cluster_cfg cfg_dispatch %d", err);
cluster_dead(0);
+ }
}
int setup_cluster_cfg(void)
@@ -319,3 +331,34 @@ void close_cluster_cfg(void)
corosync_cfg_finalize(ch);
}
+int setup_node_config(void)
+{
+ char key[CMAP_KEYNAME_MAXLEN+1];
+ cmap_handle_t h;
+ cs_error_t err;
+ uint32_t nodeid;
+ int i;
+
+ err = cmap_initialize(&h);
+ if (err != CS_OK) {
+ log_error("corosync cmap init error %d", err);
+ return -1;
+ }
+
+ for (i = 0; i < MAX_NODES; i++) {
+ snprintf(key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%d.nodeid", i);
+
+ err = cmap_get_uint32(h, key, &nodeid);
+ if (err != CS_OK)
+ break;
+
+ log_debug("node_config %d", nodeid);
+
+ if (cfgd_enable_fencing && cfgd_startup_fence)
+ add_startup_node(nodeid);
+ }
+
+ cmap_finalize(h);
+ return 0;
+}
+
diff --git a/dlm_controld/plock.c b/dlm_controld/plock.c
index b6dc0a4..ed98e9c 100644
--- a/dlm_controld/plock.c
+++ b/dlm_controld/plock.c
@@ -27,8 +27,6 @@ static struct timeval plock_rate_last;
static int plock_device_fd = -1;
-extern int message_flow_control_on;
-
#define RD_CONTINUE 0x00000001
struct resource_data {
@@ -1480,14 +1478,6 @@ int limit_plocks(void)
{
struct timeval now;
- /* Don't send more messages while the cpg message queue is backed up */
-
- if (message_flow_control_on) {
- update_flow_control_status();
- if (message_flow_control_on)
- return 1;
- }
-
if (!cfgd_plock_rate_limit || !plock_read_count)
return 0;
diff --git a/dlm_tool/main.c b/dlm_tool/main.c
index c9a15db..c378f2b 100644
--- a/dlm_tool/main.c
+++ b/dlm_tool/main.c
@@ -38,6 +38,7 @@
#define OP_LOCKDUMP 8
#define OP_LOCKDEBUG 9
#define OP_LOG_PLOCK 10
+#define OP_FENCE_ACK 11
static char *prog_name;
static char *lsname;
@@ -182,7 +183,7 @@ static void print_usage(void)
printf("\n");
printf("dlm_tool [options] [join | leave | lockdump | lockdebug |\n"
" ls | dump | log_plock | plocks |\n"
- " deadlock_check]\n");
+ " fence_ack]\n");
printf("\n");
printf("Options:\n");
printf(" -n Show all node information in ls\n");
@@ -321,6 +322,11 @@ static void decode_arguments(int argc, char **argv)
operation = OP_DEADLOCK_CHECK;
opt_ind = optind + 1;
break;
+ } else if (!strncmp(argv[optind], "fence_ack", 9) &&
+ (strlen(argv[optind]) == 9)) {
+ operation = OP_FENCE_ACK;
+ opt_ind = optind + 1;
+ break;
} else if (!strncmp(argv[optind], "dump", 4) &&
(strlen(argv[optind]) == 4)) {
operation = OP_DUMP;
@@ -1241,6 +1247,11 @@ static void do_deadlock_check(char *name)
dlmc_deadlock_check(name);
}
+static void do_fence_ack(char *name)
+{
+ dlmc_fence_ack(name);
+}
+
static void do_plocks(char *name)
{
char buf[DLMC_DUMP_SIZE];
@@ -1335,6 +1346,10 @@ int main(int argc, char **argv)
case OP_LOCKDEBUG:
do_lockdebug(lsname);
break;
+
+ case OP_FENCE_ACK:
+ do_fence_ack(lsname);
+ break;
}
return 0;
}
diff --git a/fence/Makefile b/fence/Makefile
new file mode 100644
index 0000000..c6ff2c0
--- /dev/null
+++ b/fence/Makefile
@@ -0,0 +1,56 @@
+DESTDIR=
+PREFIX=/usr
+BINDIR=$(PREFIX)/sbin
+#MANDIR=$(PREFIX)/share/man
+
+BIN_TARGET = dlm_stonith
+#MAN_TARGET = dlm_stonith.8
+
+BIN_SOURCE = stonith_helper.c
+
+BIN_CFLAGS += -D_GNU_SOURCE -g \
+ -Wall \
+ -Wformat \
+ -Wformat-security \
+ -Wmissing-prototypes \
+ -Wnested-externs \
+ -Wpointer-arith \
+ -Wextra -Wshadow \
+ -Wcast-align \
+ -Wwrite-strings \
+ -Waggregate-return \
+ -Wstrict-prototypes \
+ -Winline \
+ -Wredundant-decls \
+ -Wno-sign-compare \
+ -Wno-unused-parameter \
+ -Wp,-D_FORTIFY_SOURCE=2 \
+ -fexceptions \
+ -fasynchronous-unwind-tables \
+ -fdiagnostics-show-option \
+
+BIN_CFLAGS += -fPIE -DPIE
+BIN_CFLAGS += `xml2-config --cflags`
+BIN_CFLAGS += -I../include
+
+BIN_LDFLAGS += -Wl,-z,now -Wl,-z,relro -pie
+BIN_LDFLAGS += `xml2-config --libs`
+
+all: $(BIN_TARGET)
+
+$(BIN_TARGET): $(BIN_SOURCE)
+ $(CC) $(BIN_CFLAGS) $(BIN_LDFLAGS) $(BIN_SOURCE) -o $@ -L.
+
+clean:
+ rm -f *.o *.so *.so.* $(BIN_TARGET)
+
+
+INSTALL=$(shell which install)
+
+.PHONY: install
+install: all
+ $(INSTALL) -d $(DESTDIR)/$(BINDIR)
+ $(INSTALL) -d $(DESTDIR)/$(MANDIR)/man8
+ $(INSTALL) -c -m 755 $(BIN_TARGET) $(DESTDIR)/$(BINDIR)
+# $(INSTALL) -m 644 $(MAN_TARGET) $(DESTDIR)/$(MANDIR)/man8/
+
diff --git a/fence/stonith_helper.c b/fence/stonith_helper.c
new file mode 100644
index 0000000..df44219
--- /dev/null
+++ b/fence/stonith_helper.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2012 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <syslog.h>
+#include <pacemaker/crm/stonith-ng.h>
+
+int nodeid;
+uint64_t fail_time;
+
+#define MAX_ARG_LEN 1024
+
+static int get_options(int argc, char *argv[])
+{
+ char arg[MAX_ARG_LEN];
+ char key[MAX_ARG_LEN];
+ char val[MAX_ARG_LEN];
+ char c;
+ int rv;
+
+ if (argc > 1) {
+ while ((c = getopt(argc, argv, "n:t:")) != -1) {
+ switch (c) {
+ case 'n':
+ nodeid = atoi(optarg);
+ break;
+ case 't':
+ fail_time = strtoull(optarg, NULL, 0);
+ break;
+ }
+ }
+ } else {
+ while (fgets(arg, sizeof(arg), stdin)) {
+ rv = sscanf(arg, "%[^=]=%s\n", key, val);
+ if (rv != 2)
+ continue;
+
+ if (!strcmp(key, "node"))
+ nodeid = atoi(val);
+ else if (!strcmp(key, "fail_time"))
+ fail_time = strtoull(val, NULL, 0);
+ }
+ }
+
+ if (!nodeid) {
+ fprintf(stderr, "no node\n");
+ return -1;
+ }
+
+ if (!fail_time) {
+ fprintf(stderr, "no fail_time\n");
+ return -1;
+ }
+
+ return 0;
+}
+
+int main(int argc, char *argv[])
+{
+ uint64_t t;
+ int rv;
+
+ rv = get_options(argc, argv);
+ if (rv)
+ return rv;
+
+ t = stonith_api_time_helper(nodeid, 0);
+ if (t >= fail_time)
+ return 0;
+
+ rv = stonith_api_kick_helper(nodeid, 300, 1);
+ if (rv) {
+ fprintf(stderr, "kick_helper error %d nodeid %d\n", rv, nodeid);
+ openlog("stonith_helper", LOG_CONS | LOG_PID, LOG_DAEMON);
+ syslog(LOG_ERR, "kick_helper error %d nodeid %d\n", rv, nodeid);
+ return rv;
+ }
+
+ while (1) {
+ t = stonith_api_time_helper(nodeid, 0);
+ if (t >= fail_time)
+ return 0;
+ sleep(1);
+ }
+
+ return -1;
+}
+
More information about the cluster-commits
mailing list