Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=028... Commit: 02850b642b64ec70c1bdbb411eb653624c135384 Parent: 1498ce20c091845100456fd1f235658f6056d140 Author: David Teigland teigland@redhat.com AuthorDate: Fri Jan 6 14:06:31 2012 -0600 Committer: David Teigland teigland@redhat.com CommitterDate: Fri Jan 6 14:06:31 2012 -0600
dlm_controld: wait for ringid
Ensure we don't process a nodedown confchg before getting the corresponding ringid cb.
Improve some debugging.
Signed-off-by: David Teigland teigland@redhat.com --- dlm_controld/cpg.c | 61 +++++++++++++++++++++++++++++++++++++++++++- dlm_controld/dlm_daemon.h | 1 + dlm_controld/member.c | 16 +++++------ 3 files changed, 67 insertions(+), 11 deletions(-)
diff --git a/dlm_controld/cpg.c b/dlm_controld/cpg.c index 183879e..4f63fb0 100644 --- a/dlm_controld/cpg.c +++ b/dlm_controld/cpg.c @@ -564,6 +564,27 @@ static void node_history_start(struct lockspace *ls, int nodeid)
static int check_ringid_done(struct lockspace *ls) { + /* If we've received a confchg due to a nodedown, but not + the corresponding ringid callback, then we should wait + for the ringid callback. Once we have both conf and ring + callbacks, we can compare cpg/quorum ringids. + + Otherwise, there's a possible problem if we receive a + confchg before both ringid callback and quorum callback. + Then we'd get through this function by comparing the old, + matching ringids. + + (We seem to usually get the quorum callback before any cpg + callbacks, in which case we wouldn't need cpg_ringid_wait, + but that's probably not guaranteed.) */ + + if (ls->cpg_ringid_wait) { + log_group(ls, "check_ringid wait cluster %u cpg %u:%llu", + cluster_ringid_seq, ls->cpg_ringid.nodeid, + (unsigned long long)ls->cpg_ringid.seq); + return 0; + } + if (cluster_ringid_seq != (uint32_t)ls->cpg_ringid.seq) { log_group(ls, "check_ringid cluster %u cpg %u:%llu", cluster_ringid_seq, ls->cpg_ringid.nodeid, @@ -1434,6 +1455,24 @@ void process_lockspace_changes(void) } }
+static const char *reason_str(int reason) +{ + switch (reason) { + case CPG_REASON_JOIN: + return "join"; + case CPG_REASON_LEAVE: + return "leave"; + case CPG_REASON_NODEDOWN: + return "nodedown"; + case CPG_REASON_NODEUP: + return "nodeup"; + case CPG_REASON_PROCDOWN: + return "procdown"; + default: + return "unknown"; + }; +} + static int add_change(struct lockspace *ls, const struct cpg_address *member_list, size_t member_list_entries, @@ -1486,6 +1525,9 @@ static int add_change(struct lockspace *ls, } list_add_tail(&memb->list, &cg->removed);
+ if (left_list[i].reason == CPG_REASON_NODEDOWN) + ls->cpg_ringid_wait = 1; + if (memb->failed) { node_history_lockspace_fail(ls, memb->nodeid, cg, left_list[i].reason, now); @@ -1493,8 +1535,8 @@ static int add_change(struct lockspace *ls, node_history_lockspace_left(ls, memb->nodeid, cg, now); }
- log_group(ls, "add_change cg %u remove nodeid %d reason %d", - cg->seq, memb->nodeid, left_list[i].reason); + log_group(ls, "add_change cg %u remove nodeid %d reason %s", + cg->seq, memb->nodeid, reason_str(left_list[i].reason));
if (left_list[i].reason == CPG_REASON_PROCDOWN) kick_node_from_cluster(memb->nodeid); @@ -1829,6 +1871,7 @@ static void totem_cb(cpg_handle_t handle,
ls->cpg_ringid.nodeid = ring_id.nodeid; ls->cpg_ringid.seq = ring_id.seq; + ls->cpg_ringid_wait = 0;
apply_changes(ls); } @@ -2450,12 +2493,26 @@ static void confchg_cb_daemon(cpg_handle_t handle, { struct node_daemon *node; uint64_t now = monotime(); + int nodedown = 0, procdown = 0, leave = 0; int i;
log_config(group_name, member_list, member_list_entries, left_list, left_list_entries, joined_list, joined_list_entries);
+ for (i = 0; i < left_list_entries; i++) { + if (left_list[i].reason == CPG_REASON_NODEDOWN) + nodedown++; + else if (left_list[i].reason == CPG_REASON_PROCDOWN) + procdown++; + else if (left_list[i].reason == CPG_REASON_LEAVE) + leave++; + } + + if (nodedown || procdown || leave) + log_debug("%s left nodedown %d procdown %d leave %d", + group_name->value, nodedown, procdown, leave); + if (joined_list_entries) send_protocol(&our_protocol);
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h index 2d27caa..f1ea8ea 100644 --- a/dlm_controld/dlm_daemon.h +++ b/dlm_controld/dlm_daemon.h @@ -226,6 +226,7 @@ struct lockspace {
cpg_handle_t cpg_handle; struct cpg_ring_id cpg_ringid; + int cpg_ringid_wait; int cpg_client; int cpg_fd; int joining; diff --git a/dlm_controld/member.c b/dlm_controld/member.c index d39e541..58ef70e 100644 --- a/dlm_controld/member.c +++ b/dlm_controld/member.c @@ -243,8 +243,7 @@ void kick_node_from_cluster(int nodeid) corosync_cfg_try_shutdown(ch, COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE); } else { - log_error("telling corosync to remove nodeid %d from cluster", - nodeid); + log_error("tell corosync to remove nodeid %d from cluster", nodeid); corosync_cfg_kill_node(ch, nodeid, "dlm_controld"); } } @@ -253,13 +252,12 @@ static void shutdown_callback(corosync_cfg_handle_t h, corosync_cfg_shutdown_flags_t flags) { if (flags & COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) { - if (list_empty(&lockspaces)) - corosync_cfg_replyto_shutdown(ch, - COROSYNC_CFG_SHUTDOWN_FLAG_YES); - else { - log_debug("no to corosync shutdown"); - corosync_cfg_replyto_shutdown(ch, - COROSYNC_CFG_SHUTDOWN_FLAG_NO); + if (list_empty(&lockspaces)) { + log_debug("shutdown request yes"); + corosync_cfg_replyto_shutdown(ch, COROSYNC_CFG_SHUTDOWN_FLAG_YES); + } else { + log_debug("shutdown request no"); + corosync_cfg_replyto_shutdown(ch, COROSYNC_CFG_SHUTDOWN_FLAG_NO); } } }
cluster-commits@lists.fedorahosted.org