cluster: RHEL6 - Fix operation when quorum is lost: - NAK all pending requests - Drop all locks - Until quorum is regained, NAK all new lock requests
Ryan McCabe
rmccabe at fedoraproject.org
Tue May 1 15:01:45 UTC 2012
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=9310d1ad3a19fbe1aff6238ebc8ce9730a43440f
Commit: 9310d1ad3a19fbe1aff6238ebc8ce9730a43440f
Parent: ff5569b44f106a518908d8a1c8c2791a09744ccd
Author: Ryan McCabe <rmccabe at redhat.com>
AuthorDate: Tue Apr 24 23:54:36 2012 -0400
Committer: Ryan McCabe <rmccabe at redhat.com>
CommitterDate: Tue May 1 10:59:44 2012 -0400
Fix operation when quorum is lost: - NAK all pending requests - Drop all locks - Until quorum is regained, NAK all new lock requests
Signed-off-by: Ryan McCabe <rmccabe at redhat.com>
Reviewed-by: Lon Hohberger <lhh at redhat.com>
Reviewed-by: Fabio M. Di Nitto <fdinitto at redhat.com>
---
rgmanager/src/daemons/cpglockd.c | 110 ++++++++++++++++++++++++++++++++------
1 files changed, 94 insertions(+), 16 deletions(-)
diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c
index d4d43b0..a3943a2 100644
--- a/rgmanager/src/daemons/cpglockd.c
+++ b/rgmanager/src/daemons/cpglockd.c
@@ -588,15 +588,20 @@ del_client(int fd)
logt_print(LOG_DEBUG, "RECOVERY: Looking for locks held by PID %d\n", pid);
- /* This may not be needed */
- purge_requests(my_node_id, pid);
+ if (cluster_quorate) {
+ /* This may not be needed */
+ purge_requests(my_node_id, pid);
- memset(&m, 0, sizeof(m));
- m.request = MSG_PURGE;
- m.owner_nodeid = my_node_id;
- m.owner_pid = pid;
+ memset(&m, 0, sizeof(m));
+ m.request = MSG_PURGE;
+ m.owner_nodeid = my_node_id;
+ m.owner_pid = pid;
- send_lock_msg(&m);
+ send_lock_msg(&m);
+ }
+
+ if (!cluster_quorate && locks != NULL)
+ logt_print(LOG_DEBUG, "RECOVERY: not quorate but locks exist\n");
list_for(&locks, l, x) {
if (l->l.owner_nodeid != my_node_id ||
@@ -1385,6 +1390,36 @@ static void init_logging(int reconf)
logt_conf("cpglockd", mode, syslog_facility, syslog_priority, logfile_priority, logfile);
}
+static void
+clear_local_state(void)
+{
+ struct request_node *cur_req = NULL;
+ struct lock_node *cur_lock = NULL;
+
+ while ((cur_lock = locks) != NULL) {
+ logt_print(LOG_DEBUG,
+ "CLEAR STATE: Dropping lock %s [%d:%d]\n",
+ cur_lock->l.resource,
+ cur_lock->l.owner_nodeid, cur_lock->l.owner_pid);
+ list_remove(&locks, cur_lock);
+ free(cur_lock);
+ }
+
+ while ((cur_req = requests) != NULL) {
+ if (cur_req->l.owner_nodeid == my_node_id) {
+ nak_client(cur_req);
+ logt_print(LOG_DEBUG, "CLEAR STATE: sent NAK %s [%d:%d:%d]\n",
+ cur_req->l.resource, cur_req->l.owner_nodeid,
+ cur_req->l.owner_pid, cur_req->l.owner_tid);
+ }
+ logt_print(LOG_DEBUG, "CLEAR STATE: Dropping request %s [%d:%d:%d]\n",
+ cur_req->l.resource, cur_req->l.owner_nodeid,
+ cur_req->l.owner_pid, cur_req->l.owner_tid);
+ list_remove(&requests, cur_req);
+ free(cur_req);
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -1394,6 +1429,7 @@ main(int argc, char **argv)
int afd = -1;
int cman_fd;
int n,x;
+ int quorate_last = 0;
int wait_for_quorum = 1;
int wait_for_fencing = 1;
int opt;
@@ -1525,6 +1561,10 @@ main(int argc, char **argv)
goto out;
}
+ if (wait_for_quorum)
+ wait_for_quorum_formation(cman_handle);
+ quorate_last = cluster_quorate;
+
logt_print(LOG_INFO, "cpglockd entering normal operation\n");
while (!shutdown_pending) {
struct timeval tv;
@@ -1569,6 +1609,16 @@ main(int argc, char **argv)
goto out;
}
+ if (!quorate_last && cluster_quorate)
+ logt_print(LOG_INFO, "Cluster quorum regained\n");
+
+ if (quorate_last && !cluster_quorate) {
+ logt_print(LOG_INFO,
+ "Cluster lost quorum. Clearing local state.\n");
+ clear_local_state();
+ }
+ quorate_last = cluster_quorate;
+
fence_check:
list_for(&pending_fencing, pf_node, x) {
int victim;
@@ -1640,16 +1690,23 @@ main(int argc, char **argv)
if (FD_ISSET(fd, &rfds)) {
afd = accept(fd, NULL, NULL);
- insert_client(afd);
+ if (afd < 0) {
+ logt_print(LOG_DEBUG, "Error accepting new client: %s\n",
+ strerror(errno));
+ } else {
+ insert_client(afd);
+ }
--n;
}
if (FD_ISSET(cpgfd, &rfds)) {
- if (cpg_dispatch(cpg, CPG_DISPATCH_ALL) != CPG_OK) {
+ --n;
+ if (cluster_quorate &&
+ cpg_dispatch(cpg, CPG_DISPATCH_ALL) != CPG_OK)
+ {
logt_print(LOG_ERR, "Fatal: Lost CPG connection.\n");
return -1;
}
- --n;
}
if (n <= 0)
@@ -1660,9 +1717,10 @@ main(int argc, char **argv)
if (!FD_ISSET(client->fd, &rfds))
continue;
--n;
+
if (read_retry(client->fd, &m, sizeof(m), NULL) < 0) {
logt_print(LOG_DEBUG, "Closing client fd %d pid %d: %d\n",
- client->fd, client->pid, errno);
+ client->fd, client->pid, errno);
del_client(client->fd);
break;
@@ -1671,15 +1729,35 @@ main(int argc, char **argv)
/* send lock request */
/* XXX check for dup connection */
if (m.request == MSG_LOCK) {
- client->pid = m.owner_pid;
- send_lock(&m);
+ if (!cluster_quorate) {
+ logt_print(LOG_DEBUG, "Sending NAK for new lock request while not quorate (%s [%d:%d:%d])\n",
+ m.resource, m.owner_nodeid,
+ m.owner_pid, m.owner_tid);
+
+ m.request = MSG_NAK;
+ if (write_retry(client->fd, &m, sizeof(m), NULL) < 0) {
+ logt_print(LOG_DEBUG,
+ "Error sending NAK for %s [%d:%d:%d]: %s\n",
+ m.resource, m.owner_nodeid,
+ m.owner_pid, m.owner_tid, strerror(errno));
+ }
+ } else {
+ client->pid = m.owner_pid;
+ send_lock(&m);
+ }
}
if (m.request == MSG_UNLOCK) {
//logt_print(LOG_DEBUG, "Unlock from fd %d\n", client->fd);
- find_lock(&m);
- if (grant_next(&m) == 0)
- send_unlock(&m);
+ /*
+ ** When not quorate, all locks and requests have already
+ ** been dropped.
+ */
+ if (cluster_quorate) {
+ find_lock(&m);
+ if (grant_next(&m) == 0)
+ send_unlock(&m);
+ }
}
if (m.request == MSG_DUMP) {
More information about the cluster-commits
mailing list