cluster: STABLE32 - rgmanager: Retry connection to cpglockd
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 26c3db58dac85cabef6001b9b9abc3a089b77c26
Parent: f5cf113f8e43a3f9a68c55a625cc6ebc8825e4c4
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Fri Apr 27 18:33:34 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Mon Apr 30 09:19:36 2012 -0400
rgmanager: Retry connection to cpglockd
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/clulib/cpg_lock.c | 27 ++++++++++++++++++---------
1 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/rgmanager/src/clulib/cpg_lock.c b/rgmanager/src/clulib/cpg_lock.c
index d257cf4..7c1c8c3 100644
--- a/rgmanager/src/clulib/cpg_lock.c
+++ b/rgmanager/src/clulib/cpg_lock.c
@@ -105,18 +105,27 @@ _cpg_lock_finished(const char *name)
int
cpg_lock_initialize(void)
{
- int ret, err;
+ int ret, err, retries = 0;
- pthread_mutex_lock(&_default_lock);
- if (_cpgh) {
+ while (retries < 10) {
+ pthread_mutex_lock(&_default_lock);
+ if (_cpgh) {
+ pthread_mutex_unlock(&_default_lock);
+ return 0;
+ }
+
+ cpg_lock_init(&_cpgh);
+ ret = (_cpgh == NULL);
+ err = errno;
pthread_mutex_unlock(&_default_lock);
- return 0;
- }
- cpg_lock_init(&_cpgh);
- ret = (_cpgh == NULL);
- err = errno;
- pthread_mutex_unlock(&_default_lock);
+ if (ret == 0)
+ break;
+ else {
+ sleep(1);
+ ++retries;
+ }
+ }
/* Set up function pointers */
clu_lock = _cpg_lock;
12 years, 1 month
cluster: STABLE32 - Increment the recovered locks count when a client is disconnected.
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: f5cf113f8e43a3f9a68c55a625cc6ebc8825e4c4
Parent: 2e191d461d982b78b030227a402a847152731304
Author: Ryan McCabe <rmccabe(a)redhat.com>
AuthorDate: Fri Apr 27 18:31:47 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Fri Apr 27 18:33:07 2012 -0400
Increment the recovered locks count when a client is disconnected.
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/daemons/cpglockd.c | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c
index 5d0fed9..1a5832c 100644
--- a/rgmanager/src/daemons/cpglockd.c
+++ b/rgmanager/src/daemons/cpglockd.c
@@ -643,6 +643,7 @@ del_client(int fd)
l->l.state = LOCK_FREE;
strncpy(m.resource, l->l.resource, sizeof(m.resource));
+ ++recovered;
if (grant_next(&m) == 0)
send_unlock(&m);
}
12 years, 1 month
cluster: STABLE32 - Replace printf and fprintf calls with logt_print in the dlm locking module.
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 2e191d461d982b78b030227a402a847152731304
Parent: 124c727e34a946a248af0b8a47e7a2f0d52bd50e
Author: Ryan McCabe <rmccabe(a)redhat.com>
AuthorDate: Fri Apr 27 18:30:54 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Fri Apr 27 18:30:54 2012 -0400
Replace printf and fprintf calls with logt_print in the dlm locking module.
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/clulib/dlm_lock.c | 12 ++++++------
1 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/rgmanager/src/clulib/dlm_lock.c b/rgmanager/src/clulib/dlm_lock.c
index d3f4250..0752b41 100644
--- a/rgmanager/src/clulib/dlm_lock.c
+++ b/rgmanager/src/clulib/dlm_lock.c
@@ -12,6 +12,7 @@
#include <sys/types.h>
#include <sys/select.h>
#include <pthread.h>
+#include <logging.h>
/* Default lockspace stuff */
static dlm_lshandle_t _default_ls = NULL;
@@ -50,9 +51,8 @@ clu_ls_lock(dlm_lshandle_t ls,
int ret;
if (!ls || !lksb || !resource || !strlen(resource)) {
- printf("%p %p %p %d\n", ls, lksb, resource,
- (int)strlen(resource));
- printf("INVAL...\n");
+ logt_print(LOG_DEBUG, "Invalid lock request: %p %p %p %d\n",
+ ls, lksb, resource, resource ? strlen(resource) : 0);
errno = EINVAL;
return -1;
}
@@ -69,8 +69,8 @@ clu_ls_lock(dlm_lshandle_t ls,
}
if ((ret = (wait_for_dlm_event(ls) < 0))) {
- fprintf(stderr, "wait_for_dlm_event: %d / %d\n",
- ret, errno);
+ logt_print(LOG_DEBUG, "wait_for_dlm_event: %d / %d\n",
+ ret, errno);
return -1;
}
@@ -108,7 +108,7 @@ clu_open_lockspace(const char *lsname)
if (errno == ENOENT)
continue;
- fprintf(stderr, "failed acquiring lockspace: %s\n",
+ logt_print(LOG_DEBUG, "failed acquiring lockspace: %s\n",
strerror(errno));
return NULL;
12 years, 1 month
cluster: STABLE32 - Poll for fenced node updates less aggressively while waiting for fencing to complete.
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 124c727e34a946a248af0b8a47e7a2f0d52bd50e
Parent: af8536f05c6d908b1ea8af115a53f5e4f3ae8e79
Author: Ryan McCabe <rmccabe(a)redhat.com>
AuthorDate: Thu Apr 26 14:16:12 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Thu Apr 26 14:16:12 2012 -0400
Poll for fenced node updates less aggressively while waiting for fencing to complete.
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/daemons/cpglockd.c | 17 +++++++----------
1 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c
index f101084..5d0fed9 100644
--- a/rgmanager/src/daemons/cpglockd.c
+++ b/rgmanager/src/daemons/cpglockd.c
@@ -1675,17 +1675,14 @@ main(int argc, char **argv)
** don't need to wait for it.
*/
if (!victim && !x && !pf_node->force_wait) {
- int retries = 0;
/* Wait up to 1s for fenced to set victim */
- do {
- usleep(250000);
- if (fenced_node_info(pf_node->nodeid, &fn) < 0) {
- logt_print(LOG_DEBUG,
- "Unable to get fenced data for node %d\n",
- pf_node->nodeid);
- } else
- victim = fn.victim;
- } while (!victim && retries++ < 4);
+ sleep(1);
+ if (fenced_node_info(pf_node->nodeid, &fn) < 0) {
+ logt_print(LOG_DEBUG,
+ "Unable to get fenced data for node %d\n",
+ pf_node->nodeid);
+ } else
+ victim = fn.victim;
if (!victim) {
logt_print(LOG_DEBUG, "First entry in list and victim == 0, removing %d from pending fencing\n", pf_node->nodeid);
12 years, 1 month
cluster: STABLE32 - Work around a fenced hang/delay that can result in us unpausing too soon while waiting for fencing to complete.
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: af8536f05c6d908b1ea8af115a53f5e4f3ae8e79
Parent: 32c6ea0595c7091f13bcfbae8879c7e9ef035d98
Author: Ryan McCabe <rmccabe(a)redhat.com>
AuthorDate: Thu Apr 26 14:12:37 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Thu Apr 26 14:12:37 2012 -0400
Work around a fenced hang/delay that can result in us unpausing too soon while waiting for fencing to complete.
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/daemons/cpglockd.c | 53 +++++++++++++++++++++++++-------------
1 files changed, 35 insertions(+), 18 deletions(-)
diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c
index fa40e98..f101084 100644
--- a/rgmanager/src/daemons/cpglockd.c
+++ b/rgmanager/src/daemons/cpglockd.c
@@ -40,6 +40,7 @@ struct lock_node {
struct pending_fence_node {
list_head();
int nodeid;
+ int force_wait;
uint64_t fail_time;
};
@@ -96,6 +97,22 @@ flag_shutdown(int __attribute__ ((unused)) sig)
shutdown_pending = 1;
}
+
+static int
+is_member(uint32_t nodeid)
+{
+ struct member_node *n;
+ int x;
+
+ list_for(&group_members, n, x) {
+ if (n->nodeid == nodeid)
+ return 1;
+ }
+
+ return 0;
+}
+
+
static int
cman_nodes_lost(cman_node_t *old_nodes,
size_t old_node_len,
@@ -221,6 +238,14 @@ cman_callback(cman_handle_t ch, void *privdata, int reason, int arg)
pf = do_alloc(sizeof(*pf));
pf->nodeid = cur_nodeid;
pf->fail_time = cur_time;
+ /*
+ ** If the node is also a member of the cpglock group, wait
+ ** for positive confirmation from fenced that it was fenced.
+ ** It cannot have shut down cleanly if we did not process a
+ ** DELETE for it yet.
+ */
+ if (is_member(cur_nodeid))
+ pf->force_wait = 1;
list_append(&pending_fencing, pf);
} else {
logt_print(LOG_DEBUG, "Lost node %d but fencing not configured\n",
@@ -904,21 +929,6 @@ process_lock(struct cpg_lock_msg *m)
static int
-is_member(uint32_t nodeid)
-{
- struct member_node *n;
- int x;
-
- list_for(&group_members, n, x) {
- if (n->nodeid == nodeid)
- return 1;
- }
-
- return 0;
-}
-
-
-static int
process_grant(struct cpg_lock_msg *m, uint32_t nodeid)
{
struct lock_node *l;
@@ -1645,7 +1655,8 @@ main(int argc, char **argv)
}
if (lft > pf_node->fail_time) {
- logt_print(LOG_DEBUG, "Fencing for node %d finished at %ld (>%ld)\n",
+ logt_print(LOG_DEBUG,
+ "Fencing for node %d finished at %ld (>%ld)\n",
pf_node->nodeid, lft, pf_node->fail_time);
list_remove(&pending_fencing, pf_node);
free(pf_node);
@@ -1663,13 +1674,14 @@ main(int argc, char **argv)
** victim to 1 by now, we can deduce it has left cleanly, and we
** don't need to wait for it.
*/
- if (!victim && !x) {
+ if (!victim && !x && !pf_node->force_wait) {
int retries = 0;
/* Wait up to 1s for fenced to set victim */
do {
usleep(250000);
if (fenced_node_info(pf_node->nodeid, &fn) < 0) {
- logt_print(LOG_DEBUG, "Unable to get fenced data for node %d\n",
+ logt_print(LOG_DEBUG,
+ "Unable to get fenced data for node %d\n",
pf_node->nodeid);
} else
victim = fn.victim;
@@ -1685,6 +1697,11 @@ main(int argc, char **argv)
}
goto fence_check;
}
+ if (!victim && !x && pf_node->force_wait) {
+ logt_print(LOG_DEBUG, "Would have removed %d but now waiting\n",
+ pf_node->nodeid);
+ }
+
}
if (shutdown_pending)
12 years, 1 month
cluster: STABLE32 - rgmanager: fix conditional check for cpglockd startup
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 32c6ea0595c7091f13bcfbae8879c7e9ef035d98
Parent: df174aa0a33e21dc787b93852f087fa804bfdaec
Author: Fabio M. Di Nitto <fdinitto(a)redhat.com>
AuthorDate: Thu Apr 26 15:50:34 2012 +0200
Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com>
CommitterDate: Thu Apr 26 15:50:34 2012 +0200
rgmanager: fix conditional check for cpglockd startup
Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com>
---
rgmanager/init.d/rgmanager.in | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/rgmanager/init.d/rgmanager.in b/rgmanager/init.d/rgmanager.in
index 40347ef..931df37 100644
--- a/rgmanager/init.d/rgmanager.in
+++ b/rgmanager/init.d/rgmanager.in
@@ -78,8 +78,8 @@ stop_cluster()
start_cpglockd()
{
rings="$(corosync-objctl 2>/dev/null |grep ringnumber | wc -l)"
- [ "$rings" -gt "1" ] && service cpglockd start
- while ! cpglockdump > /dev/null 2>&1; do sleep 1; done
+ [ "$rings" -gt "1" ] && service cpglockd start && \
+ while ! cpglockdump > /dev/null 2>&1; do sleep 1; done
}
rtrn=0
12 years, 1 month
cluster: STABLE32 - Copy lock messages before swapping bytes because we may use the lock message struct again after calling send_lock_msg()
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: df174aa0a33e21dc787b93852f087fa804bfdaec
Parent: 83b24abdf0f0617d25cf9d5b72d75c9e6cc0c529
Author: Ryan McCabe <rmccabe(a)redhat.com>
AuthorDate: Wed Apr 25 21:23:21 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Wed Apr 25 21:23:21 2012 -0400
Copy lock messages before swapping bytes because we may use the lock message struct again after calling send_lock_msg()
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/daemons/cpglockd.c | 8 +++++---
1 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c
index df18eb3..fa40e98 100644
--- a/rgmanager/src/daemons/cpglockd.c
+++ b/rgmanager/src/daemons/cpglockd.c
@@ -427,11 +427,13 @@ send_lock_msg(struct cpg_lock_msg *m)
{
struct iovec iov;
int ret;
+ struct cpg_lock_msg out_msg;
- swab_cpg_lock_msg(m);
+ memcpy(&out_msg, m, sizeof(out_msg));
+ swab_cpg_lock_msg(&out_msg);
- iov.iov_base = m;
- iov.iov_len = sizeof (*m);
+ iov.iov_base = &out_msg;
+ iov.iov_len = sizeof(out_msg);
do {
ret = cpg_mcast_joined(cpg, CPG_TYPE_AGREED, &iov, 1);
12 years, 1 month
cluster: STABLE32 - Set m.owner_nodeid to the our node id when NAKing requests when the cluster is inquorate.
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 83b24abdf0f0617d25cf9d5b72d75c9e6cc0c529
Parent: 24838db6f1e47802188765783f7b1e18ed16a880
Author: Ryan McCabe <rmccabe(a)redhat.com>
AuthorDate: Wed Apr 25 15:31:09 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Wed Apr 25 15:31:09 2012 -0400
Set m.owner_nodeid to the our node id when NAKing requests when the cluster is inquorate.
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/daemons/cpglockd.c | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c
index 1f69ef8..df18eb3 100644
--- a/rgmanager/src/daemons/cpglockd.c
+++ b/rgmanager/src/daemons/cpglockd.c
@@ -1736,11 +1736,13 @@ main(int argc, char **argv)
/* XXX check for dup connection */
if (m.request == MSG_LOCK) {
if (!cluster_quorate) {
+ m.request = MSG_NAK;
+ m.owner_nodeid = my_node_id;
+
logt_print(LOG_DEBUG, "Sending NAK for new lock request while not quorate (%s [%d:%d:%d])\n",
m.resource, m.owner_nodeid,
m.owner_pid, m.owner_tid);
- m.request = MSG_NAK;
if (write_retry(client->fd, &m, sizeof(m), NULL) < 0) {
logt_print(LOG_DEBUG,
"Error sending NAK for %s [%d:%d:%d]: %s\n",
12 years, 1 month
cluster: STABLE32 - Add some additional debugging logging for try-locks
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 24838db6f1e47802188765783f7b1e18ed16a880
Parent: c3e2d472b80f8d1c47c142b5f5baeb250b0edb9e
Author: Ryan McCabe <rmccabe(a)redhat.com>
AuthorDate: Wed Apr 25 13:07:54 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Wed Apr 25 13:07:54 2012 -0400
Add some additional debugging logging for try-locks
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/daemons/cpglockd.c | 5 +++++
1 files changed, 5 insertions(+), 0 deletions(-)
diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c
index 689c494..1f69ef8 100644
--- a/rgmanager/src/daemons/cpglockd.c
+++ b/rgmanager/src/daemons/cpglockd.c
@@ -478,6 +478,8 @@ send_grant(struct request_node *n)
static int
send_nak(struct cpg_lock_msg *m)
{
+ logt_print(LOG_DEBUG, "-> sending NAK %s to %d:%d:%d\n",
+ m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid);
m->request = MSG_NAK;
return send_lock_msg(m);
@@ -1013,6 +1015,9 @@ process_nak(struct cpg_lock_msg *m, uint32_t nodeid)
if (!joined)
return 0;
+ logt_print(LOG_DEBUG, "NAK %s for %d:%d:%d\n", m->resource,
+ m->owner_nodeid, m->owner_pid, m->owner_tid);
+
list_for(&requests, r, y) {
if (strcmp(r->l.resource, m->resource))
continue;
12 years, 1 month
cluster: STABLE32 - Print cpg_mcast_joined() return value and errno if it fails
by Ryan McCabe
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: c3e2d472b80f8d1c47c142b5f5baeb250b0edb9e
Parent: 16ae7b906511976910aa17f879c9e2a6e14b0cfa
Author: Ryan McCabe <rmccabe(a)redhat.com>
AuthorDate: Wed Apr 25 09:34:03 2012 -0400
Committer: Ryan McCabe <rmccabe(a)redhat.com>
CommitterDate: Wed Apr 25 09:34:58 2012 -0400
Print cpg_mcast_joined() return value and errno if it fails
Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com>
---
rgmanager/src/daemons/cpglockd.c | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c
index a3943a2..689c494 100644
--- a/rgmanager/src/daemons/cpglockd.c
+++ b/rgmanager/src/daemons/cpglockd.c
@@ -436,7 +436,8 @@ send_lock_msg(struct cpg_lock_msg *m)
do {
ret = cpg_mcast_joined(cpg, CPG_TYPE_AGREED, &iov, 1);
if (ret != CPG_OK) {
- logt_print(LOG_DEBUG, "send_lock_msg() failed\n");
+ logt_print(LOG_DEBUG, "send_lock_msg() failed %d: %s\n",
+ ret, strerror(errno));
usleep(250000);
}
} while (ret != CPG_OK && !shutdown_pending);
12 years, 1 month