cluster: RHEL6 - rgmanager: Add convalesce operation
Lon Hohberger
lon at fedoraproject.org
Tue Feb 1 18:10:39 UTC 2011
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=abe50ac2721ec8124aa2a614c2a0a05e4cfa3ad7
Commit: abe50ac2721ec8124aa2a614c2a0a05e4cfa3ad7
Parent: b631ffdb818f7cf3512840dd99b8844aa230b03d
Author: Lon Hohberger <lhh at redhat.com>
AuthorDate: Wed Sep 1 15:24:22 2010 -0400
Committer: Lon Hohberger <lhh at redhat.com>
CommitterDate: Tue Feb 1 12:31:11 2011 -0500
rgmanager: Add convalesce operation
This option enables the administrator to repair
a partially-failed service without affecting
the rest of the service.
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh at redhat.com>
---
rgmanager/include/resgroup.h | 1 +
rgmanager/include/reslist.h | 1 +
rgmanager/src/clulib/rg_strings.c | 1 +
rgmanager/src/daemons/groups.c | 3 +
rgmanager/src/daemons/restree.c | 14 ++++
rgmanager/src/daemons/rg_state.c | 119 ++++++++++++++++++++++++++++++++++++-
rgmanager/src/daemons/rg_thread.c | 20 ++++++
rgmanager/src/utils/clusvcadm.c | 7 ++-
8 files changed, 162 insertions(+), 4 deletions(-)
diff --git a/rgmanager/include/resgroup.h b/rgmanager/include/resgroup.h
index a0b6fc1..2e81eb9 100644
--- a/rgmanager/include/resgroup.h
+++ b/rgmanager/include/resgroup.h
@@ -157,6 +157,7 @@ int svc_disable(const char *svcName);
int svc_fail(const char *svcName);
int svc_freeze(const char *svcName);
int svc_unfreeze(const char *svcName);
+int svc_convalesce(const char *svcName);
int svc_migrate(const char *svcName, int target);
int svc_start_remote(const char *svcName, int request, uint32_t target);
int svc_report_failure(const char *svcName);
diff --git a/rgmanager/include/reslist.h b/rgmanager/include/reslist.h
index a8acd8e..921c483 100644
--- a/rgmanager/include/reslist.h
+++ b/rgmanager/include/reslist.h
@@ -130,6 +130,7 @@ int res_start(resource_node_t **tree, resource_t *res, void *ret);
int res_stop(resource_node_t **tree, resource_t *res, void *ret);
int res_status(resource_node_t **tree, resource_t *res, void *ret);
int res_status_inquiry(resource_node_t **tree, resource_t *res, void *ret);
+int res_convalesce(resource_node_t **tree, resource_t *res, void *ret);
int res_condstart(resource_node_t **tree, resource_t *res, void *ret);
int res_condstop(resource_node_t **tree, resource_t *res, void *ret);
int res_exec(resource_node_t *node, int op, const char *arg, int depth);
diff --git a/rgmanager/src/clulib/rg_strings.c b/rgmanager/src/clulib/rg_strings.c
index fdb1dad..b0cd184 100644
--- a/rgmanager/src/clulib/rg_strings.c
+++ b/rgmanager/src/clulib/rg_strings.c
@@ -58,6 +58,7 @@ const struct string_val rg_req_strings[] = {
{RG_QUERY_LOCK, "lock status inquiry"},
{RG_MIGRATE, "migrate"},
{RG_STATUS_INQUIRY, "out of band service status inquiry"},
+ {RG_CONVALESCE, "convalesce"},
{RG_NONE, "none"},
{0, NULL}
};
diff --git a/rgmanager/src/daemons/groups.c b/rgmanager/src/daemons/groups.c
index afce054..1d3c518 100644
--- a/rgmanager/src/daemons/groups.c
+++ b/rgmanager/src/daemons/groups.c
@@ -975,6 +975,9 @@ group_op(const char *groupname, int op)
case RG_CONDSTART:
ret = res_condstart(&_tree, res, NULL);
break;
+ case RG_CONVALESCE:
+ ret = res_convalesce(&_tree, res, NULL);
+ break;
}
pthread_rwlock_unlock(&resource_lock);
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index a717376..a5f27b8 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1615,6 +1615,20 @@ res_condstop(resource_node_t **tree, resource_t *res, void *ret)
/**
+ Repair/fix/convalesce all occurrences of a resource in a tree
+
+ @param tree Tree to search for our resource.
+ @param res Resource to start/stop
+ @param ret Unused
+ */
+int
+res_convalesce(resource_node_t **tree, resource_t *res, void *ret)
+{
+ return _res_op(tree, res, NULL, ret, RS_CONVALESCE);
+}
+
+
+/**
Start all occurrences of a resource in a tree
@param tree Tree to search for our resource.
diff --git a/rgmanager/src/daemons/rg_state.c b/rgmanager/src/daemons/rg_state.c
index 9049a7c..9eb686f 100644
--- a/rgmanager/src/daemons/rg_state.c
+++ b/rgmanager/src/daemons/rg_state.c
@@ -803,6 +803,92 @@ svc_start(const char *svcName, int req)
/**
+ * Fix stuff
+ */
+int
+svc_convalesce(const char *svcName)
+{
+ struct dlm_lksb lockp;
+ rg_state_t svcStatus;
+ int ret;
+
+ if (rg_lock(svcName, &lockp) < 0) {
+ logt_print(LOG_ERR, "#451: Unable to obtain cluster lock: %s\n",
+ strerror(errno));
+ return RG_EFAIL;
+ }
+
+ if (get_rg_state(svcName, &svcStatus) != 0) {
+ rg_unlock(&lockp);
+ logt_print(LOG_ERR, "#461: Failed getting status for RG %s\n",
+ svcName);
+ return RG_EFAIL;
+ }
+
+ switch(svcStatus.rs_state) {
+ case RG_STATE_STARTED:
+ break;
+ case RG_STATE_STARTING:
+ case RG_STATE_STOPPING:
+ case RG_STATE_RECOVER:
+ case RG_STATE_MIGRATE:
+ case RG_STATE_ERROR:
+ rg_unlock(&lockp);
+ return RG_EAGAIN;
+ default:
+ rg_unlock(&lockp);
+ return RG_EINVAL;
+ }
+
+ if (svcStatus.rs_flags & RG_FLAG_FROZEN) {
+ rg_unlock(&lockp);
+ return RG_EFROZEN;
+ }
+
+ if (svcStatus.rs_owner != (uint32_t)my_id()) {
+ rg_unlock(&lockp);
+ return RG_EFORWARD;
+ }
+
+ if (!(svcStatus.rs_flags & RG_FLAG_PARTIAL)) {
+ rg_unlock(&lockp);
+ return RG_ERUN;
+ }
+
+ rg_unlock(&lockp);
+
+ logt_print(LOG_INFO, "Repairing %s\n", svcName);
+ ret = group_op(svcName, RG_CONVALESCE);
+
+ switch(ret) {
+ default:
+ logt_print(LOG_WARNING, "Failed to repair %s\n", svcName);
+ /* Fail to restart a non-critical resource
+ * does not fail the service. */
+ return RG_EFAIL;
+ case 0:
+ logt_print(LOG_INFO, "Repair of %s was successful\n", svcName);
+ break;
+ }
+
+ /* Success - flip owner in state info */
+ if (rg_lock(svcName, &lockp) < 0) {
+ logt_print(LOG_ERR, "#455: Unable to obtain cluster lock: %s\n",
+ strerror(errno));
+ return RG_EFAIL;
+ }
+
+ /* No need for a 'get' here since the service is still STARTED */
+ svcStatus.rs_flags &= ~RG_FLAG_PARTIAL;
+
+ set_rg_state(svcName, &svcStatus);
+ rg_unlock(&lockp);
+
+ return 0;
+}
+
+
+/**
* Migrate a service to another node. Relies on agent
* operating synchronously
*/
@@ -1111,8 +1197,31 @@ svc_status(const char *svcName)
}
/* For running services, if the return code is 0, we're done*/
- if (svcStatus.rs_state == RG_STATE_STARTED)
- return handle_started_status(svcName, ret, &svcStatus);
+ if (svcStatus.rs_state == RG_STATE_STARTED) {
+ ret = handle_started_status(svcName, ret, &svcStatus);
+
+ if (ret & SFL_PARTIAL) {
+ ret &= ~SFL_PARTIAL;
+ svcStatus.rs_flags |= RG_FLAG_PARTIAL;
+ if (rg_lock(svcName, &lockp) < 0) {
+ logt_print(LOG_ERR,
+ "#481: Unable to obtain cluster lock: %s\n",
+ strerror(errno));
+ return RG_EFAIL;
+ }
+
+ if (set_rg_state(svcName, &svcStatus) != 0) {
+ rg_unlock(&lockp);
+ logt_print(LOG_ERR,
+ "#482: Failed setting status for RG %s\n",
+ svcName);
+ return RG_EFAIL;
+ }
+ rg_unlock(&lockp);
+ }
+
+ return ret;
+ }
return handle_migrate_status(svcName, ret, &svcStatus);
}
@@ -1153,9 +1262,10 @@ handle_started_status(const char *svcName, int ret,
svcName);
if (ret & SFL_PARTIAL) {
logt_print(LOG_NOTICE, "Note: Some non-critical "
- "resources are still stopped.\n");
+ "resources were stopped during recovery.\n");
logt_print(LOG_NOTICE, "Run 'clusvcadm -c %s' to "
"restore them to operation.\n", svcName);
+ return SFL_PARTIAL;
}
return 0;
@@ -1262,6 +1372,7 @@ _svc_stop(const char *svcName, int req, int recover, uint32_t newstate)
svcStatus.rs_last_owner = svcStatus.rs_owner;
svcStatus.rs_owner = 0;
svcStatus.rs_state = RG_STATE_STOPPED;
+ svcStatus.rs_flags = 0;
if (set_rg_state(svcName, &svcStatus) != 0) {
rg_unlock(&lockp);
return RG_EFAIL;
@@ -1292,6 +1403,7 @@ _svc_stop(const char *svcName, int req, int recover, uint32_t newstate)
logt_print(LOG_DEBUG, "%s is clean; skipping double-stop\n",
svcName);
svcStatus.rs_state = newstate;
+ svcStatus.rs_flags = 0;
if (set_rg_state(svcName, &svcStatus) != 0) {
rg_unlock(&lockp);
@@ -1372,6 +1484,7 @@ _svc_stop_finish(const char *svcName, int failed, uint32_t newstate)
}
svcStatus.rs_state = newstate;
+ svcStatus.rs_flags = 0;
logt_print(LOG_NOTICE, "Service %s is %s\n", svcName,
rg_state_str(svcStatus.rs_state));
diff --git a/rgmanager/src/daemons/rg_thread.c b/rgmanager/src/daemons/rg_thread.c
index 446f2d2..72b5f96 100644
--- a/rgmanager/src/daemons/rg_thread.c
+++ b/rgmanager/src/daemons/rg_thread.c
@@ -277,6 +277,26 @@ resgroup_thread_main(void *arg)
ret = RG_NONE;
break;
+ case RG_CONVALESCE:
+ error = svc_convalesce(myname);
+
+ if (error == 0) {
+ ret = RG_SUCCESS;
+
+ pthread_mutex_lock(&my_queue_mutex);
+ purge_status_checks(&my_queue);
+ pthread_mutex_unlock(&my_queue_mutex);
+ } else if (error == RG_EFORWARD) {
+ ret = RG_NONE;
+ break;
+ } else {
+ /*
+ * Bad news.
+ */
+ ret = RG_EFAIL;
+ }
+ break;
+
case RG_MIGRATE:
error = svc_migrate(myname, req->rr_target);
diff --git a/rgmanager/src/utils/clusvcadm.c b/rgmanager/src/utils/clusvcadm.c
index 137837a..bd0723f 100644
--- a/rgmanager/src/utils/clusvcadm.c
+++ b/rgmanager/src/utils/clusvcadm.c
@@ -235,7 +235,7 @@ main(int argc, char **argv)
const char *actionstr = NULL;
cluster_member_list_t *membership;
- while ((opt = getopt(argc, argv, "lSue:M:d:r:n:m:FvR:s:Z:U:qh?")) != EOF) {
+ while ((opt = getopt(argc, argv, "lSue:M:d:r:n:c:m:FvR:s:Z:U:qh?")) != EOF) {
switch (opt) {
case 'l':
return do_lock();
@@ -312,6 +312,11 @@ main(int argc, char **argv)
action = RG_UNFREEZE;
svcname = optarg;
break;
+ case 'c':
+ actionstr = "convalescing";
+ action = RG_CONVALESCE;
+ svcname = optarg;
+ break;
case 'q':
close(STDOUT_FILENO);
break;
More information about the cluster-commits
mailing list