cluster: RHEL6 - rgmanager: Add convalesce operation

Lon Hohberger lon at fedoraproject.org
Tue Feb 1 18:10:39 UTC 2011


Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=abe50ac2721ec8124aa2a614c2a0a05e4cfa3ad7
Commit:        abe50ac2721ec8124aa2a614c2a0a05e4cfa3ad7
Parent:        b631ffdb818f7cf3512840dd99b8844aa230b03d
Author:        Lon Hohberger <lhh at redhat.com>
AuthorDate:    Wed Sep 1 15:24:22 2010 -0400
Committer:     Lon Hohberger <lhh at redhat.com>
CommitterDate: Tue Feb 1 12:31:11 2011 -0500

rgmanager: Add convalesce operation

This option enables the administrator to repair
a partially-failed service without affecting
the rest of the service.

Resolves: rhbz#634277

Signed-off-by: Lon Hohberger <lhh at redhat.com>
---
 rgmanager/include/resgroup.h      |    1 +
 rgmanager/include/reslist.h       |    1 +
 rgmanager/src/clulib/rg_strings.c |    1 +
 rgmanager/src/daemons/groups.c    |    3 +
 rgmanager/src/daemons/restree.c   |   14 ++++
 rgmanager/src/daemons/rg_state.c  |  119 ++++++++++++++++++++++++++++++++++++-
 rgmanager/src/daemons/rg_thread.c |   20 ++++++
 rgmanager/src/utils/clusvcadm.c   |    7 ++-
 8 files changed, 162 insertions(+), 4 deletions(-)

diff --git a/rgmanager/include/resgroup.h b/rgmanager/include/resgroup.h
index a0b6fc1..2e81eb9 100644
--- a/rgmanager/include/resgroup.h
+++ b/rgmanager/include/resgroup.h
@@ -157,6 +157,7 @@ int svc_disable(const char *svcName);
 int svc_fail(const char *svcName);
 int svc_freeze(const char *svcName);
 int svc_unfreeze(const char *svcName);
+int svc_convalesce(const char *svcName);
 int svc_migrate(const char *svcName, int target);
 int svc_start_remote(const char *svcName, int request, uint32_t target);
 int svc_report_failure(const char *svcName);
diff --git a/rgmanager/include/reslist.h b/rgmanager/include/reslist.h
index a8acd8e..921c483 100644
--- a/rgmanager/include/reslist.h
+++ b/rgmanager/include/reslist.h
@@ -130,6 +130,7 @@ int res_start(resource_node_t **tree, resource_t *res, void *ret);
 int res_stop(resource_node_t **tree, resource_t *res, void *ret);
 int res_status(resource_node_t **tree, resource_t *res, void *ret);
 int res_status_inquiry(resource_node_t **tree, resource_t *res, void *ret);
+int res_convalesce(resource_node_t **tree, resource_t *res, void *ret);
 int res_condstart(resource_node_t **tree, resource_t *res, void *ret);
 int res_condstop(resource_node_t **tree, resource_t *res, void *ret);
 int res_exec(resource_node_t *node, int op, const char *arg, int depth);
diff --git a/rgmanager/src/clulib/rg_strings.c b/rgmanager/src/clulib/rg_strings.c
index fdb1dad..b0cd184 100644
--- a/rgmanager/src/clulib/rg_strings.c
+++ b/rgmanager/src/clulib/rg_strings.c
@@ -58,6 +58,7 @@ const struct string_val rg_req_strings[] = {
 	{RG_QUERY_LOCK, "lock status inquiry"},
 	{RG_MIGRATE, "migrate"},
 	{RG_STATUS_INQUIRY, "out of band service status inquiry"},
+	{RG_CONVALESCE, "convalesce"},
 	{RG_NONE, "none"},
 	{0, NULL}
 };
diff --git a/rgmanager/src/daemons/groups.c b/rgmanager/src/daemons/groups.c
index afce054..1d3c518 100644
--- a/rgmanager/src/daemons/groups.c
+++ b/rgmanager/src/daemons/groups.c
@@ -975,6 +975,9 @@ group_op(const char *groupname, int op)
 	case RG_CONDSTART:
 		ret = res_condstart(&_tree, res, NULL);
 		break;
+	case RG_CONVALESCE:
+		ret = res_convalesce(&_tree, res, NULL);
+		break;
 	}
 	pthread_rwlock_unlock(&resource_lock);
 
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index a717376..a5f27b8 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1615,6 +1615,20 @@ res_condstop(resource_node_t **tree, resource_t *res, void *ret)
 
 
 /**
+   Repair/fix/convalesce all occurrences of a resource in a tree
+
+   @param tree		Tree to search for our resource.
+   @param res		Resource to start/stop
+   @param ret		Unused
+ */
+int
+res_convalesce(resource_node_t **tree, resource_t *res, void *ret)
+{
+	return _res_op(tree, res, NULL, ret, RS_CONVALESCE);
+}
+
+
+/**
    Start all occurrences of a resource in a tree
 
    @param tree		Tree to search for our resource.
diff --git a/rgmanager/src/daemons/rg_state.c b/rgmanager/src/daemons/rg_state.c
index 9049a7c..9eb686f 100644
--- a/rgmanager/src/daemons/rg_state.c
+++ b/rgmanager/src/daemons/rg_state.c
@@ -803,6 +803,92 @@ svc_start(const char *svcName, int req)
 
 
 /**
+ * Fix stuff
+ */
+int
+svc_convalesce(const char *svcName)
+{
+	struct dlm_lksb lockp;
+	rg_state_t svcStatus;
+	int ret;
+
+	if (rg_lock(svcName, &lockp) < 0) {
+		logt_print(LOG_ERR, "#451: Unable to obtain cluster lock: %s\n",
+		       strerror(errno));
+		return RG_EFAIL;
+	}
+
+	if (get_rg_state(svcName, &svcStatus) != 0) {
+		rg_unlock(&lockp);
+		logt_print(LOG_ERR, "#461: Failed getting status for RG %s\n",
+		       svcName);
+		return RG_EFAIL;
+	}
+
+	switch(svcStatus.rs_state) {
+	case RG_STATE_STARTED:
+		break;
+	case RG_STATE_STARTING:
+	case RG_STATE_STOPPING:
+	case RG_STATE_RECOVER:
+	case RG_STATE_MIGRATE:
+	case RG_STATE_ERROR:
+		rg_unlock(&lockp);
+		return RG_EAGAIN;
+	default:
+		rg_unlock(&lockp);
+		return RG_EINVAL;
+	}
+
+	if (svcStatus.rs_flags & RG_FLAG_FROZEN) {
+		rg_unlock(&lockp);
+		return RG_EFROZEN;
+	}
+
+	if (svcStatus.rs_owner != (uint32_t)my_id()) {
+		rg_unlock(&lockp);
+		return RG_EFORWARD;
+	}
+
+	if (!(svcStatus.rs_flags & RG_FLAG_PARTIAL)) {
+		rg_unlock(&lockp);
+		return RG_ERUN;
+	}
+
+	rg_unlock(&lockp);
+
+	logt_print(LOG_INFO, "Repairing %s\n", svcName);
+	ret = group_op(svcName, RG_CONVALESCE);
+
+	switch(ret) {
+	default:
+		logt_print(LOG_WARNING, "Failed to repair %s\n", svcName);
+		/* Fail to restart a non-critical resource
+		 * does not fail the service. */
+		return RG_EFAIL;
+	case 0:
+		logt_print(LOG_INFO, "Repair of %s was successful\n", svcName);
+		break;
+	}
+
+	/* Success - flip owner in state info */
+	if (rg_lock(svcName, &lockp) < 0) {
+		logt_print(LOG_ERR, "#455: Unable to obtain cluster lock: %s\n",
+			   strerror(errno));
+		return RG_EFAIL;
+	}
+
+	/* No need for a 'get' here since the service is still STARTED */
+	svcStatus.rs_flags &= ~RG_FLAG_PARTIAL;
+
+	set_rg_state(svcName, &svcStatus);
+	rg_unlock(&lockp);
+
+	return 0;
+}
+
+
+/**
  * Migrate a service to another node.  Relies on agent
  * operating synchronously
  */
@@ -1111,8 +1197,31 @@ svc_status(const char *svcName)
 	}
 
 	/* For running services, if the return code is 0, we're done*/
-	if (svcStatus.rs_state == RG_STATE_STARTED)
-		return handle_started_status(svcName, ret, &svcStatus);
+	if (svcStatus.rs_state == RG_STATE_STARTED) {
+		ret = handle_started_status(svcName, ret, &svcStatus);
+
+		if (ret & SFL_PARTIAL) {
+			ret &= ~SFL_PARTIAL;
+			svcStatus.rs_flags |= RG_FLAG_PARTIAL;
+			if (rg_lock(svcName, &lockp) < 0) {
+				logt_print(LOG_ERR,
+					   "#481: Unable to obtain cluster lock: %s\n",
+					   strerror(errno));
+				return RG_EFAIL;
+			}
+
+			if (set_rg_state(svcName, &svcStatus) != 0) {
+				rg_unlock(&lockp);
+				logt_print(LOG_ERR,
+					   "#482: Failed setting status for RG %s\n",
+					   svcName);
+				return RG_EFAIL;
+			}
+			rg_unlock(&lockp);
+		}
+
+		return ret;
+	}
 	
 	return handle_migrate_status(svcName, ret, &svcStatus);
 }
@@ -1153,9 +1262,10 @@ handle_started_status(const char *svcName, int ret,
 			   svcName);
 		if (ret & SFL_PARTIAL) {
 			logt_print(LOG_NOTICE, "Note: Some non-critical "
-				   "resources are still stopped.\n");
+				   "resources were stopped during recovery.\n");
 			logt_print(LOG_NOTICE, "Run 'clusvcadm -c %s' to "
 				   "restore them to operation.\n", svcName);
+			return SFL_PARTIAL;
 		}
 
 		return 0;
@@ -1262,6 +1372,7 @@ _svc_stop(const char *svcName, int req, int recover, uint32_t newstate)
 		svcStatus.rs_last_owner = svcStatus.rs_owner;
 		svcStatus.rs_owner = 0;
 		svcStatus.rs_state = RG_STATE_STOPPED;
+		svcStatus.rs_flags = 0;
 		if (set_rg_state(svcName, &svcStatus) != 0) {
 			rg_unlock(&lockp);
 			return RG_EFAIL;
@@ -1292,6 +1403,7 @@ _svc_stop(const char *svcName, int req, int recover, uint32_t newstate)
 		logt_print(LOG_DEBUG, "%s is clean; skipping double-stop\n",
 		       svcName);
 		svcStatus.rs_state = newstate;
+		svcStatus.rs_flags = 0;
 
 		if (set_rg_state(svcName, &svcStatus) != 0) {
 			rg_unlock(&lockp);
@@ -1372,6 +1484,7 @@ _svc_stop_finish(const char *svcName, int failed, uint32_t newstate)
 	}
 
 	svcStatus.rs_state = newstate;
+	svcStatus.rs_flags = 0;
 
 	logt_print(LOG_NOTICE, "Service %s is %s\n", svcName,
 	       rg_state_str(svcStatus.rs_state));
diff --git a/rgmanager/src/daemons/rg_thread.c b/rgmanager/src/daemons/rg_thread.c
index 446f2d2..72b5f96 100644
--- a/rgmanager/src/daemons/rg_thread.c
+++ b/rgmanager/src/daemons/rg_thread.c
@@ -277,6 +277,26 @@ resgroup_thread_main(void *arg)
 				ret = RG_NONE;
 			break;
 
+		case RG_CONVALESCE:
+			error = svc_convalesce(myname);
+
+			if (error == 0) {
+				ret = RG_SUCCESS;
+
+				pthread_mutex_lock(&my_queue_mutex);
+				purge_status_checks(&my_queue);
+				pthread_mutex_unlock(&my_queue_mutex);
+			} else if (error == RG_EFORWARD) {
+				ret = RG_NONE;
+				break;
+			} else {
+				/*
+				 * Bad news.
+				 */
+				ret = RG_EFAIL;
+			}
+			break;
+
 		case RG_MIGRATE:
 			error = svc_migrate(myname, req->rr_target);
 
diff --git a/rgmanager/src/utils/clusvcadm.c b/rgmanager/src/utils/clusvcadm.c
index 137837a..bd0723f 100644
--- a/rgmanager/src/utils/clusvcadm.c
+++ b/rgmanager/src/utils/clusvcadm.c
@@ -235,7 +235,7 @@ main(int argc, char **argv)
 	const char *actionstr = NULL;
 	cluster_member_list_t *membership;
 
-	while ((opt = getopt(argc, argv, "lSue:M:d:r:n:m:FvR:s:Z:U:qh?")) != EOF) {
+	while ((opt = getopt(argc, argv, "lSue:M:d:r:n:c:m:FvR:s:Z:U:qh?")) != EOF) {
 		switch (opt) {
 		case 'l':
 			return do_lock();
@@ -312,6 +312,11 @@ main(int argc, char **argv)
 			action = RG_UNFREEZE;
 			svcname = optarg;
 			break;
+		case 'c':
+			actionstr = "convalescing";
+			action = RG_CONVALESCE;
+			svcname = optarg;
+			break;
 		case 'q':
 			close(STDOUT_FILENO);
 			break;


More information about the cluster-commits mailing list