Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=abe... Commit: abe50ac2721ec8124aa2a614c2a0a05e4cfa3ad7 Parent: b631ffdb818f7cf3512840dd99b8844aa230b03d Author: Lon Hohberger lhh@redhat.com AuthorDate: Wed Sep 1 15:24:22 2010 -0400 Committer: Lon Hohberger lhh@redhat.com CommitterDate: Tue Feb 1 12:31:11 2011 -0500
rgmanager: Add convalesce operation
This option enables the administrator to repair a partially-failed service without affecting the rest of the service.
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger lhh@redhat.com --- rgmanager/include/resgroup.h | 1 + rgmanager/include/reslist.h | 1 + rgmanager/src/clulib/rg_strings.c | 1 + rgmanager/src/daemons/groups.c | 3 + rgmanager/src/daemons/restree.c | 14 ++++ rgmanager/src/daemons/rg_state.c | 119 ++++++++++++++++++++++++++++++++++++- rgmanager/src/daemons/rg_thread.c | 20 ++++++ rgmanager/src/utils/clusvcadm.c | 7 ++- 8 files changed, 162 insertions(+), 4 deletions(-)
diff --git a/rgmanager/include/resgroup.h b/rgmanager/include/resgroup.h index a0b6fc1..2e81eb9 100644 --- a/rgmanager/include/resgroup.h +++ b/rgmanager/include/resgroup.h @@ -157,6 +157,7 @@ int svc_disable(const char *svcName); int svc_fail(const char *svcName); int svc_freeze(const char *svcName); int svc_unfreeze(const char *svcName); +int svc_convalesce(const char *svcName); int svc_migrate(const char *svcName, int target); int svc_start_remote(const char *svcName, int request, uint32_t target); int svc_report_failure(const char *svcName); diff --git a/rgmanager/include/reslist.h b/rgmanager/include/reslist.h index a8acd8e..921c483 100644 --- a/rgmanager/include/reslist.h +++ b/rgmanager/include/reslist.h @@ -130,6 +130,7 @@ int res_start(resource_node_t **tree, resource_t *res, void *ret); int res_stop(resource_node_t **tree, resource_t *res, void *ret); int res_status(resource_node_t **tree, resource_t *res, void *ret); int res_status_inquiry(resource_node_t **tree, resource_t *res, void *ret); +int res_convalesce(resource_node_t **tree, resource_t *res, void *ret); int res_condstart(resource_node_t **tree, resource_t *res, void *ret); int res_condstop(resource_node_t **tree, resource_t *res, void *ret); int res_exec(resource_node_t *node, int op, const char *arg, int depth); diff --git a/rgmanager/src/clulib/rg_strings.c b/rgmanager/src/clulib/rg_strings.c index fdb1dad..b0cd184 100644 --- a/rgmanager/src/clulib/rg_strings.c +++ b/rgmanager/src/clulib/rg_strings.c @@ -58,6 +58,7 @@ const struct string_val rg_req_strings[] = { {RG_QUERY_LOCK, "lock status inquiry"}, {RG_MIGRATE, "migrate"}, {RG_STATUS_INQUIRY, "out of band service status inquiry"}, + {RG_CONVALESCE, "convalesce"}, {RG_NONE, "none"}, {0, NULL} }; diff --git a/rgmanager/src/daemons/groups.c b/rgmanager/src/daemons/groups.c index afce054..1d3c518 100644 --- a/rgmanager/src/daemons/groups.c +++ b/rgmanager/src/daemons/groups.c @@ -975,6 +975,9 @@ group_op(const char *groupname, int op) case RG_CONDSTART: ret = res_condstart(&_tree, res, NULL); break; + case RG_CONVALESCE: + ret = res_convalesce(&_tree, res, NULL); + break; } pthread_rwlock_unlock(&resource_lock);
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c index a717376..a5f27b8 100644 --- a/rgmanager/src/daemons/restree.c +++ b/rgmanager/src/daemons/restree.c @@ -1615,6 +1615,20 @@ res_condstop(resource_node_t **tree, resource_t *res, void *ret)
/** + Repair/fix/convalesce all occurrences of a resource in a tree + + @param tree Tree to search for our resource. + @param res Resource to start/stop + @param ret Unused + */ +int +res_convalesce(resource_node_t **tree, resource_t *res, void *ret) +{ + return _res_op(tree, res, NULL, ret, RS_CONVALESCE); +} + + +/** Start all occurrences of a resource in a tree
@param tree Tree to search for our resource. diff --git a/rgmanager/src/daemons/rg_state.c b/rgmanager/src/daemons/rg_state.c index 9049a7c..9eb686f 100644 --- a/rgmanager/src/daemons/rg_state.c +++ b/rgmanager/src/daemons/rg_state.c @@ -803,6 +803,92 @@ svc_start(const char *svcName, int req)
/** + * Fix stuff + */ +int +svc_convalesce(const char *svcName) +{ + struct dlm_lksb lockp; + rg_state_t svcStatus; + int ret; + + if (rg_lock(svcName, &lockp) < 0) { + logt_print(LOG_ERR, "#451: Unable to obtain cluster lock: %s\n", + strerror(errno)); + return RG_EFAIL; + } + + if (get_rg_state(svcName, &svcStatus) != 0) { + rg_unlock(&lockp); + logt_print(LOG_ERR, "#461: Failed getting status for RG %s\n", + svcName); + return RG_EFAIL; + } + + switch(svcStatus.rs_state) { + case RG_STATE_STARTED: + break; + case RG_STATE_STARTING: + case RG_STATE_STOPPING: + case RG_STATE_RECOVER: + case RG_STATE_MIGRATE: + case RG_STATE_ERROR: + rg_unlock(&lockp); + return RG_EAGAIN; + default: + rg_unlock(&lockp); + return RG_EINVAL; + } + + if (svcStatus.rs_flags & RG_FLAG_FROZEN) { + rg_unlock(&lockp); + return RG_EFROZEN; + } + + if (svcStatus.rs_owner != (uint32_t)my_id()) { + rg_unlock(&lockp); + return RG_EFORWARD; + } + + if (!(svcStatus.rs_flags & RG_FLAG_PARTIAL)) { + rg_unlock(&lockp); + return RG_ERUN; + } + + rg_unlock(&lockp); + + logt_print(LOG_INFO, "Repairing %s\n", svcName); + ret = group_op(svcName, RG_CONVALESCE); + + switch(ret) { + default: + logt_print(LOG_WARNING, "Failed to repair %s\n", svcName); + /* Fail to restart a non-critical resource + * does not fail the service. */ + return RG_EFAIL; + case 0: + logt_print(LOG_INFO, "Repair of %s was successful\n", svcName); + break; + } + + /* Success - flip owner in state info */ + if (rg_lock(svcName, &lockp) < 0) { + logt_print(LOG_ERR, "#455: Unable to obtain cluster lock: %s\n", + strerror(errno)); + return RG_EFAIL; + } + + /* No need for a 'get' here since the service is still STARTED */ + svcStatus.rs_flags &= ~RG_FLAG_PARTIAL; + + set_rg_state(svcName, &svcStatus); + rg_unlock(&lockp); + + return 0; +} + + +/** * Migrate a service to another node. Relies on agent * operating synchronously */ @@ -1111,8 +1197,31 @@ svc_status(const char *svcName) }
/* For running services, if the return code is 0, we're done*/ - if (svcStatus.rs_state == RG_STATE_STARTED) - return handle_started_status(svcName, ret, &svcStatus); + if (svcStatus.rs_state == RG_STATE_STARTED) { + ret = handle_started_status(svcName, ret, &svcStatus); + + if (ret & SFL_PARTIAL) { + ret &= ~SFL_PARTIAL; + svcStatus.rs_flags |= RG_FLAG_PARTIAL; + if (rg_lock(svcName, &lockp) < 0) { + logt_print(LOG_ERR, + "#481: Unable to obtain cluster lock: %s\n", + strerror(errno)); + return RG_EFAIL; + } + + if (set_rg_state(svcName, &svcStatus) != 0) { + rg_unlock(&lockp); + logt_print(LOG_ERR, + "#482: Failed setting status for RG %s\n", + svcName); + return RG_EFAIL; + } + rg_unlock(&lockp); + } + + return ret; + } return handle_migrate_status(svcName, ret, &svcStatus); } @@ -1153,9 +1262,10 @@ handle_started_status(const char *svcName, int ret, svcName); if (ret & SFL_PARTIAL) { logt_print(LOG_NOTICE, "Note: Some non-critical " - "resources are still stopped.\n"); + "resources were stopped during recovery.\n"); logt_print(LOG_NOTICE, "Run 'clusvcadm -c %s' to " "restore them to operation.\n", svcName); + return SFL_PARTIAL; }
return 0; @@ -1262,6 +1372,7 @@ _svc_stop(const char *svcName, int req, int recover, uint32_t newstate) svcStatus.rs_last_owner = svcStatus.rs_owner; svcStatus.rs_owner = 0; svcStatus.rs_state = RG_STATE_STOPPED; + svcStatus.rs_flags = 0; if (set_rg_state(svcName, &svcStatus) != 0) { rg_unlock(&lockp); return RG_EFAIL; @@ -1292,6 +1403,7 @@ _svc_stop(const char *svcName, int req, int recover, uint32_t newstate) logt_print(LOG_DEBUG, "%s is clean; skipping double-stop\n", svcName); svcStatus.rs_state = newstate; + svcStatus.rs_flags = 0;
if (set_rg_state(svcName, &svcStatus) != 0) { rg_unlock(&lockp); @@ -1372,6 +1484,7 @@ _svc_stop_finish(const char *svcName, int failed, uint32_t newstate) }
svcStatus.rs_state = newstate; + svcStatus.rs_flags = 0;
logt_print(LOG_NOTICE, "Service %s is %s\n", svcName, rg_state_str(svcStatus.rs_state)); diff --git a/rgmanager/src/daemons/rg_thread.c b/rgmanager/src/daemons/rg_thread.c index 446f2d2..72b5f96 100644 --- a/rgmanager/src/daemons/rg_thread.c +++ b/rgmanager/src/daemons/rg_thread.c @@ -277,6 +277,26 @@ resgroup_thread_main(void *arg) ret = RG_NONE; break;
+ case RG_CONVALESCE: + error = svc_convalesce(myname); + + if (error == 0) { + ret = RG_SUCCESS; + + pthread_mutex_lock(&my_queue_mutex); + purge_status_checks(&my_queue); + pthread_mutex_unlock(&my_queue_mutex); + } else if (error == RG_EFORWARD) { + ret = RG_NONE; + break; + } else { + /* + * Bad news. + */ + ret = RG_EFAIL; + } + break; + case RG_MIGRATE: error = svc_migrate(myname, req->rr_target);
diff --git a/rgmanager/src/utils/clusvcadm.c b/rgmanager/src/utils/clusvcadm.c index 137837a..bd0723f 100644 --- a/rgmanager/src/utils/clusvcadm.c +++ b/rgmanager/src/utils/clusvcadm.c @@ -235,7 +235,7 @@ main(int argc, char **argv) const char *actionstr = NULL; cluster_member_list_t *membership;
- while ((opt = getopt(argc, argv, "lSue:M:d:r:n:m:FvR:s:Z:U:qh?")) != EOF) { + while ((opt = getopt(argc, argv, "lSue:M:d:r:n:c:m:FvR:s:Z:U:qh?")) != EOF) { switch (opt) { case 'l': return do_lock(); @@ -312,6 +312,11 @@ main(int argc, char **argv) action = RG_UNFREEZE; svcname = optarg; break; + case 'c': + actionstr = "convalescing"; + action = RG_CONVALESCE; + svcname = optarg; + break; case 'q': close(STDOUT_FILENO); break;
cluster-commits@lists.fedorahosted.org