cluster: RHEL6 - rgmanager: Add -c option to man page & clusvcadm -h
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 4e2261f72411aae2604d9d3b771f221b11ef4b6b
Parent: 3b03b46fe7d3c7d747db9a2b7721cc56aef458f2
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Fri Sep 10 11:49:10 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:32:49 2011 -0500
rgmanager: Add -c option to man page & clusvcadm -h
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/man/clusvcadm.8 | 9 +++++++++
rgmanager/src/utils/clusvcadm.c | 3 +++
2 files changed, 12 insertions(+), 0 deletions(-)
diff --git a/rgmanager/man/clusvcadm.8 b/rgmanager/man/clusvcadm.8
index d1435ef..d5e09eb 100644
--- a/rgmanager/man/clusvcadm.8
+++ b/rgmanager/man/clusvcadm.8
@@ -33,6 +33,9 @@ clusvcadm \- Cluster User Service Administration Utility
.B [\-s
.I <service>
.B ]
+.B [\-c
+.I <service>
+.B ]
.B [\-v]
.SH "DESCRIPTION"
@@ -117,6 +120,12 @@ Stops the service named
service
until a member transition or until it is enabled again.
+.IP "\-c <service>"
+Attempt to convalesce the named
+.I
+service
+by restarting failed, non-critical components.
+
.IP \-v
Display version information and exit.
diff --git a/rgmanager/src/utils/clusvcadm.c b/rgmanager/src/utils/clusvcadm.c
index bd0723f..8d0b502 100644
--- a/rgmanager/src/utils/clusvcadm.c
+++ b/rgmanager/src/utils/clusvcadm.c
@@ -167,6 +167,9 @@ printf(" -Z <group> Freeze resource group. This prevents\n"
" stopping the whole service.\n");
printf(" -U <group> Unfreeze (thaw) resource group. Restores\n"
" a group to normal operation.\n");
+printf(" -c <group> Convalesce (repair, fix) resource group.\n"
+ " Attempts to start failed, non-critical \n"
+ " resources within a resource group.\n");
printf("Resource Group Locking (for cluster Shutdown / Debugging):\n");
printf(" -l Lock local resource group managers.\n"
13 years, 4 months
cluster: RHEL6 - rgmanager: Support convalesce w/ central_processing
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 3b03b46fe7d3c7d747db9a2b7721cc56aef458f2
Parent: c2fa8fe7c8f2a3cbf1023a170f3f78a8de559b7a
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Thu Sep 9 10:58:17 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:32:31 2011 -0500
rgmanager: Support convalesce w/ central_processing
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/include/event.h | 1 +
rgmanager/src/daemons/service_op.c | 84 +++++++++++++++++++++++
rgmanager/src/daemons/slang_event.c | 18 +++++-
rgmanager/src/resources/default_event_script.sl | 4 +
4 files changed, 106 insertions(+), 1 deletions(-)
diff --git a/rgmanager/include/event.h b/rgmanager/include/event.h
index a9a8235..ad3fe39 100644
--- a/rgmanager/include/event.h
+++ b/rgmanager/include/event.h
@@ -124,6 +124,7 @@ void set_transition_throttling(int nsecs);
int service_op_start(char *svcName, int *target_list, int target_list_len,
int *new_owner);
int service_op_stop(char *svcName, int do_disable, int event_type);
+int service_op_convalesce(const char *svcName);
int service_op_migrate(char *svcName, int target_node);
diff --git a/rgmanager/src/daemons/service_op.c b/rgmanager/src/daemons/service_op.c
index 112b267..f094129 100644
--- a/rgmanager/src/daemons/service_op.c
+++ b/rgmanager/src/daemons/service_op.c
@@ -186,6 +186,90 @@ service_op_stop(char *svcName, int do_disable, int event_type)
}
+int
+service_op_convalesce(const char *svcName)
+{
+ SmMessageSt msg;
+ int msg_ret;
+ msgctx_t ctx;
+ rg_state_t svcStatus;
+ int msgtarget = my_id();
+
+ /* Build the message header */
+ msg.sm_hdr.gh_magic = GENERIC_HDR_MAGIC;
+ msg.sm_hdr.gh_command = RG_ACTION_REQUEST;
+ msg.sm_hdr.gh_arg1 = RG_ACTION_MASTER;
+ msg.sm_hdr.gh_length = sizeof (SmMessageSt);
+
+ msg.sm_data.d_action = RG_CONVALESCE;
+
+ strncpy(msg.sm_data.d_svcName, svcName,
+ sizeof(msg.sm_data.d_svcName));
+ msg.sm_data.d_ret = 0;
+ msg.sm_data.d_svcOwner = 0;
+
+ /* Open a connection to the local node - it will decide what to
+ do in this case. XXX inefficient; should queue requests
+ locally and immediately forward requests otherwise */
+
+ if (get_service_state_internal(svcName, &svcStatus) < 0)
+ return RG_EFAIL;
+ if (svcStatus.rs_owner > 0) {
+ if (member_online(svcStatus.rs_owner)) {
+ msgtarget = svcStatus.rs_owner;
+ } else {
+ /* If the owner is not online,
+ mark the service as 'stopped' but
+ otherwise, do nothing.
+ */
+ return svc_stop(svcName, RG_STOP);
+ }
+ }
+
+ if (msg_open(MSG_CLUSTER, msgtarget, RG_PORT, &ctx, 2 * cluster_timeout)< 0) {
+ logt_print(LOG_ERR,
+ "#58: Failed opening connection to member #%d\n",
+ my_id());
+ return -1;
+ }
+
+ /* Encode */
+ swab_SmMessageSt(&msg);
+
+ /* Send stop message to the other node */
+ if (msg_send(&ctx, &msg, sizeof (SmMessageSt)) <
+ (int)sizeof (SmMessageSt)) {
+ logt_print(LOG_ERR, "Failed to send complete message\n");
+ msg_close(&ctx);
+ return -1;
+ }
+
+ /* Check the response */
+ do {
+ msg_ret = msg_receive(&ctx, &msg,
+ sizeof (SmMessageSt), 10);
+ if ((msg_ret == -1 && errno != ETIMEDOUT) ||
+ (msg_ret > 0)) {
+ break;
+ }
+ } while(1);
+
+ if (msg_ret != sizeof (SmMessageSt)) {
+ logt_print(LOG_WARNING, "Strange response size: %d vs %d\n",
+ msg_ret, (int)sizeof(SmMessageSt));
+ return 0; /* XXX really UNKNOWN */
+ }
+
+ /* Got a valid response from other node. */
+ msg_close(&ctx);
+
+ /* Decode */
+ swab_SmMessageSt(&msg);
+
+ return msg.sm_data.d_ret;
+}
+
+
/*
service_op_migrate() - send a virtual machine to another host
in the cluster
diff --git a/rgmanager/src/daemons/slang_event.c b/rgmanager/src/daemons/slang_event.c
index 55f0341..0b9f0d0 100644
--- a/rgmanager/src/daemons/slang_event.c
+++ b/rgmanager/src/daemons/slang_event.c
@@ -86,7 +86,8 @@ static int
_user_restart = RG_RESTART,
_user_migrate = RG_MIGRATE,
_user_freeze = RG_FREEZE,
- _user_unfreeze = RG_UNFREEZE;
+ _user_unfreeze = RG_UNFREEZE,
+ _user_convalesce = RG_CONVALESCE;
SLang_Intrin_Var_Type rgmanager_vars[] =
@@ -139,6 +140,7 @@ SLang_Intrin_Var_Type rgmanager_vars[] =
MAKE_VARIABLE((char *)"USER_MIGRATE", &_user_migrate, SLANG_INT_TYPE, 1),
MAKE_VARIABLE((char *)"USER_FREEZE", &_user_freeze, SLANG_INT_TYPE, 1),
MAKE_VARIABLE((char *)"USER_UNFREEZE", &_user_unfreeze,SLANG_INT_TYPE, 1),
+ MAKE_VARIABLE((char *)"USER_CONVALESCE",&_user_convalesce,SLANG_INT_TYPE, 1),
/* Errors */
MAKE_VARIABLE((char *)"rg_error", &_rg_err, SLANG_INT_TYPE, 1),
@@ -444,6 +446,18 @@ sl_service_property(char *svcName, char *prop)
/**
usage:
+ service_convalesce(name);
+ */
+static int
+sl_convalesce_service(const char *svcname)
+{
+ return service_op_convalesce(svcname);
+}
+
+
+/**
+ usage:
+
stop_service(name, disable_flag);
*/
static int
@@ -1034,6 +1048,8 @@ static SLang_Intrin_Fun_Type rgmanager_slang[] =
SLANG_VOID_TYPE),
MAKE_INTRINSIC_0((char *)"service_stop", sl_stop_service,
SLANG_INT_TYPE),
+ MAKE_INTRINSIC_S((char *)"service_convalesce", sl_convalesce_service,
+ SLANG_INT_TYPE),
MAKE_INTRINSIC_0((char *)"service_start", sl_start_service,
SLANG_INT_TYPE),
MAKE_INTRINSIC_0((char *)"service_migrate", sl_migrate_service,
diff --git a/rgmanager/src/resources/default_event_script.sl b/rgmanager/src/resources/default_event_script.sl
index fad22ac..4eae767 100644
--- a/rgmanager/src/resources/default_event_script.sl
+++ b/rgmanager/src/resources/default_event_script.sl
@@ -597,6 +597,10 @@ define default_user_event_handler()
ret = service_migrate(service_name, user_target);
+ } else if (user_request == USER_CONVALESCE) {
+
+ ret = service_convalesce(service_name);
+
}
return ret;
13 years, 4 months
cluster: RHEL6 - rgmanager: Do not fail service if non-critical resources fail to stop
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: c2fa8fe7c8f2a3cbf1023a170f3f78a8de559b7a
Parent: 109a4f729592e2f9039ec369df440cbb21a078c7
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Wed Sep 8 16:20:19 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:32:16 2011 -0500
rgmanager: Do not fail service if non-critical resources fail to stop
During non-critical component recovery (e.g. stop and leave dead),
rgmanager stops parts up to the __independent_subtree="2" flag.
If any stop operations fail during this phase, rgmanager should
ignore them since the resources are non-critical.
During a normal relocation/restart/enable/disable/etc, these
operations are critical and a fail to stop will result in the
service being placed in to the failed state.
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/restree.c | 15 +++++++++++++++
1 files changed, 15 insertions(+), 0 deletions(-)
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index ab0e20d..8719791 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1510,6 +1510,21 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
node->rn_flags &= ~RF_NEEDSTOP;
rv |= res_exec(node, op, NULL, 0);
+ if (node->rn_flags & RF_QUIESCE) {
+ /* Non-critical resources = do not fail
+ * service if the resource fails to stop
+ */
+ if (rv & SFL_FAILURE) {
+ logt_print(LOG_WARNING, "Failure to stop %s:%s"
+ " during non-critical recovery "
+ "operation\n",
+ node->rn_resource->r_rule->rr_type,
+ primary_attr_value(
+ node->rn_resource));
+ rv &= ~SFL_FAILURE;
+ }
+ }
+
if (rv == 0 && (node->rn_state == RES_STARTED ||
node->rn_state == RES_FAILED)) {
assert(node->rn_resource->r_incarnations >= 0);
13 years, 4 months
cluster: RHEL6 - rgmanager: Fix handling of independent subtrees
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 109a4f729592e2f9039ec369df440cbb21a078c7
Parent: c5db095bea06e76e021577bd56d2658f90ebbecc
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Wed Sep 8 15:56:28 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:32:00 2011 -0500
rgmanager: Fix handling of independent subtrees
Independent subtrees were not getting handled
correctly in all cases. This prevented correct
recovery (e.g. stop) of resources which failed
when an ancestor was an independent subtree.
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/restree.c | 17 +++++++++--------
1 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index e7866e2..ab0e20d 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1149,8 +1149,8 @@ mark_nodes(resource_node_t *node, int state, int setflags, int clearflags)
resource_node_t *child;
list_for(&node->rn_child, child, x) {
- if (child->rn_child)
- mark_nodes(child->rn_child, state, setflags,
+ if (child)
+ mark_nodes(child, state, setflags,
clearflags);
}
@@ -1375,7 +1375,7 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
printf("Node %s:%s - Convalesce\n",
node->rn_resource->r_rule->rr_type,
primary_attr_value(node->rn_resource));
- node->rn_state = RES_STOPPED;
+ mark_nodes(node, RES_STOPPED, RF_NEEDSTART, 0);
op = RS_START;
}
@@ -1470,8 +1470,10 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
return SFL_FAILURE;
}
- if (node->rn_state != RES_DISABLED)
- mark_nodes(node, RES_STARTED, 0, RF_NEEDSTOP);
+ if (node->rn_state != RES_DISABLED) {
+ node->rn_state = RES_STARTED;
+ node->rn_flags &= ~RF_NEEDSTOP;
+ }
}
if (node->rn_child) {
@@ -1483,7 +1485,7 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
and all our children as failed and return a flag stating
that this section is recoverable apart from siblings in
the resource tree. */
- if (op == RS_STATUS && (rv & SFL_FAILURE) &&
+ if (op == RS_STATUS && (rv & (SFL_FAILURE|SFL_RECOVERABLE)) &&
(node->rn_flags & RF_INDEPENDENT)) {
rv = SFL_RECOVERABLE;
if (node->rn_flags & RF_NON_CRITICAL) {
@@ -1520,8 +1522,7 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
}
pthread_mutex_unlock(&node->rn_resource->r_mutex);
- if (node->rn_flags & RF_QUIESCE &&
- node->rn_flags & RF_NON_CRITICAL) {
+ if (node->rn_flags & RF_QUIESCE) {
node->rn_flags &= ~RF_QUIESCE;
node->rn_state = RES_DISABLED;
} else {
13 years, 4 months
cluster: RHEL6 - rgmanager: Ensure state is preserved across config changes
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: c5db095bea06e76e021577bd56d2658f90ebbecc
Parent: 57232d8ad1dde6927a7d8cd267d1f3813e2bf0ca
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Wed Sep 8 15:22:49 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:31:47 2011 -0500
rgmanager: Ensure state is preserved across config changes
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/restree.c | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index a5f27b8..e7866e2 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1796,6 +1796,9 @@ resource_tree_delta(resource_node_t **ltree, resource_node_t **rtree)
if (rc == 2)
rn->rn_flags |= RF_NEEDSTART | RF_RECONFIG;
+ if (rc == 0)
+ rn->rn_state = ln->rn_state;
+
/* Ok, same resource. Recurse. */
ln->rn_flags |= RF_COMMON;
rn->rn_flags |= RF_COMMON;
13 years, 4 months
cluster: RHEL6 - rgmanager: Allow restart,disable recovery policy
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 57232d8ad1dde6927a7d8cd267d1f3813e2bf0ca
Parent: abe50ac2721ec8124aa2a614c2a0a05e4cfa3ad7
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Thu Sep 2 15:06:09 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:31:32 2011 -0500
rgmanager: Allow restart,disable recovery policy
This policy allows administrators to allow
rgmanager to start and monitor services
where failover is not required.
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/rg_state.c | 7 +++++++
1 files changed, 7 insertions(+), 0 deletions(-)
diff --git a/rgmanager/src/daemons/rg_state.c b/rgmanager/src/daemons/rg_state.c
index 9eb686f..9328a88 100644
--- a/rgmanager/src/daemons/rg_state.c
+++ b/rgmanager/src/daemons/rg_state.c
@@ -2220,6 +2220,13 @@ handle_recover_req(char *svcName, int *new_owner)
/* Check restart counter/timer for this resource */
if (check_restart(svcName) > 0) {
clear_restart(svcName);
+
+ if (strstr(policy, "disable")) {
+ logt_print(LOG_NOTICE,
+ "Restart threshold for %s exceeded; "
+ "disabling\n", svcName);
+ return svc_disable(svcName);
+ }
logt_print(LOG_NOTICE, "Restart threshold for %s exceeded; "
"attempting to relocate\n", svcName);
return handle_relocate_req(svcName, RG_START_RECOVER, -1,
13 years, 4 months
cluster: RHEL6 - rgmanager: Add convalesce operation
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: abe50ac2721ec8124aa2a614c2a0a05e4cfa3ad7
Parent: b631ffdb818f7cf3512840dd99b8844aa230b03d
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Wed Sep 1 15:24:22 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:31:11 2011 -0500
rgmanager: Add convalesce operation
This option enables the administrator to repair
a partially-failed service without affecting
the rest of the service.
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/include/resgroup.h | 1 +
rgmanager/include/reslist.h | 1 +
rgmanager/src/clulib/rg_strings.c | 1 +
rgmanager/src/daemons/groups.c | 3 +
rgmanager/src/daemons/restree.c | 14 ++++
rgmanager/src/daemons/rg_state.c | 119 ++++++++++++++++++++++++++++++++++++-
rgmanager/src/daemons/rg_thread.c | 20 ++++++
rgmanager/src/utils/clusvcadm.c | 7 ++-
8 files changed, 162 insertions(+), 4 deletions(-)
diff --git a/rgmanager/include/resgroup.h b/rgmanager/include/resgroup.h
index a0b6fc1..2e81eb9 100644
--- a/rgmanager/include/resgroup.h
+++ b/rgmanager/include/resgroup.h
@@ -157,6 +157,7 @@ int svc_disable(const char *svcName);
int svc_fail(const char *svcName);
int svc_freeze(const char *svcName);
int svc_unfreeze(const char *svcName);
+int svc_convalesce(const char *svcName);
int svc_migrate(const char *svcName, int target);
int svc_start_remote(const char *svcName, int request, uint32_t target);
int svc_report_failure(const char *svcName);
diff --git a/rgmanager/include/reslist.h b/rgmanager/include/reslist.h
index a8acd8e..921c483 100644
--- a/rgmanager/include/reslist.h
+++ b/rgmanager/include/reslist.h
@@ -130,6 +130,7 @@ int res_start(resource_node_t **tree, resource_t *res, void *ret);
int res_stop(resource_node_t **tree, resource_t *res, void *ret);
int res_status(resource_node_t **tree, resource_t *res, void *ret);
int res_status_inquiry(resource_node_t **tree, resource_t *res, void *ret);
+int res_convalesce(resource_node_t **tree, resource_t *res, void *ret);
int res_condstart(resource_node_t **tree, resource_t *res, void *ret);
int res_condstop(resource_node_t **tree, resource_t *res, void *ret);
int res_exec(resource_node_t *node, int op, const char *arg, int depth);
diff --git a/rgmanager/src/clulib/rg_strings.c b/rgmanager/src/clulib/rg_strings.c
index fdb1dad..b0cd184 100644
--- a/rgmanager/src/clulib/rg_strings.c
+++ b/rgmanager/src/clulib/rg_strings.c
@@ -58,6 +58,7 @@ const struct string_val rg_req_strings[] = {
{RG_QUERY_LOCK, "lock status inquiry"},
{RG_MIGRATE, "migrate"},
{RG_STATUS_INQUIRY, "out of band service status inquiry"},
+ {RG_CONVALESCE, "convalesce"},
{RG_NONE, "none"},
{0, NULL}
};
diff --git a/rgmanager/src/daemons/groups.c b/rgmanager/src/daemons/groups.c
index afce054..1d3c518 100644
--- a/rgmanager/src/daemons/groups.c
+++ b/rgmanager/src/daemons/groups.c
@@ -975,6 +975,9 @@ group_op(const char *groupname, int op)
case RG_CONDSTART:
ret = res_condstart(&_tree, res, NULL);
break;
+ case RG_CONVALESCE:
+ ret = res_convalesce(&_tree, res, NULL);
+ break;
}
pthread_rwlock_unlock(&resource_lock);
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index a717376..a5f27b8 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -1615,6 +1615,20 @@ res_condstop(resource_node_t **tree, resource_t *res, void *ret)
/**
+ Repair/fix/convalesce all occurrences of a resource in a tree
+
+ @param tree Tree to search for our resource.
+ @param res Resource to start/stop
+ @param ret Unused
+ */
+int
+res_convalesce(resource_node_t **tree, resource_t *res, void *ret)
+{
+ return _res_op(tree, res, NULL, ret, RS_CONVALESCE);
+}
+
+
+/**
Start all occurrences of a resource in a tree
@param tree Tree to search for our resource.
diff --git a/rgmanager/src/daemons/rg_state.c b/rgmanager/src/daemons/rg_state.c
index 9049a7c..9eb686f 100644
--- a/rgmanager/src/daemons/rg_state.c
+++ b/rgmanager/src/daemons/rg_state.c
@@ -803,6 +803,92 @@ svc_start(const char *svcName, int req)
/**
+ * Fix stuff
+ */
+int
+svc_convalesce(const char *svcName)
+{
+ struct dlm_lksb lockp;
+ rg_state_t svcStatus;
+ int ret;
+
+ if (rg_lock(svcName, &lockp) < 0) {
+ logt_print(LOG_ERR, "#451: Unable to obtain cluster lock: %s\n",
+ strerror(errno));
+ return RG_EFAIL;
+ }
+
+ if (get_rg_state(svcName, &svcStatus) != 0) {
+ rg_unlock(&lockp);
+ logt_print(LOG_ERR, "#461: Failed getting status for RG %s\n",
+ svcName);
+ return RG_EFAIL;
+ }
+
+ switch(svcStatus.rs_state) {
+ case RG_STATE_STARTED:
+ break;
+ case RG_STATE_STARTING:
+ case RG_STATE_STOPPING:
+ case RG_STATE_RECOVER:
+ case RG_STATE_MIGRATE:
+ case RG_STATE_ERROR:
+ rg_unlock(&lockp);
+ return RG_EAGAIN;
+ default:
+ rg_unlock(&lockp);
+ return RG_EINVAL;
+ }
+
+ if (svcStatus.rs_flags & RG_FLAG_FROZEN) {
+ rg_unlock(&lockp);
+ return RG_EFROZEN;
+ }
+
+ if (svcStatus.rs_owner != (uint32_t)my_id()) {
+ rg_unlock(&lockp);
+ return RG_EFORWARD;
+ }
+
+ if (!(svcStatus.rs_flags & RG_FLAG_PARTIAL)) {
+ rg_unlock(&lockp);
+ return RG_ERUN;
+ }
+
+ rg_unlock(&lockp);
+
+ logt_print(LOG_INFO, "Repairing %s\n", svcName);
+ ret = group_op(svcName, RG_CONVALESCE);
+
+ switch(ret) {
+ default:
+ logt_print(LOG_WARNING, "Failed to repair %s\n", svcName);
+ /* Fail to restart a non-critical resource
+ * does not fail the service. */
+ return RG_EFAIL;
+ case 0:
+ logt_print(LOG_INFO, "Repair of %s was successful\n", svcName);
+ break;
+ }
+
+ /* Success - flip owner in state info */
+ if (rg_lock(svcName, &lockp) < 0) {
+ logt_print(LOG_ERR, "#455: Unable to obtain cluster lock: %s\n",
+ strerror(errno));
+ return RG_EFAIL;
+ }
+
+ /* No need for a 'get' here since the service is still STARTED */
+ svcStatus.rs_flags &= ~RG_FLAG_PARTIAL;
+
+ set_rg_state(svcName, &svcStatus);
+ rg_unlock(&lockp);
+
+ return 0;
+}
+
+
+/**
* Migrate a service to another node. Relies on agent
* operating synchronously
*/
@@ -1111,8 +1197,31 @@ svc_status(const char *svcName)
}
/* For running services, if the return code is 0, we're done*/
- if (svcStatus.rs_state == RG_STATE_STARTED)
- return handle_started_status(svcName, ret, &svcStatus);
+ if (svcStatus.rs_state == RG_STATE_STARTED) {
+ ret = handle_started_status(svcName, ret, &svcStatus);
+
+ if (ret & SFL_PARTIAL) {
+ ret &= ~SFL_PARTIAL;
+ svcStatus.rs_flags |= RG_FLAG_PARTIAL;
+ if (rg_lock(svcName, &lockp) < 0) {
+ logt_print(LOG_ERR,
+ "#481: Unable to obtain cluster lock: %s\n",
+ strerror(errno));
+ return RG_EFAIL;
+ }
+
+ if (set_rg_state(svcName, &svcStatus) != 0) {
+ rg_unlock(&lockp);
+ logt_print(LOG_ERR,
+ "#482: Failed setting status for RG %s\n",
+ svcName);
+ return RG_EFAIL;
+ }
+ rg_unlock(&lockp);
+ }
+
+ return ret;
+ }
return handle_migrate_status(svcName, ret, &svcStatus);
}
@@ -1153,9 +1262,10 @@ handle_started_status(const char *svcName, int ret,
svcName);
if (ret & SFL_PARTIAL) {
logt_print(LOG_NOTICE, "Note: Some non-critical "
- "resources are still stopped.\n");
+ "resources were stopped during recovery.\n");
logt_print(LOG_NOTICE, "Run 'clusvcadm -c %s' to "
"restore them to operation.\n", svcName);
+ return SFL_PARTIAL;
}
return 0;
@@ -1262,6 +1372,7 @@ _svc_stop(const char *svcName, int req, int recover, uint32_t newstate)
svcStatus.rs_last_owner = svcStatus.rs_owner;
svcStatus.rs_owner = 0;
svcStatus.rs_state = RG_STATE_STOPPED;
+ svcStatus.rs_flags = 0;
if (set_rg_state(svcName, &svcStatus) != 0) {
rg_unlock(&lockp);
return RG_EFAIL;
@@ -1292,6 +1403,7 @@ _svc_stop(const char *svcName, int req, int recover, uint32_t newstate)
logt_print(LOG_DEBUG, "%s is clean; skipping double-stop\n",
svcName);
svcStatus.rs_state = newstate;
+ svcStatus.rs_flags = 0;
if (set_rg_state(svcName, &svcStatus) != 0) {
rg_unlock(&lockp);
@@ -1372,6 +1484,7 @@ _svc_stop_finish(const char *svcName, int failed, uint32_t newstate)
}
svcStatus.rs_state = newstate;
+ svcStatus.rs_flags = 0;
logt_print(LOG_NOTICE, "Service %s is %s\n", svcName,
rg_state_str(svcStatus.rs_state));
diff --git a/rgmanager/src/daemons/rg_thread.c b/rgmanager/src/daemons/rg_thread.c
index 446f2d2..72b5f96 100644
--- a/rgmanager/src/daemons/rg_thread.c
+++ b/rgmanager/src/daemons/rg_thread.c
@@ -277,6 +277,26 @@ resgroup_thread_main(void *arg)
ret = RG_NONE;
break;
+ case RG_CONVALESCE:
+ error = svc_convalesce(myname);
+
+ if (error == 0) {
+ ret = RG_SUCCESS;
+
+ pthread_mutex_lock(&my_queue_mutex);
+ purge_status_checks(&my_queue);
+ pthread_mutex_unlock(&my_queue_mutex);
+ } else if (error == RG_EFORWARD) {
+ ret = RG_NONE;
+ break;
+ } else {
+ /*
+ * Bad news.
+ */
+ ret = RG_EFAIL;
+ }
+ break;
+
case RG_MIGRATE:
error = svc_migrate(myname, req->rr_target);
diff --git a/rgmanager/src/utils/clusvcadm.c b/rgmanager/src/utils/clusvcadm.c
index 137837a..bd0723f 100644
--- a/rgmanager/src/utils/clusvcadm.c
+++ b/rgmanager/src/utils/clusvcadm.c
@@ -235,7 +235,7 @@ main(int argc, char **argv)
const char *actionstr = NULL;
cluster_member_list_t *membership;
- while ((opt = getopt(argc, argv, "lSue:M:d:r:n:m:FvR:s:Z:U:qh?")) != EOF) {
+ while ((opt = getopt(argc, argv, "lSue:M:d:r:n:c:m:FvR:s:Z:U:qh?")) != EOF) {
switch (opt) {
case 'l':
return do_lock();
@@ -312,6 +312,11 @@ main(int argc, char **argv)
action = RG_UNFREEZE;
svcname = optarg;
break;
+ case 'c':
+ actionstr = "convalescing";
+ action = RG_CONVALESCE;
+ svcname = optarg;
+ break;
case 'q':
close(STDOUT_FILENO);
break;
13 years, 4 months
cluster: RHEL6 - rgmanager: Mark non-critical resources disabled
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: b631ffdb818f7cf3512840dd99b8844aa230b03d
Parent: 5907340776e360b327642f24f7ace0ae812b7a81
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Mon Aug 30 13:37:38 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:30:58 2011 -0500
rgmanager: Mark non-critical resources disabled
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/include/resgroup.h | 1 +
rgmanager/include/reslist.h | 2 +
rgmanager/src/daemons/restree.c | 75 ++++++++++++++++++++++++++------------
rgmanager/src/daemons/rg_state.c | 27 +++++++++-----
4 files changed, 72 insertions(+), 33 deletions(-)
diff --git a/rgmanager/include/resgroup.h b/rgmanager/include/resgroup.h
index 13b4dae..a0b6fc1 100644
--- a/rgmanager/include/resgroup.h
+++ b/rgmanager/include/resgroup.h
@@ -229,6 +229,7 @@ const char *rg_strerror(int val);
*/
#define SFL_FAILURE (1<<0)
#define SFL_RECOVERABLE (1<<1)
+#define SFL_PARTIAL (1<<2)
//#define DEBUG
#ifdef DEBUG
diff --git a/rgmanager/include/reslist.h b/rgmanager/include/reslist.h
index 142fce6..a8acd8e 100644
--- a/rgmanager/include/reslist.h
+++ b/rgmanager/include/reslist.h
@@ -29,12 +29,14 @@
the configuration */
#define RF_ENFORCE_TIMEOUTS (1<<9) /** Enforce timeouts for this node */
#define RF_NON_CRITICAL (1<<10) /** stop this resource if it fails */
+#define RF_QUIESCE (1<<11) /** don't restart this resource */
#define RES_STOPPED (0)
#define RES_STARTED (1)
#define RES_FAILED (2)
+#define RES_DISABLED (3)
#ifndef SHAREDIR
#define SHAREDIR "/usr/share/rgmanager"
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index e480cc8..a717376 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -962,7 +962,8 @@ _print_resource_tree(FILE *fp, resource_node_t **tree, int level)
fprintf(fp, "NON-CRITICAL ");
fprintf(fp, "]");
}
- fprintf(fp, " {\n");
+
+ fprintf(fp, " (S%d) {\n", node->rn_state);
for (x = 0; node->rn_resource->r_attrs &&
node->rn_resource->r_attrs[x].ra_value; x++) {
@@ -1153,7 +1154,8 @@ mark_nodes(resource_node_t *node, int state, int setflags, int clearflags)
clearflags);
}
- node->rn_state = state;
+ if (state >= RES_STOPPED)
+ node->rn_state = state;
node->rn_flags |= (setflags);
node->rn_flags &= ~(clearflags);
}
@@ -1171,6 +1173,9 @@ do_status(resource_node_t *node)
int has_recover = 0;
time_t delta = 0, now = 0;
+ if (node->rn_state == RES_DISABLED)
+ return 0;
+
now = time(NULL);
for (; node->rn_actions[x].ra_name; x++) {
@@ -1365,6 +1370,15 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
CONDSTART and CONDSTOP are no-ops if
the appropriate flag is not set.
*/
+ if ((op == RS_CONVALESCE) &&
+ node->rn_state == RES_DISABLED) {
+ printf("Node %s:%s - Convalesce\n",
+ node->rn_resource->r_rule->rr_type,
+ primary_attr_value(node->rn_resource));
+ node->rn_state = RES_STOPPED;
+ op = RS_START;
+ }
+
if ((op == RS_CONDSTART) &&
(node->rn_flags & RF_NEEDSTART)) {
printf("Node %s:%s - CONDSTART\n",
@@ -1385,7 +1399,12 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
/* Start starts before children */
if (me && (op == RS_START)) {
+ if (node->rn_state == RES_DISABLED)
+ /* Nothing to do - children are also disabled */
+ return 0;
+
pthread_mutex_lock(&node->rn_resource->r_mutex);
+
if (node->rn_flags & RF_RECONFIG &&
realop == RS_CONDSTART) {
rv = res_exec(node, RS_RECONFIG, NULL, 0);
@@ -1430,21 +1449,29 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
dependent children are failed, whether or not this
node is independent or not.
*/
- mark_nodes(node, RES_FAILED,
- RF_NEEDSTART | RF_NEEDSTOP, 0);
/* If we're an independent subtree, return a flag
stating that this section is recoverable apart
from siblings in the resource tree. All child
resources of this node must be restarted,
but siblings of this node are not affected. */
- if (node->rn_flags & RF_INDEPENDENT)
- return SFL_RECOVERABLE;
+ if (node->rn_flags & RF_INDEPENDENT) {
+ if (node->rn_flags & RF_NON_CRITICAL) {
+ mark_nodes(node, RES_FAILED,
+ RF_NEEDSTOP | RF_QUIESCE, 0);
+ return SFL_RECOVERABLE|SFL_PARTIAL;
+ } else {
+ mark_nodes(node, RES_FAILED,
+ RF_NEEDSTART | RF_NEEDSTOP, 0);
+ return SFL_RECOVERABLE;
+ }
+ }
return SFL_FAILURE;
}
- mark_nodes(node, RES_STARTED, 0, RF_NEEDSTOP);
+ if (node->rn_state != RES_DISABLED)
+ mark_nodes(node, RES_STARTED, 0, RF_NEEDSTOP);
}
if (node->rn_child) {
@@ -1458,14 +1485,16 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
the resource tree. */
if (op == RS_STATUS && (rv & SFL_FAILURE) &&
(node->rn_flags & RF_INDEPENDENT)) {
- if (node->rn_flags & RF_NON_CRITICAL)
+ rv = SFL_RECOVERABLE;
+ if (node->rn_flags & RF_NON_CRITICAL) {
/* if non-critical, just stop */
- mark_nodes(node, RES_FAILED,
- RF_NEEDSTOP, 0);
- else
+ mark_nodes(node, RES_FAILED, RF_NEEDSTOP | RF_QUIESCE, 0);
+
+ rv |= SFL_PARTIAL;
+ } else {
mark_nodes(node, RES_FAILED,
RF_NEEDSTOP | RF_NEEDSTART, 0);
- rv = SFL_RECOVERABLE;
+ }
}
}
@@ -1475,27 +1504,27 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
/* Decrease incarnations so the script knows how many *other*
incarnations there are. */
pthread_mutex_lock(&node->rn_resource->r_mutex);
- if (node->rn_state == RES_STARTED) {
- assert(node->rn_resource->r_incarnations >= 0);
- if (node->rn_resource->r_incarnations > 0)
- --node->rn_resource->r_incarnations;
- }
node->rn_flags &= ~RF_NEEDSTOP;
rv |= res_exec(node, op, NULL, 0);
- if (rv != 0) {
- if (node->rn_state == RES_STARTED) {
- /* Fail to stop: fix incarnations */
- ++node->rn_resource->r_incarnations;
- }
+ if (rv == 0 && (node->rn_state == RES_STARTED ||
+ node->rn_state == RES_FAILED)) {
+ assert(node->rn_resource->r_incarnations >= 0);
+ if (node->rn_resource->r_incarnations > 0)
+ --node->rn_resource->r_incarnations;
+ } else if (node->rn_state != RES_DISABLED) {
node->rn_state = RES_FAILED;
pthread_mutex_unlock(&node->rn_resource->r_mutex);
return SFL_FAILURE;
}
pthread_mutex_unlock(&node->rn_resource->r_mutex);
- if (node->rn_state != RES_STOPPED) {
+ if (node->rn_flags & RF_QUIESCE &&
+ node->rn_flags & RF_NON_CRITICAL) {
+ node->rn_flags &= ~RF_QUIESCE;
+ node->rn_state = RES_DISABLED;
+ } else {
node->rn_state = RES_STOPPED;
}
}
diff --git a/rgmanager/src/daemons/rg_state.c b/rgmanager/src/daemons/rg_state.c
index d31ea2b..9049a7c 100644
--- a/rgmanager/src/daemons/rg_state.c
+++ b/rgmanager/src/daemons/rg_state.c
@@ -1122,7 +1122,7 @@ static int
handle_started_status(const char *svcName, int ret,
rg_state_t __attribute__((unused)) *svcStatus)
{
- int newowner;
+ int newowner, ret2;
if (ret & SFL_FAILURE) {
newowner = msvc_check_cluster(svcName);
@@ -1138,20 +1138,27 @@ handle_started_status(const char *svcName, int ret,
logt_print(LOG_WARNING, "Some independent resources in %s failed; "
"Attempting inline recovery\n", svcName);
- ret = group_op(svcName, RG_CONDSTOP);
- if (!(ret & SFL_FAILURE)) {
- ret = group_op(svcName, RG_CONDSTART);
+ ret2 = group_op(svcName, RG_CONDSTOP);
+ if (!(ret2 & SFL_FAILURE)) {
+ ret2 = group_op(svcName, RG_CONDSTART);
}
- if (ret) {
+ if (ret2) {
logt_print(LOG_WARNING, "Inline recovery of %s failed\n",
svcName);
- } else {
- logt_print(LOG_NOTICE,
- "Inline recovery of %s succeeded\n",
- svcName);
- return 0;
+ return ret;
}
+
+ logt_print(LOG_NOTICE, "Inline recovery of %s succeeded\n",
+ svcName);
+ if (ret & SFL_PARTIAL) {
+ logt_print(LOG_NOTICE, "Note: Some non-critical "
+ "resources are still stopped.\n");
+ logt_print(LOG_NOTICE, "Run 'clusvcadm -c %s' to "
+ "restore them to operation.\n", svcName);
+ }
+
+ return 0;
}
return ret;
13 years, 4 months
cluster: RHEL6 - rgmanager: Present all flags in clustat output
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 5907340776e360b327642f24f7ace0ae812b7a81
Parent: 0e5e14cd1471464edf14776bd7ac84d14623a03d
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Wed Aug 25 14:34:59 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:30:46 2011 -0500
rgmanager: Present all flags in clustat output
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/utils/clustat.c | 11 ++++++++---
1 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/rgmanager/src/utils/clustat.c b/rgmanager/src/utils/clustat.c
index d3137e0..b64d474 100644
--- a/rgmanager/src/utils/clustat.c
+++ b/rgmanager/src/utils/clustat.c
@@ -455,6 +455,7 @@ _txt_rg_state(rg_state_t *rs, cluster_member_list_t *members, int flags,
{
char owner[MAXHOSTNAMELEN+1];
char state_string[255] = "";
+ char flags_string[16] = "";
char *name = rs->rs_name, *ptr;
int l;
@@ -484,10 +485,14 @@ _txt_rg_state(rg_state_t *rs, cluster_member_list_t *members, int flags,
}
/* Show a frozen service */
- if (rs->rs_flags & RG_FLAG_FROZEN) {
+ if (rs->rs_flags) {
+ rg_flags_str(flags_string, sizeof(flags_string), rs->rs_flags,
+ NULL);
+
snprintf(state_string, sizeof(state_string),
- "%-*.*s[Z]", statsize-3, statsize-3,
- rg_state_str(rs->rs_state));
+ "%-*.*s[%s]", (int)(statsize-(2+strlen(flags_string))),
+ (int)(statsize-(2+strlen(flags_string))),
+ rg_state_str(rs->rs_state), flags_string);
} else {
snprintf(state_string, sizeof(state_string),
"%-*.*s", statsize, statsize,
13 years, 4 months
cluster: RHEL6 - rgmanager: Use rg_strings.c for flags
by Lon Hohberger
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: 0e5e14cd1471464edf14776bd7ac84d14623a03d
Parent: 5203d9eefe530a13525dc32d9f48568fbabfd495
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Wed Aug 25 13:50:55 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Tue Feb 1 12:30:14 2011 -0500
rgmanager: Use rg_strings.c for flags
Resolves: rhbz#634277
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/clulib/rg_strings.c | 14 ++++++++++++--
1 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/rgmanager/src/clulib/rg_strings.c b/rgmanager/src/clulib/rg_strings.c
index 24ee479..fdb1dad 100644
--- a/rgmanager/src/clulib/rg_strings.c
+++ b/rgmanager/src/clulib/rg_strings.c
@@ -81,9 +81,15 @@ const struct string_val rg_state_strings[] = {
const struct string_val rg_flags_strings[] = {
{RG_FLAG_FROZEN, "frozen"},
+ {RG_FLAG_PARTIAL, "partial"},
{0, NULL}
};
+const struct string_val rg_flags_short[] = {
+ {RG_FLAG_FROZEN, "Z"},
+ {RG_FLAG_PARTIAL, "P"},
+ {0, NULL}
+};
const struct string_val agent_ops[] = {
{RS_START, "start"},
@@ -185,14 +191,18 @@ rg_flag_str(int val)
const char *
rg_flags_str(char *flags_string, size_t size, int val, char *separator)
{
+ const struct string_val *table = rg_flags_strings;
int i;
const char *string;
+ if (!separator)
+ table = rg_flags_short;
+
for (i = 0; i < (sizeof(val) * 8); i++) {
if ( val & (1 << i)) {
- if (strlen(flags_string))
+ if (strlen(flags_string) && separator)
strncat(flags_string, separator, size - (strlen(flags_string) + strlen(separator) + 1));
- string = rg_search_table(rg_flags_strings, (1 << i));
+ string = rg_search_table(table, (1 << i));
strncat(flags_string, string, size - (strlen(flags_string) + strlen(string) + 1));
}
}
13 years, 4 months