Gitweb:
http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=...
Commit: b05534b22a95d15be95035448a20f6fa321a9a10
Parent: 2a7f8f06f369fe255a5eb5e9cd6b2b2d577ed7c5
Author: Lon Hohberger <lhh(a)redhat.com>
AuthorDate: Fri Sep 10 14:45:15 2010 -0400
Committer: Lon Hohberger <lhh(a)redhat.com>
CommitterDate: Fri Sep 24 16:16:03 2010 -0400
rgmanager: Add independent subtree restart thresholds
__max_restarts => number of restarts to tolerate
__restart_expire_time => in the amount of time given here
__independent_subtree="1":
- after threshold is exceeded for the subtree, restart the
service
- if no threshold, never restart the service
__independent_subtree="2"
- after threshold is exceeded for the subtree, stop the
subtree
- if no threshold, stop the subtree on first failure
Resolves: rhbz#605733
Signed-off-by: Lon Hohberger <lhh(a)redhat.com>
---
rgmanager/src/daemons/restree.c | 97 ++++++++++++++++++++++++++++++++-------
1 files changed, 80 insertions(+), 17 deletions(-)
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index 0494cdc..60743a9 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -522,20 +522,54 @@ res_exec(resource_node_t *node, int op, const char *arg, int depth)
static inline void
assign_restart_policy(resource_t *curres, resource_node_t *parent,
- resource_node_t *node)
+ resource_node_t *node, int ccsfd, char *base)
{
char *val;
int max_restarts = 0;
time_t restart_expire_time = 0;
+ char tok[1024];
node->rn_restart_counter = NULL;
if (!curres || !node)
return;
- if (parent) /* Non-parents don't get one for now */
+ if (parent &&
+ !(node->rn_flags & RF_INDEPENDENT))
return;
- val = res_attr_value(curres, "max_restarts");
+ if (node->rn_flags & RF_INDEPENDENT) {
+ /* per-resource-node failures / expire times */
+ snprintf(tok, sizeof(tok), "%s/@__max_restarts", base);
+#ifndef NO_CCS
+ if (ccs_get(ccsfd, tok, &val) == 0) {
+#else
+ if (conf_get(tok, &val) == 0) {
+#endif
+ max_restarts = atoi(val);
+ if (max_restarts <= 0)
+ max_restarts = 0;
+ free(val);
+
+ if (!max_restarts)
+ return;
+ }
+
+ snprintf(tok, sizeof(tok), "%s/@__restart_expire_time", base);
+#ifndef NO_CCS
+ if (ccs_get(ccsfd, tok, &val) == 0) {
+#else
+ if (conf_get(tok, &val) == 0) {
+#endif
+ restart_expire_time = (time_t)expand_time(val);
+ if ((int64_t)restart_expire_time < 0)
+ restart_expire_time = 0;
+ free(val);
+ }
+
+ goto out_assign;
+ }
+
+ val = (char *)res_attr_value(curres, "max_restarts");
if (!val)
return;
max_restarts = atoi(val);
@@ -548,6 +582,7 @@ assign_restart_policy(resource_t *curres, resource_node_t *parent,
return;
}
+out_assign:
node->rn_restart_counter = restart_init(restart_expire_time,
max_restarts);
}
@@ -637,20 +672,25 @@ do_load_resource(int ccsfd, char *base,
node->rn_state = RES_STOPPED;
node->rn_flags = 0;
node->rn_actions = (resource_act_t *)act_dup(curres->r_actions);
- assign_restart_policy(curres, parent, node);
- snprintf(tok, sizeof(tok), "%s/@__independent_subtree", base);
+
+ if (parent) {
+ /* Independent subtree / non-critical for top-level is
+ * not useful and can interfere with restart thresholds for
+ * non critical resources */
+ snprintf(tok, sizeof(tok), "%s/@__independent_subtree", base);
#ifndef NO_CCS
- if (ccs_get(ccsfd, tok, &ref) == 0) {
+ if (ccs_get(ccsfd, tok, &ref) == 0) {
#else
- if (conf_get(tok, &ref) == 0) {
+ if (conf_get(tok, &ref) == 0) {
#endif
- if (atoi(ref) == 1 || strcasecmp(ref, "yes") == 0)
- node->rn_flags |= RF_INDEPENDENT;
- if (atoi(ref) == 2 || strcasecmp(ref, "non-critical") == 0) {
- curres->r_flags |= RF_NON_CRITICAL;
+ if (atoi(ref) == 1 || strcasecmp(ref, "yes") == 0)
+ node->rn_flags |= RF_INDEPENDENT;
+ if (atoi(ref) == 2 || strcasecmp(ref, "non-critical") == 0) {
+ curres->r_flags |= RF_NON_CRITICAL;
+ }
+ free(ref);
}
- free(ref);
}
snprintf(tok, sizeof(tok), "%s/@__enforce_timeouts", base);
@@ -708,8 +748,11 @@ do_load_resource(int ccsfd, char *base,
* resource is non-critical
*/
node->rn_flags |= RF_NON_CRITICAL | RF_INDEPENDENT;
+
}
+ assign_restart_policy(curres, parent, node, ccsfd, base);
+
*newnode = node;
list_insert(tree, node);
@@ -1377,7 +1420,7 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
char *type, void *__attribute__((unused))ret, int realop,
resource_node_t *node)
{
- int rv = 0, me, op;
+ int rv = 0, me, op, rte = 0;
/* Restore default operation. */
op = realop;
@@ -1443,6 +1486,10 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
/* Nothing to do - children are also disabled */
return 0;
+ if ((realop == RS_START || realop == RS_CONVALESCE) &&
+ node->rn_flags & RF_INDEPENDENT)
+ restart_clear(node->rn_restart_counter);
+
pthread_mutex_lock(&node->rn_resource->r_mutex);
if (node->rn_flags & RF_RECONFIG &&
@@ -1496,14 +1543,22 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
resources of this node must be restarted,
but siblings of this node are not affected. */
if (node->rn_flags & RF_INDEPENDENT) {
- if (node->rn_flags & RF_NON_CRITICAL) {
+
+ rte = restart_threshold_exceeded(node->rn_restart_counter);
+ if ((node->rn_flags & RF_NON_CRITICAL) && (rte ||
+ !node->rn_restart_counter)) {
mark_nodes(node, RES_FAILED,
RF_NEEDSTOP | RF_QUIESCE, 0);
+ restart_clear(node->rn_restart_counter);
return SFL_RECOVERABLE|SFL_PARTIAL;
- } else {
+ } else if (!rte) {
+ restart_add(node->rn_restart_counter);
mark_nodes(node, RES_FAILED,
RF_NEEDSTART | RF_NEEDSTOP, 0);
return SFL_RECOVERABLE;
+ } else {
+ restart_clear(node->rn_restart_counter);
+ return SFL_FAILURE;
}
}
@@ -1527,15 +1582,23 @@ _res_op_internal(resource_node_t __attribute__ ((unused)) **tree,
the resource tree. */
if (op == RS_STATUS && (rv & (SFL_FAILURE|SFL_RECOVERABLE)) &&
(node->rn_flags & RF_INDEPENDENT)) {
+
+ rte = restart_threshold_exceeded(node->rn_restart_counter);
+
rv = SFL_RECOVERABLE;
- if (node->rn_flags & RF_NON_CRITICAL) {
+ if ((node->rn_flags & RF_NON_CRITICAL) && (rte ||
+ !node->rn_restart_counter)) {
/* if non-critical, just stop */
mark_nodes(node, RES_FAILED, RF_NEEDSTOP | RF_QUIESCE, 0);
rv |= SFL_PARTIAL;
- } else {
+ } else if (!rte) {
+ restart_add(node->rn_restart_counter);
mark_nodes(node, RES_FAILED,
RF_NEEDSTOP | RF_NEEDSTART, 0);
+ } else {
+ restart_clear(node->rn_restart_counter);
+ rv = SFL_FAILURE;
}
}
}