cluster: STABLE3 - fenced: limit repeating failure messages

Fabio M. Di Nitto fabbione at fedoraproject.org
Mon Oct 5 03:50:05 UTC 2009


Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=334039edb71ed9e5577726b71ed3bf67ee80dbe1
Commit:        334039edb71ed9e5577726b71ed3bf67ee80dbe1
Parent:        bcc5fdef8473d99399c624a7bc15423a2af645c1
Author:        David Teigland <teigland at redhat.com>
AuthorDate:    Tue Sep 15 14:53:14 2009 -0500
Committer:     David Teigland <teigland at redhat.com>
CommitterDate: Tue Sep 15 14:53:14 2009 -0500

fenced: limit repeating failure messages

When fencing is failing, don't log the same failure message
over and over which fills up the logs and pushes other history
out of the debug log.

Signed-off-by: David Teigland <teigland at redhat.com>
---
 fence/fenced/cpg.c         |    2 +-
 fence/fenced/member_cman.c |    2 +-
 fence/fenced/recover.c     |   34 ++++++++++++++++++++++++++++++++--
 3 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/fence/fenced/cpg.c b/fence/fenced/cpg.c
index c4d649d..be89fce 100644
--- a/fence/fenced/cpg.c
+++ b/fence/fenced/cpg.c
@@ -2119,7 +2119,7 @@ int is_clean_daemon_member(int nodeid)
 			return 0;
 		}
 	}
-	log_debug("daemon_member %d not member", nodeid);
+	/* log_debug("daemon_member %d not member", nodeid); */
 	return 0;
 }
 
diff --git a/fence/fenced/member_cman.c b/fence/fenced/member_cman.c
index a621173..5e977a6 100644
--- a/fence/fenced/member_cman.c
+++ b/fence/fenced/member_cman.c
@@ -314,7 +314,7 @@ int is_cman_member_reread(int nodeid)
 	if (rv)
 		return 1;
 
-	log_debug("cman_member %d not member", nodeid);
+	/* log_debug("cman_member %d not member", nodeid); */
 	return 0;
 }
 
diff --git a/fence/fenced/recover.c b/fence/fenced/recover.c
index aa1cfe5..e0d4fd8 100644
--- a/fence/fenced/recover.c
+++ b/fence/fenced/recover.c
@@ -278,13 +278,15 @@ static const char *fe_str(int r)
 
 #define FL_SIZE 32
 static struct fence_log flog[FL_SIZE];
+static struct fence_log prev_flog[FL_SIZE];
 
 void fence_victims(struct fd *fd)
 {
 	struct node *node;
-	int error, i, ll, flog_count;
+	int error, i, ll, flog_count, prev_flog_count;
 	int override = -1;
 	int cman_member, cpg_member, ext;
+	unsigned int limit, retries;
 
 	list_for_each_entry(node, &fd->victims, list) {
 		if (node->local_victim_done) {
@@ -304,7 +306,22 @@ void fence_victims(struct fd *fd)
 				  node->nodeid);
 			continue;
 		}
+
+		/* limit repeated logging of the same failure messages
+		   when retrying fencing */
+
+		limit = 0;
+		retries = 0;
+		memset(&prev_flog, 0, sizeof(flog));
+		prev_flog_count = 0;
+
  retry:
+		if (retries > 2)
+			limit = 1;
+		if (limit && !(retries % 600))
+			log_level(LOG_INFO, "fencing node %s still retrying",
+				  node->name);
+
 		/* for queries */
 		fd->current_victim = node->nodeid;
 
@@ -329,7 +346,8 @@ void fence_victims(struct fd *fd)
 		memset(&flog, 0, sizeof(flog));
 		flog_count = 0;
 
-		log_level(LOG_INFO, "fencing node %s", node->name);
+		if (!limit)
+			log_level(LOG_INFO, "fencing node %s", node->name);
 
 		query_unlock();
 		error = fence_node(node->name, flog, FL_SIZE, &flog_count);
@@ -340,6 +358,15 @@ void fence_victims(struct fd *fd)
 			flog_count = FL_SIZE;
 		}
 
+		if (limit && error &&
+		    flog_count == prev_flog_count &&
+		    !memcmp(&flog, &prev_flog, sizeof(flog))) {
+			goto skip_log_message;
+		}
+
+		memcpy(&prev_flog, &flog, sizeof(flog));
+		prev_flog_count = flog_count;
+
 		for (i = 0; i < flog_count; i++) {
 			ll = (flog[i].error == FE_AGENT_SUCCESS) ? LOG_DEBUG:
 								  LOG_ERR;
@@ -354,6 +381,7 @@ void fence_victims(struct fd *fd)
 		log_error("fence %s %s", node->name,
 			  error ? "failed" : "success");
 
+ skip_log_message:
 		if (!error) {
 			node->local_victim_done = 1;
 			victim_done(fd, node->nodeid, VIC_DONE_AGENT);
@@ -364,6 +392,7 @@ void fence_victims(struct fd *fd)
 			query_unlock();
 			sleep(5);
 			query_lock();
+			retries++;
 			goto retry;
 		}
 
@@ -379,6 +408,7 @@ void fence_victims(struct fd *fd)
 			continue;
 		} else {
 			close_override(&override, cfgd_override_path);
+			retries++;
 			goto retry;
 		}
 	}


More information about the cluster-commits mailing list