FYI, mongo restart leads to iwhd abort

Jim Meyering jim at meyering.net
Wed Sep 21 16:40:40 UTC 2011


Jim Meyering wrote:
> Trying to test Pete's restart-handling patch, I managed to make iwhd
> abort due to an unhandled mongo::SocketException.  (only system I've
> tried so far is my F16 i686 laptop) Downloading 400MB of debuginfo,
> then I'll know more...

Hi Pete,

Thanks for your reconnect patch.
I wrote a test for it (below) and discovered
that there is one more exception we need to handle:
mongo::SocketException.  I've added a commit message
to your patch and will amend it to include the
new SocketException handler.
I also added a NEWS entry describing the bug fix.

Subject: [PATCH] meta.cpp: handle SocketException, too
...
diff --git a/meta.cpp b/meta.cpp
index e88839a..fa2c591 100644
--- a/meta.cpp
+++ b/meta.cpp
@@ -164,6 +164,10 @@ RepoMeta::GetCursor (Query &q)
 		log_msg(IW_LOG_INFO, "reconnection to %s failed", addr);
 		curs.reset();
 	}
+	catch (SocketException &ce) {
+		log_msg(IW_LOG_INFO, "reconnection to %s failed", addr);
+		curs.reset();
+	}
 	return curs;
 }

--
1.7.6.2


Here are the two change sets:


>From 7033bc854b623e70eb63aea780edbeccdc2944d7 Mon Sep 17 00:00:00 2001
From: Pete Zaitcev <zaitcev at redhat.com>
Date: Wed, 21 Sep 2011 06:45:20 +0200
Subject: [PATCH 1/2] iwhd: handle a lost-then-restored mongo connection

If I read the code right, the intent was that client.connect() reset
the .failed flag.  But actually it does not.  So, although we actually
reconnect, isFailed() always evaluates to true and we get no cursor.
Unfortunately, it seems that there is NO WAY to reset .failed, except
through checkConnect(), and that 1) works only if auto-reconnect is
enabled, and 2) is not a public method.  It is called by query()
itself though.

This patch deals with all this simply by enabling auto-reconnect,
which basically does exactly what we did explicitly before, only it
happens inside query() now.
* meta.cpp: Handle SocketException, too
---
 meta.cpp |   46 ++++++++++++++++++++++------------------------
 1 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/meta.cpp b/meta.cpp
index 7b3a5d7..fa2c591 100644
--- a/meta.cpp
+++ b/meta.cpp
@@ -118,7 +118,8 @@ public:

 static RepoMeta *it;

-RepoMeta::RepoMeta ()
+RepoMeta::RepoMeta () :
+    client(true)
 {
 	if (!verbose) {
 		cout.rdbuf(0);
@@ -155,27 +156,18 @@ auto_ptr<DBClientCursor>
 RepoMeta::GetCursor (Query &q)
 {
 	auto_ptr<DBClientCursor> curs;
-	bool looping = false;
-
-	for (;;) {
-		if (!client.isFailed()) {
-		        curs = client.query(MAIN_TBL,q);
-		        if (curs.get()) {
-		                break;
-		        }
-		}
-		if (looping) {
-		        break;
-		}
-		try {
-		        client.connect(addr);
-		}
-		catch (ConnectException &ce) {
-		        log_msg(IW_LOG_INFO, "reconnection to %s failed", addr);
-		}
-		looping = true;
-	}

+	try {
+		curs = client.query(MAIN_TBL,q);
+	}
+	catch (ConnectException &ce) {
+		log_msg(IW_LOG_INFO, "reconnection to %s failed", addr);
+		curs.reset();
+	}
+	catch (SocketException &ce) {
+		log_msg(IW_LOG_INFO, "reconnection to %s failed", addr);
+		curs.reset();
+	}
 	return curs;
 }

@@ -434,7 +426,13 @@ RepoQuery::RepoQuery (const char *bucket, const char *key, const char *qstr,
 		expr = NULL;
 	}

-	curs = parent.GetCursor(q).release();
+	tmp = parent.GetCursor(q);
+	if (!tmp.get()) {
+		/* TBD: What to do if this happens? */
+		log_msg(IW_LOG_INFO, "no cursor");
+		return;
+	}
+	curs = tmp.release();
 	bucket = NULL;
 	key = NULL;
 }
@@ -582,7 +580,7 @@ RepoMeta::GetSize (const char *bucket, const char *key)

 	q = QUERY("_bucket"<<bucket<<"_key"<<key);
 	curs = GetCursor(q);
-
+	/* TBD: What to do if GetCursor() fails to reconnect? */
 	if (!curs->more()) {
 		return 0;
 	}
@@ -651,7 +649,7 @@ RepoMeta::GetAttrList (const char *bucket, const char *key)

 	q = QUERY("_bucket"<<bucket<<"_key"<<key);
 	curs = GetCursor(q);
-
+	/* TBD: What to do if GetCursor() fails to reconnect? */
 	if (!curs->more()) {
 		return NULL;
 	}
--
1.7.6.2


>From 2e78218f28078cdd57181827cca16f3d30cf1998 Mon Sep 17 00:00:00 2001
From: Jim Meyering <meyering at redhat.com>
Date: Wed, 21 Sep 2011 17:55:33 +0200
Subject: [PATCH 2/2] tests: add a test to exercise mongo-reconnect processing

* t/mongo-restart: New test.
* t/Makefile.am (TESTS): Add it here.
* NEWS (Bug fixes): Mention it.
---
 NEWS            |    3 ++
 t/Makefile.am   |    1 +
 t/mongo-restart |   80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 84 insertions(+), 0 deletions(-)
 create mode 100755 t/mongo-restart

diff --git a/NEWS b/NEWS
index be681f9..c7467a9 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,9 @@ iwhd NEWS                                                   -*- outline -*-

 ** Bug fixes

+  iwhd now continues to work properly when mongo dies and is restarted.
+  Before, it would dump core due to an unhandled SocketException.
+
   iwhd's /etc/init.d startup script now reports failure reliably, for
   example when failing to start because no MongoDB server is running.
   Before it would print [FAILED], but mistakenly exit 0 (successful).
diff --git a/t/Makefile.am b/t/Makefile.am
index ae2bdf3..a3b39d7 100644
--- a/t/Makefile.am
+++ b/t/Makefile.am
@@ -20,6 +20,7 @@ TESTS =						\
   creation-code					\
   delete-missing				\
   exercise					\
+  mongo-restart					\
   provider					\
   replication					\
   auto						\
diff --git a/t/mongo-restart b/t/mongo-restart
new file mode 100755
index 0000000..1447952
--- /dev/null
+++ b/t/mongo-restart
@@ -0,0 +1,80 @@
+#!/bin/sh
+# Ensure iwhd recovers after mongo stop/restart.
+# Before 0.99 it did not.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+mkdir FS mongod iwhd || framework_failure_ mkdir failed
+
+ulimit -c unlimited
+m_port=$(get_port $mongo_base_port $lock_dir/m-) \
+  || fail_ "failed to get mongodb port"
+
+mongod --help > /dev/null || fail_ "the mongod program is not installed"
+
+mongod --port $m_port --pidfilepath mongod/pid --dbpath mongod > mongod.log 2>&1 &
+mongo_pid=$!
+cleanup_() { kill -9 $mongo_pid; }
+
+# Wait for up to 5 seconds for mongod to begin listening.
+wait_for .1 50 'mongo localhost:$m_port < /dev/null' \
+  || framework_failure_ mongod failed to start
+
+port=$(get_port 9095 $lock_dir/i-) || fail_ "failed to get iwhd port"
+
+printf '[{"path": "FS", "type": "fs", "name": "primary"}]\n' \
+  > iwhd.cfg || fail=1
+
+iwhd -v -p $port -c iwhd.cfg -d localhost:$m_port &
+iwhd_pid=$!
+cleanup_() { kill -9 $mongo_pid; kill $iwhd_pid; }
+
+# Wait for up to 5 seconds for iwhd to begin listening on $port.
+wait_for .1 50 "curl -s http://localhost:$port" \
+  || fail_ iwhd failed to listen
+
+curl_w() { curl --write-out '%{http_code}' "$@"; }
+
+# Show that bucket,object,attribute-creation all evoke 201.
+creation_evokes_201()
+{
+  local bucket_name=$1
+
+  # Create a bucket.
+  local b=http://localhost:$port/$bucket_name
+  curl_w -XPUT $b > http-code || fail=1
+  test "$(cat http-code)" = 201 || fail=1
+
+  # Create an object in that bucket.
+  local obj=$b/obj
+  curl_w -XPUT $obj > http-code || fail=1
+  test "$(cat http-code)" = 201 || fail=1
+
+  # Create an attribute.
+  local attr=$obj/color
+  printf blue | curl_w -T - $attr > http-code || fail=1
+  test "$(cat http-code)" = 201 || fail=1
+}
+
+
+creation_evokes_201 b1
+
+# So far, this is identical to the creation-code test.
+# But now, we kill mongo and restart it, and then repeat the test.
+
+kill $mongo_pid
+
+# Wait for mongo to stop listening.
+no_mongo() { mongo localhost:$m_port < /dev/null; test $? != 0; }
+wait_for .1 50 no_mongo || fail_ failed to kill mongo
+
+mongod --port $m_port --pidfilepath mongod/pid --dbpath mongod &
+mongo_pid=$!
+
+# Wait for up to 5 seconds for mongod to begin listening.
+wait_for .1 50 'mongo localhost:$m_port < /dev/null' \
+  || framework_failure_ mongod failed to start
+
+creation_evokes_201 b2
+
+Exit $fail
--
1.7.6.2


More information about the iwhd-devel mailing list