[patch] Let us delete objects if back-end lost them

Pete Zaitcev zaitcev at redhat.com
Wed Aug 10 23:11:59 UTC 2011


If for some reason the back-end loses an object, it cannot be deleted.
This is a "should not happen" situation, which happens when a filesystem
checker unlinks something or when sysadmin makes a mistake.

The obious solution is to proceed with removal from the database even if
fails to find the object (a file in case of filesystem).

However, we cannot simply do ignore errors, because this is how we
discover that a non-existing object was deleted, and return 404.
So, we make an extra lookup in the database, and only return an
error if there was nothing there.

But wait! There's more: why cannot we look up an object before even
reaching for the back-end? That is because we delete replicated objects
by first removing them from the database, and then requesting redundant
iwhd's to delete. They have to proceed with the deletion from their
back-ends first, and then report 404 if they like. This is a little
tricky, but works.

This patch also includes a test for this condition (t/delete-missing),
written by Jim Meyering.

Tracking ticket:
https://bugzilla.redhat.com/show_bug.cgi?id=710666

---
 backend.c        |   17 ++++++++++-
 rest.c           |    3 ++
 t/Makefile.am    |    1 
 t/delete-missing |   64 +++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 83 insertions(+), 2 deletions(-)

diff --git a/backend.c b/backend.c
index efede7f..20a5d1c 100644
--- a/backend.c
+++ b/backend.c
@@ -1327,8 +1327,21 @@ fs_delete (const provider_t *prov, const char *bucket, const char *key,
 	(void)key;
 
 	if (unlink(url+1) < 0) {
-		error (0, errno, _("%s: failed to unlink"), url+1);
-		return MHD_HTTP_NOT_FOUND;
+		switch (errno) {
+		case ENOENT:
+			/* Mostly harmless consistency loss. */
+			DPRINTF("%s: already deleted\n", url+1);
+			return MHD_HTTP_OK;
+		case EPERM:
+		case EROFS:
+		case EACCES:
+			/* Possible correctable outside of iwhd by system tools. */
+			error (0, errno, _("%s: failed to unlink"), url+1);
+			return MHD_HTTP_FORBIDDEN;
+		default:
+			error (0, errno, _("%s: failed to unlink"), url+1);
+			return MHD_HTTP_NOT_FOUND;
+		}
 	}
 
 	return MHD_HTTP_OK;
diff --git a/rest.c b/rest.c
index a3aa834..eeae361 100644
--- a/rest.c
+++ b/rest.c
@@ -944,6 +944,7 @@ proxy_delete (void *cctx, struct MHD_Connection *conn, const char *url,
 	      size_t *data_size, void **rctx)
 {
 	my_state		*ms	= *rctx;
+	char			*etag;
 	char			*copied_url;
 	char			*bucket;
 	char			*key;
@@ -963,6 +964,8 @@ proxy_delete (void *cctx, struct MHD_Connection *conn, const char *url,
 	int rc = ms->thunk.prov->func_tbl->delete_func(main_prov,
 						       ms->bucket,ms->key,url);
 	if (rc == MHD_HTTP_OK) {
+		if (meta_get_value(ms->bucket,ms->key,"_etag",&etag))
+			return do_resp(conn,ms,MHD_HTTP_NOT_FOUND,NULL,NULL);
 		copied_url = strdup(url);
 		assert (copied_url);
 		bucket = strtok_r(copied_url,"/",&stctx);
diff --git a/t/Makefile.am b/t/Makefile.am
index 6a668a1..ae2bdf3 100644
--- a/t/Makefile.am
+++ b/t/Makefile.am
@@ -18,6 +18,7 @@ TESTS =						\
   parse-test					\
   basic						\
   creation-code					\
+  delete-missing				\
   exercise					\
   provider					\
   replication					\
diff --git a/t/delete-missing b/t/delete-missing
new file mode 100644
index 0000000..0c98c25
--- /dev/null
+++ b/t/delete-missing
@@ -0,0 +1,64 @@
+#!/bin/sh
+# Ensure that when an object has disappeared from a backend (say via
+# accidental deletion from an FS-backend), an attempt to delete it via
+# iwhd still removes it from the database.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+mkdir FS mongod iwhd || framework_failure_ mkdir failed
+
+m_port=$(get_port $mongo_base_port $lock_dir/m-) \
+  || fail_ "failed to get mongodb port"
+
+mongod --help > /dev/null || fail_ "the mongod program is not installed"
+
+mongod --port $m_port --pidfilepath mongod/pid --dbpath mongod > mongod.log 2>&1 &
+mongo_pid=$!
+cleanup_() { kill -9 $mongo_pid; }
+
+# Wait for up to 5 seconds for mongod to begin listening.
+wait_for .1 50 'mongo localhost:$m_port < /dev/null' \
+  || framework_failure_ mongod failed to start
+
+port=$(get_port 9095 $lock_dir/i-) || fail_ "failed to get iwhd port"
+
+printf '[{"path": "FS", "type": "fs", "name": "primary"}]\n' \
+  > iwhd.cfg || fail=1
+
+iwhd -v -p $port -c iwhd.cfg -d localhost:$m_port &
+iwhd_pid=$!
+cleanup_() { kill -9 $mongo_pid; kill $iwhd_pid; }
+
+# Wait for up to 5 seconds for iwhd to begin listening on $port.
+wait_for .1 50 "curl -s http://localhost:$port" \
+  || { echo iwhd failed to listen; Exit 1; }
+
+
+curl_w() { curl --write-out '%{http_code}' "$@"; }
+
+# Deleting an FS-removed-behind-our-back object should succeed.
+# With iwhd-0.97 and earlier, this would fail.
+delete_FS_deleted_evokes_200()
+{
+
+  # Create a bucket.
+  local b=http://localhost:$port/b9
+  curl_w -XPUT $b > http-code || fail=1
+  test "$(cat http-code)" = 201 || fail=1
+
+  # Create an object in that bucket.
+  local obj=$b/obj
+  curl_w -XPUT $obj > http-code || fail=1
+  test "$(cat http-code)" = 201 || fail=1
+
+  # Remove that object via the file system.
+  rm FS/b9/obj || fail=1
+
+  # Attempt to remove it via iwhd.
+  curl_w -f -X DELETE $b/obj > http-code || fail=1
+  test "$(cat http-code)" = 200 || fail=1
+}
+
+delete_FS_deleted_evokes_200
+
+Exit $fail



More information about the iwhd-devel mailing list