commit 2df124f0bbe3d66c21d1fe58f446f73d7822f93e Author: Pete Zaitcev zaitcev@yahoo.com Date: Thu Aug 11 14:06:16 2011 -0600
Let us delete objects if back-end lost them
If for some reason the back-end loses an object, it cannot be deleted. This is a "should not happen" situation, which happens when a filesystem checker unlinks something or when sysadmin makes a mistake.
The obious solution is to proceed with removal from the database even if fails to find the object (a file in case of filesystem).
However, we cannot simply ignore errors, because this is how we discover that a non-existing object was deleted, and return 404. So, we make an extra lookup in the database, and only return an error if there was nothing there.
But wait! There's more: why cannot we look up an object before even reaching for the back-end? That is because we delete replicated objects by first removing them from the database, and then requesting redundant iwhd's to delete. They have to proceed with the deletion from their back-ends first, and then report 404 if they like. This is a little tricky, but works.
This patch also includes a test for this condition (t/delete-missing), written by Jim Meyering.
Tracking ticket: https://bugzilla.redhat.com/show_bug.cgi?id=710666
backend.c | 17 ++++++++++++- rest.c | 3 ++ t/Makefile.am | 1 + t/delete-missing | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) --- diff --git a/backend.c b/backend.c index efede7f..20a5d1c 100644 --- a/backend.c +++ b/backend.c @@ -1327,8 +1327,21 @@ fs_delete (const provider_t *prov, const char *bucket, const char *key, (void)key;
if (unlink(url+1) < 0) { - error (0, errno, _("%s: failed to unlink"), url+1); - return MHD_HTTP_NOT_FOUND; + switch (errno) { + case ENOENT: + /* Mostly harmless consistency loss. */ + DPRINTF("%s: already deleted\n", url+1); + return MHD_HTTP_OK; + case EPERM: + case EROFS: + case EACCES: + /* Possible correctable outside of iwhd by system tools. */ + error (0, errno, _("%s: failed to unlink"), url+1); + return MHD_HTTP_FORBIDDEN; + default: + error (0, errno, _("%s: failed to unlink"), url+1); + return MHD_HTTP_NOT_FOUND; + } }
return MHD_HTTP_OK; diff --git a/rest.c b/rest.c index a3aa834..eeae361 100644 --- a/rest.c +++ b/rest.c @@ -944,6 +944,7 @@ proxy_delete (void *cctx, struct MHD_Connection *conn, const char *url, size_t *data_size, void **rctx) { my_state *ms = *rctx; + char *etag; char *copied_url; char *bucket; char *key; @@ -963,6 +964,8 @@ proxy_delete (void *cctx, struct MHD_Connection *conn, const char *url, int rc = ms->thunk.prov->func_tbl->delete_func(main_prov, ms->bucket,ms->key,url); if (rc == MHD_HTTP_OK) { + if (meta_get_value(ms->bucket,ms->key,"_etag",&etag)) + return do_resp(conn,ms,MHD_HTTP_NOT_FOUND,NULL,NULL); copied_url = strdup(url); assert (copied_url); bucket = strtok_r(copied_url,"/",&stctx); diff --git a/t/Makefile.am b/t/Makefile.am index 6a668a1..ae2bdf3 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -18,6 +18,7 @@ TESTS = \ parse-test \ basic \ creation-code \ + delete-missing \ exercise \ provider \ replication \ diff --git a/t/delete-missing b/t/delete-missing new file mode 100644 index 0000000..9bc82d6 --- /dev/null +++ b/t/delete-missing @@ -0,0 +1,64 @@ +#!/bin/sh +# Ensure that when an object has disappeared from a backend (say via +# accidental deletion from an FS-backend), an attempt to delete it via +# iwhd still removes it from the database. + +. "${srcdir=.}/init.sh"; path_prepend_ .. + +mkdir FS mongod iwhd || framework_failure_ mkdir failed + +m_port=$(get_port $mongo_base_port $lock_dir/m-) \ + || fail_ "failed to get mongodb port" + +mongod --help > /dev/null || fail_ "the mongod program is not installed" + +mongod --port $m_port --pidfilepath mongod/pid --dbpath mongod > mongod.log 2>&1 & +mongo_pid=$! +cleanup_() { kill -9 $mongo_pid; } + +# Wait for up to 5 seconds for mongod to begin listening. +wait_for .1 50 'mongo localhost:$m_port < /dev/null' \ + || framework_failure_ mongod failed to start + +port=$(get_port 9095 $lock_dir/i-) || fail_ "failed to get iwhd port" + +printf '[{"path": "FS", "type": "fs", "name": "primary"}]\n' \ + > iwhd.cfg || fail=1 + +iwhd -v -p $port -c iwhd.cfg -d localhost:$m_port & +iwhd_pid=$! +cleanup_() { kill -9 $mongo_pid; kill $iwhd_pid; } + +# Wait for up to 5 seconds for iwhd to begin listening on $port. +wait_for .1 50 "curl -s http://localhost:$port" \ + || { echo iwhd failed to listen; Exit 1; } + + +curl_w() { curl --write-out '%{http_code}' "$@"; } + +# Deleting an FS-removed-behind-our-back object should succeed. +# With iwhd-0.98 and earlier, this would fail. +delete_FS_deleted_evokes_200() +{ + + # Create a bucket. + local b=http://localhost:$port/b9 + curl_w -XPUT $b > http-code || fail=1 + test "$(cat http-code)" = 201 || fail=1 + + # Create an object in that bucket. + local obj=$b/obj + curl_w -XPUT $obj > http-code || fail=1 + test "$(cat http-code)" = 201 || fail=1 + + # Remove that object via the file system. + rm FS/b9/obj || fail=1 + + # Attempt to remove it via iwhd. + curl_w -f -X DELETE $b/obj > http-code || fail=1 + test "$(cat http-code)" = 200 || fail=1 +} + +delete_FS_deleted_evokes_200 + +Exit $fail
iwhd-commit@lists.fedorahosted.org