3 commits - xlators/cluster xlators/encryption
by Jeff Darcy
xlators/cluster/cloud/src/cloud.c | 88 +++++++++++++++++++++++++++--------
xlators/encryption/crypt/src/crypt.c | 1
2 files changed, 69 insertions(+), 20 deletions(-)
New commits:
commit f6f19fd80497c901d97c0679708353775aeffa1e
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Fri Dec 31 14:41:53 2010 -0500
Fix iobref leak.
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
index f0cd140..9d0aab3 100644
--- a/xlators/encryption/crypt/src/crypt.c
+++ b/xlators/encryption/crypt/src/crypt.c
@@ -363,6 +363,7 @@ crypt_writev_cbk (call_frame_t *frame,
if (op_ret > local->orig_size) {
op_ret = local->orig_size;
}
+ iobref_unref(local->iobref);
STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
return 0;
commit 97d36aecdec82e27515d0f0a84ce3f31218f42d2
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Fri Dec 31 14:41:33 2010 -0500
Fix mkdir, add unlink/rmdir.
diff --git a/xlators/cluster/cloud/src/cloud.c b/xlators/cluster/cloud/src/cloud.c
index be30d7c..ebfc68d 100644
--- a/xlators/cluster/cloud/src/cloud.c
+++ b/xlators/cluster/cloud/src/cloud.c
@@ -451,6 +451,42 @@ cloud_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
}
int
+cloud_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+ int op_ret, int op_errno,
+ struct iatt *preparent, struct iatt *postparent)
+{
+ STACK_UNWIND_STRICT(unlink,frame,op_ret,op_errno,preparent,postparent);
+ return 0;
+}
+
+int
+cloud_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+ xlator_t *next = get_subvol(this,frame->root->trans,"unlink");
+
+ STACK_WIND(frame,cloud_unlink_cbk,next,next->fops->unlink,loc);
+ return 0;
+}
+
+int
+cloud_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+ int op_ret, int op_errno,
+ struct iatt *preparent, struct iatt *postparent)
+{
+ STACK_UNWIND_STRICT(rmdir,frame,op_ret,op_errno,preparent,postparent);
+ return 0;
+}
+
+int
+cloud_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+{
+ xlator_t *next = get_subvol(this,frame->root->trans,"rmdir");
+
+ STACK_WIND(frame,cloud_rmdir_cbk,next,next->fops->rmdir,loc,flags);
+ return 0;
+}
+
+int
cloud_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
int op_ret, int op_errno, struct iatt *statpre,
struct iatt *statpost)
@@ -610,7 +646,9 @@ struct xlator_fops fops = {
.opendir = cloud_opendir,
.readdir = cloud_readdir,
.readdirp = cloud_readdirp,
+ .unlink = cloud_unlink,
.mkdir = cloud_mkdir,
+ .rmdir = cloud_rmdir,
.setattr = cloud_setattr,
};
commit dfaf331af4be4ea1461890c0243fff6d2bd41117
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Fri Dec 31 11:16:41 2010 -0500
Line-length fixes.
diff --git a/xlators/cluster/cloud/src/cloud.c b/xlators/cluster/cloud/src/cloud.c
index 4be48a3..be30d7c 100644
--- a/xlators/cluster/cloud/src/cloud.c
+++ b/xlators/cluster/cloud/src/cloud.c
@@ -36,18 +36,18 @@
* This translator exists primarily to run on the server and multiplex between
* subvolumes based on tenant identity, so that each tenant can have its own
* isolated namespace. To do this, we need to find out the tenant identity, so
- * we also handle that type of authentication here. Some day we might also do UID
- * mapping here.
+ * we also handle that type of authentication here. Some day we might also do
+ * UID mapping here.
*
* Right now the authentication is just simple password, passed via mkdir in a
- * special first-level subdirectory because mkdir requests are easily to generate
- * by hand and have convenient behavior in terms of fanning out to all bricks
- * even under DHT. In the future, a client-side translator will sit on top of
- * each brick and do a more sophisticated kind of authentication such as x.509 or
- * Kerberos.
+ * special first-level subdirectory because mkdir requests are easily to
+ * generate by hand and have convenient behavior in terms of fanning out to all
+ * bricks even under DHT. In the future, a client-side translator will sit on
+ * top of each brick and do a more sophisticated kind of authentication such as
+ * x.509 or Kerberos.
*/
-#define CLOUD_LEVEL GF_LOG_ERROR
+#define CLOUD_LEVEL GF_LOG_DEBUG
static char special_dir[] = "/_auth/";
@@ -124,14 +124,16 @@ cloud_check_auth (call_frame_t *frame, xlator_t *this, loc_t *loc)
continue;
}
if (strcmp(password,subvol_list[i].password)) {
- gf_log(this->name,CLOUD_LEVEL,"bad password for %s",subdir);
+ gf_log(this->name,CLOUD_LEVEL,"bad password for %s",
+ subdir);
continue;
}
if (!subvol_list[i].xlator) {
- gf_log(this->name,CLOUD_LEVEL,"no translator for %s",subdir);
+ gf_log(this->name,CLOUD_LEVEL,"no translator for %s",
+ subdir);
continue;
}
- gf_log(this->name,CLOUD_LEVEL,
+ gf_log(this->name,GF_LOG_WARNING,
"forcing this connection to subvol %s",subdir);
*(--password) = '=';
set_subvol(this,frame->root->trans,subvol_list[i].xlator);
@@ -157,13 +159,16 @@ int
cloud_mkdir (call_frame_t *frame, xlator_t *this,
loc_t *loc, mode_t mode, dict_t *params)
{
+ xlator_t *next;
+
if (!strncmp(loc->path,special_dir,sizeof(special_dir)-1)) {
cloud_check_auth(frame,this,loc);
return 0;
}
- STACK_WIND(frame,cloud_mkdir_cbk,FIRST_CHILD(this),
- FIRST_CHILD(this)->fops->mkdir,loc,mode,params);
+ next = get_subvol(this,frame->root->trans,"lookup");
+ STACK_WIND(frame,cloud_mkdir_cbk,next,next->fops->mkdir,
+ loc,mode,params);
return 0;
}
@@ -297,7 +302,8 @@ cloud_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
{
xlator_t *next = get_subvol(this,frame->root->trans,"truncate");
- STACK_WIND(frame,cloud_truncate_cbk,next,next->fops->truncate,loc,offset);
+ STACK_WIND(frame,cloud_truncate_cbk,next,next->fops->truncate,
+ loc,offset);
return 0;
}
@@ -306,7 +312,8 @@ cloud_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
{
xlator_t *next = get_subvol(this,frame->root->trans,"ftruncate");
- STACK_WIND(frame,cloud_truncate_cbk,next,next->fops->ftruncate,fd,offset);
+ STACK_WIND(frame,cloud_truncate_cbk,next,next->fops->ftruncate,
+ fd,offset);
return 0;
}
@@ -419,13 +426,14 @@ cloud_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
{
xlator_t *next = get_subvol(this,frame->root->trans,"readdir");
- STACK_WIND(frame,cloud_readdir_cbk,next,next->fops->readdir,fd,size,yoff);
+ STACK_WIND(frame,cloud_readdir_cbk,next,next->fops->readdir,
+ fd,size,yoff);
return 0;
}
int
-cloud_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
- int op_errno, gf_dirent_t *orig_entries)
+cloud_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+ int op_ret, int op_errno, gf_dirent_t *orig_entries)
{
STACK_UNWIND_STRICT(readdirp,frame,op_ret,op_errno,orig_entries);
return 0;
@@ -479,7 +487,8 @@ reconfigure (xlator_t *this, dict_t *options)
ret = dict_get_str (options, "log-level", &log_str);
if (!ret) {
- if (!is_gf_log_command(this, "trusted.glusterfs.set-log-level", log_str)) {
+ if (!is_gf_log_command(this, "trusted.glusterfs.set-log-level",
+ log_str)) {
gf_log (this->name, GF_LOG_DEBUG,
"changing log-level to %s", log_str);
}
@@ -543,7 +552,8 @@ init (xlator_t *this)
for (i = 0; subvol_list[i].name; ++i) {
for (trav = this->children; trav; trav = trav->next) {
if (!strcmp(subvol_list[i].name,trav->xlator->name)) {
- gf_log(this->name,CLOUD_LEVEL,"mapping %s -> %p",
+ gf_log(this->name,CLOUD_LEVEL,
+ "mapping %s -> %p",
subvol_list[i].name, trav->xlator->name);
subvol_list[i].xlator = trav->xlator;
break;
12 years, 8 months
4 commits - patches/wb-quota.patch scripts/filt-cloud.py scripts/filt-quota.py xlators/cluster
by Jeff Darcy
patches/wb-quota.patch | 32 ++++++++++++++++++++++++++++++++
scripts/filt-cloud.py | 9 ++++++---
scripts/filt-quota.py | 33 +++++++++++++++++++++++++++++++++
xlators/cluster/cloud/src/cloud.c | 18 ++++++++++++++++++
4 files changed, 89 insertions(+), 3 deletions(-)
New commits:
commit b4349209955b1f234a9838d73b7c686fbb6d7fdb
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Thu Dec 23 15:01:52 2010 -0500
Patch write-behind to play better with quota.
diff --git a/patches/wb-quota.patch b/patches/wb-quota.patch
new file mode 100644
index 0000000..a31b9e8
--- /dev/null
+++ b/patches/wb-quota.patch
@@ -0,0 +1,32 @@
+diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c
+index 43956a9..d97812a 100644
+--- a/xlators/performance/write-behind/src/write-behind.c
++++ b/xlators/performance/write-behind/src/write-behind.c
+@@ -1667,7 +1667,8 @@ __wb_get_other_requests (list_head_t *list, list_head_t *other_requests)
+ int32_t
+ wb_stack_unwind (list_head_t *unwinds)
+ {
+- struct iatt buf = {0,};
++ struct iatt prebuf = {0,};
++ struct iatt postbuf = {0,};
+ wb_request_t *request = NULL, *dummy = NULL;
+ call_frame_t *frame = NULL;
+ wb_local_t *local = NULL;
+@@ -1678,8 +1679,15 @@ wb_stack_unwind (list_head_t *unwinds)
+ frame = request->stub->frame;
+ local = frame->local;
+
+- STACK_UNWIND (frame, local->op_ret, local->op_errno, &buf,
+- &buf);
++ /*
++ * There are probably a lot of other problems with returning
++ * these bogus iatts, but this fix at least gives us enough
++ * information for features/quota to work (sort of).
++ */
++ postbuf.ia_blocks = (request->write_size + 511) / 512;
++
++ STACK_UNWIND (frame, local->op_ret, local->op_errno,
++ &prebuf, &postbuf);
+
+ ret = wb_request_unref (request);
+ if (ret == 0) {
commit 06b429996ff9dd0d323b2f4f3bf919eb7c845939
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Thu Dec 23 14:45:14 2010 -0500
Handle flush to avoid spurious errors from default wrapper.
diff --git a/xlators/cluster/cloud/src/cloud.c b/xlators/cluster/cloud/src/cloud.c
index 73d3c46..4be48a3 100644
--- a/xlators/cluster/cloud/src/cloud.c
+++ b/xlators/cluster/cloud/src/cloud.c
@@ -372,6 +372,23 @@ cloud_writev (call_frame_t *frame, xlator_t *this,
}
int
+cloud_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+ int op_ret, int op_errno)
+{
+ STACK_UNWIND_STRICT(flush,frame,op_ret,op_errno);
+ return 0;
+}
+
+int
+cloud_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+{
+ xlator_t *next = get_subvol(this,frame->root->trans,"flush");
+
+ STACK_WIND(frame,cloud_flush_cbk,next,next->fops->flush,fd);
+ return 0;
+}
+
+int
cloud_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
int op_ret, int op_errno, fd_t *fd)
{
@@ -579,6 +596,7 @@ struct xlator_fops fops = {
.open = cloud_open,
.readv = cloud_readv,
.writev = cloud_writev,
+ .flush = cloud_flush,
.opendir = cloud_opendir,
.readdir = cloud_readdir,
.readdirp = cloud_readdirp,
commit 9d318d1ffa5bed64de03cb3e0f6492516fb0d854
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Thu Dec 23 14:43:52 2010 -0500
Add script to auto-insert quota translator.
diff --git a/scripts/filt-quota.py b/scripts/filt-quota.py
new file mode 100755
index 0000000..36a5489
--- /dev/null
+++ b/scripts/filt-quota.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010 Red Hat, Inc.
+#
+# This file is part of CloudFS.
+#
+# CloudFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License *
+# along with CloudFS. If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import volfilter
+
+if len(sys.argv) != 3:
+ print >> sys.stderr, "Usage: %s orig_volfile quota" % sys.argv[0]
+ sys.exit(1)
+
+graph, last = volfilter.load(sys.argv[1])
+new_xl = volfilter.Translator(last.name+"-quota")
+new_xl.type = "features/quota"
+new_xl.opts = {"disk-usage-limit":sys.argv[2]}
+new_xl.subvols = last.subvols
+last.subvols = [new_xl]
+volfilter.generate(graph,last)
commit 070f669bb8b9d15f8341a9a4f18af0dc78b41b04
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Thu Dec 23 14:43:01 2010 -0500
Change per-tenant volume name to match translator's usage.
diff --git a/scripts/filt-cloud.py b/scripts/filt-cloud.py
index a055556..7389107 100755
--- a/scripts/filt-cloud.py
+++ b/scripts/filt-cloud.py
@@ -21,14 +21,17 @@ import copy
import sys
import volfilter
-def copy_stack (old_xl,suffix):
- new_name = old_xl.name + "-" + suffix
+def copy_stack (old_xl,suffix,recursive=False):
+ if recursive:
+ new_name = old_xl.name + "-" + suffix
+ else:
+ new_name = suffix
new_xl = volfilter.Translator(new_name)
new_xl.type = old_xl.type
# The results with normal assignment here are . . . amusing.
new_xl.opts = copy.deepcopy(old_xl.opts)
for sv in old_xl.subvols:
- new_xl.subvols.append(copy_stack(sv,suffix))
+ new_xl.subvols.append(copy_stack(sv,suffix,True))
# Patch up the path at the bottom.
if new_xl.type == "storage/posix":
new_xl.opts["directory"] += ("/" + suffix)
12 years, 9 months
README.md scripts/filt-cloud.py scripts/filt-crypto.py scripts/filt-log-io.py scripts/volfilter.py
by Jeff Darcy
README.md | 19 ++++++++
scripts/filt-cloud.py | 61 +++++++++++++++++++++++++++
scripts/filt-crypto.py | 35 +++++++++++++++
scripts/filt-log-io.py | 33 ++++++++++++++
scripts/volfilter.py | 108 +++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 256 insertions(+)
New commits:
commit 27c56982c5247983c22e00a596f2e48f6384ef4c
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Wed Dec 22 15:22:56 2010 -0500
Add volfile-manipulation scripts.
diff --git a/README.md b/README.md
index 105d5c8..6af15bd 100644
--- a/README.md
+++ b/README.md
@@ -103,6 +103,25 @@ a change globally, you'll need to do the following:
See the CONFIG.txt in each translator's directory for instructions specific
to that translator.
+The .../scripts directory contains some Python scripts that can help automate
+the process of modifying volfiles. Specifically:
+
+* filt-log-io.py: inserts a debug/log-io translator between a protocol/server
+ volume and each of its subvolumes
+
+* filt-crypto.py: inserts an encryption/crypto translator on top of each
+ protocol/client volume. Also disables performance/quick-read, which is
+ incompatible with encryption/crypto.
+
+* filt-cloud.py: replaces a simple translator "stack" (from storage/posix up
+ to whatever is below protocol/server) with one such stack per named tenant,
+ plus a cluster/cloud translator to tie them together.
+
+Running these scripts after each gluster "volume create" or "volume set"
+command should generate a new volfile with the desired enhancements. They use
+a common volfile parsing/modification library that might be useful for other
+tasks as well.
+
Work List
---------
diff --git a/scripts/filt-cloud.py b/scripts/filt-cloud.py
new file mode 100755
index 0000000..a055556
--- /dev/null
+++ b/scripts/filt-cloud.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+
+# Copyright (c) 2010 Red Hat, Inc.
+#
+# This file is part of CloudFS.
+#
+# CloudFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License *
+# along with CloudFS. If not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import sys
+import volfilter
+
+def copy_stack (old_xl,suffix):
+ new_name = old_xl.name + "-" + suffix
+ new_xl = volfilter.Translator(new_name)
+ new_xl.type = old_xl.type
+ # The results with normal assignment here are . . . amusing.
+ new_xl.opts = copy.deepcopy(old_xl.opts)
+ for sv in old_xl.subvols:
+ new_xl.subvols.append(copy_stack(sv,suffix))
+ # Patch up the path at the bottom.
+ if new_xl.type == "storage/posix":
+ new_xl.opts["directory"] += ("/" + suffix)
+ return new_xl
+
+if len(sys.argv) < 3:
+ print >> sys.stderr, "Usage: %s orig_volfile tenant..." % sys.argv[0]
+ sys.exit(1)
+
+graph, last = volfilter.load(sys.argv[1])
+
+# Add the pre-authentication subvolume.
+junk_sv = last.subvols[0]
+while junk_sv.type != "storage/posix":
+ junk_sv = junk_sv.subvols[0]
+subvols = [copy_stack(junk_sv,"junk")]
+
+# Add the per-tenant subvolumes.
+for suffix in sys.argv[2:]:
+ subvols.append(copy_stack(last.subvols[0],suffix))
+
+# One cloud to bring them all...
+my_xl = volfilter.Translator(last.subvols[0].name)
+my_xl.type = "cluster/cloud"
+my_xl.subvols = subvols
+
+# ...and splice it in.
+last.subvols = [my_xl]
+
+volfilter.generate(graph,last)
diff --git a/scripts/filt-crypto.py b/scripts/filt-crypto.py
new file mode 100755
index 0000000..6a1f0a6
--- /dev/null
+++ b/scripts/filt-crypto.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010 Red Hat, Inc.
+#
+# This file is part of CloudFS.
+#
+# CloudFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License *
+# along with CloudFS. If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import volfilter
+
+if len(sys.argv) != 3:
+ print >> sys.stderr, "Usage: %s orig_volfile key" % sys.argv[0]
+ sys.exit(1)
+
+graph, last = volfilter.load(sys.argv[1])
+# Quick-read bypasses the calls that crypto uses.
+to_do = [xl for xl in graph.itervalues() if xl.type == "performance/quick-read"]
+for td in to_do:
+ volfilter.delete(graph,td)
+to_do = [xl for xl in graph.itervalues() if xl.type == "protocol/client"]
+for td in to_do:
+ volfilter.push_filter(graph,td,"encryption/crypto",{"key":sys.argv[2]})
+volfilter.generate(graph,last)
diff --git a/scripts/filt-log-io.py b/scripts/filt-log-io.py
new file mode 100755
index 0000000..ac3ced8
--- /dev/null
+++ b/scripts/filt-log-io.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010 Red Hat, Inc.
+#
+# This file is part of CloudFS.
+#
+# CloudFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License *
+# along with CloudFS. If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import volfilter
+
+if len(sys.argv) != 2:
+ print >> sys.stderr, "Usage: %s orig_volfile" % sys.argv[0]
+ sys.exit(1)
+
+graph, last = volfilter.load(sys.argv[1])
+to_do = [xl for xl in graph.itervalues() if xl.type == "protocol/server"]
+for td in to_do:
+ for sv in td.subvols:
+ # TBD: deal with multiple volumes/logs per server
+ volfilter.push_filter(graph,sv,"debug/log-io")
+volfilter.generate(graph,last)
diff --git a/scripts/volfilter.py b/scripts/volfilter.py
new file mode 100755
index 0000000..3fd6250
--- /dev/null
+++ b/scripts/volfilter.py
@@ -0,0 +1,108 @@
+ #!/usr/bin/env python
+
+ # Copyright (c) 2010 Red Hat, Inc.
+ #
+ # This file is part of CloudFS.
+ #
+ # CloudFS is free software: you can redistribute it and/or modify it under the
+ # terms of the GNU Affero General Public License as published by the Free
+ # Software Foundation, either version 3 of the License, or (at your option)
+ # any later version.
+ #
+ # CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+ # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ # details.
+ #
+ # You should have received a copy of the GNU Affero General Public License *
+ # along with CloudFS. If not, see <http://www.gnu.org/licenses/>.
+
+import string
+import sys
+
+class Translator:
+ def __init__ (self, name):
+ self.name = name
+ self.type = ""
+ self.opts = {}
+ self.subvols = []
+ self.dumped = False
+ def __repr__ (self):
+ return "<Translator %s>" % self.name
+
+def load (path):
+ fp = file(path,"r")
+ all_xlators = {}
+ xlator = None
+ last_xlator = None
+ while True:
+ text = fp.readline()
+ if text == "":
+ break
+ text = text.split()
+ if not len(text):
+ continue
+ if text[0] == "volume":
+ if xlator:
+ raise RuntimeError, "nested volume definition"
+ xlator = Translator(text[1])
+ continue
+ if not xlator:
+ raise RuntimeError, "text outside volume definition"
+ if text[0] == "type":
+ xlator.type = text[1]
+ continue
+ if text[0] == "option":
+ xlator.opts[text[1]] = string.join(text[2:])
+ continue
+ if text[0] == "subvolumes":
+ for sv in text[1:]:
+ xlator.subvols.append(all_xlators[sv])
+ continue
+ if text[0] == "end-volume":
+ all_xlators[xlator.name] = xlator
+ last_xlator = xlator
+ xlator = None
+ continue
+ raise RuntimeError, "unrecognized keyword %s" % text[0]
+ if xlator:
+ raise RuntimeError, "unclosed volume definition"
+ return all_xlators, last_xlator
+
+def generate (graph, last):
+ for sv in last.subvols:
+ if not sv.dumped:
+ generate(graph,sv)
+ print ""
+ sv.dumped = True
+ print "volume %s" % last.name
+ print " type %s" % last.type
+ for k, v in last.opts.iteritems():
+ print " option %s %s" % (k, v)
+ if last.subvols:
+ print " subvolumes %s" % string.join(
+ [ sv.name for sv in last.subvols ])
+ print "end-volume"
+
+def push_filter (graph, old_xl, filt_type, opts={}):
+ suffix = string.split(old_xl.type,"/")[1]
+ new_xl = Translator(old_xl.name+"-"+suffix)
+ new_xl.type = old_xl.type
+ new_xl.opts = old_xl.opts
+ new_xl.subvols = old_xl.subvols
+ graph[new_xl.name] = new_xl
+ old_xl.type = filt_type
+ old_xl.opts = opts
+ old_xl.subvols = [new_xl]
+
+def delete (graph, victim):
+ if len(victim.subvols) != 1:
+ raise RuntimeError, "attempt to delete non-unary translator"
+ for xl in graph.itervalues():
+ while xl.subvols.count(victim):
+ i = xl.subvols.index(victim)
+ xl.subvols[i] = victim.subvols[0]
+
+if __name__ == "__main__":
+ graph, last = load(sys.argv[1])
+ generate(graph,last)
12 years, 9 months
patches/rebalance-hash.patch
by Jeff Darcy
patches/rebalance-hash.patch | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
New commits:
commit 1ae49476cf762e5da16c096ba60f80d20427b6c4
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Fri Dec 17 14:30:08 2010 -0500
Fix sizeof typo (Coverity).
diff --git a/patches/rebalance-hash.patch b/patches/rebalance-hash.patch
index 27dfdae..5cb8115 100644
--- a/patches/rebalance-hash.patch
+++ b/patches/rebalance-hash.patch
@@ -1,5 +1,5 @@
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
-index 8a5f45d..17d2b7f 100644
+index 8a5f45d..8c3b51f 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -171,8 +171,18 @@ dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
@@ -17,7 +17,7 @@ index 8a5f45d..17d2b7f 100644
+ * originals, without trying to duplicate the layout algorithms in a
+ * separate piece of code.
+ */
-+ if (!strncmp(name,pfx,sizeof(pfx-1))) {
++ if (!strncmp(name,pfx,sizeof(pfx)-1)) {
+ name += sizeof(pfx) - 1;
+ }
ret = dht_hash_compute (layout->type, name, &hash);
12 years, 9 months
patches/inode-gen.patch patches/rebalance-hash.patch patches/server-locking.patch
by Jeff Darcy
patches/inode-gen.patch | 12 +++
patches/rebalance-hash.patch | 25 +++++++
patches/server-locking.patch | 147 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 184 insertions(+)
New commits:
commit 47e657847d6a31f978dd5b69c5ea28a806452127
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Fri Dec 17 13:00:54 2010 -0500
Stray patches relevant to CloudFS.
diff --git a/patches/inode-gen.patch b/patches/inode-gen.patch
new file mode 100644
index 0000000..07e96fd
--- /dev/null
+++ b/patches/inode-gen.patch
@@ -0,0 +1,12 @@
+diff --git a/libglusterfs/src/inode.h b/libglusterfs/src/inode.h
+index 15a4f98..3c95407 100644
+--- a/libglusterfs/src/inode.h
++++ b/libglusterfs/src/inode.h
+@@ -104,6 +104,7 @@ struct _inode {
+ struct list_head list; /* active/lru/purge */
+
+ struct _inode_ctx *_ctx; /* replacement for dict_t *(inode->ctx) */
++ uint64_t gen; /* for locking code */
+ };
+
+
diff --git a/patches/rebalance-hash.patch b/patches/rebalance-hash.patch
new file mode 100644
index 0000000..27dfdae
--- /dev/null
+++ b/patches/rebalance-hash.patch
@@ -0,0 +1,25 @@
+diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
+index 8a5f45d..17d2b7f 100644
+--- a/xlators/cluster/dht/src/dht-layout.c
++++ b/xlators/cluster/dht/src/dht-layout.c
+@@ -171,8 +171,18 @@ dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
+ xlator_t *subvol = NULL;
+ int i = 0;
+ int ret = 0;
+-
+-
++ static char pfx[] = ".dht.";
++
++ /*
++ * Make sure that things with the prefix get hashed exactly the same
++ * as without. Why? So that file copies created during rebalance go
++ * exactly where they should be after they're renamed over their
++ * originals, without trying to duplicate the layout algorithms in a
++ * separate piece of code.
++ */
++ if (!strncmp(name,pfx,sizeof(pfx-1))) {
++ name += sizeof(pfx) - 1;
++ }
+ ret = dht_hash_compute (layout->type, name, &hash);
+ if (ret != 0) {
+ gf_log (this->name, GF_LOG_DEBUG,
diff --git a/patches/server-locking.patch b/patches/server-locking.patch
new file mode 100644
index 0000000..aeb6d54
--- /dev/null
+++ b/patches/server-locking.patch
@@ -0,0 +1,147 @@
+diff --git a/xlators/protocol/server/src/server3_1-fops.c b/xlators/protocol/server/src/server3_1-fops.c
+index fb3010e..f685786 100644
+--- a/xlators/protocol/server/src/server3_1-fops.c
++++ b/xlators/protocol/server/src/server3_1-fops.c
+@@ -31,6 +31,64 @@
+
+ #include "md5.h"
+
++/*
++ * This is really a hack, to demonstrate the general optimistic-locking
++ * approach. For real life, we should store this in a more sophisticated
++ * way.
++ */
++
++#define MAX_OP_LOCKS 10
++
++struct {
++ void *ptr1; /* NULL means not in use */
++ void *ptr2;
++ uint64_t value;
++} op_locks[MAX_OP_LOCKS];
++
++int
++store_op_lock (void *p1, void *p2, uint64_t v)
++{
++ int i;
++
++ for (i = 0; i < MAX_OP_LOCKS; ++i) {
++ if (!op_locks[i].ptr1) {
++ op_locks[i].ptr1 = p1;
++ op_locks[i].ptr2 = p2;
++ op_locks[i].value = v;
++ return 0;
++ }
++ }
++
++ return -1;
++}
++
++int
++fetch_op_lock (void *p1, void *p2, uint64_t *v)
++{
++ int i;
++
++ for (i = 0; i < MAX_OP_LOCKS; ++i) {
++ if ((op_locks[i].ptr1 == p1) && (op_locks[i].ptr2 == p2)) {
++ *v = op_locks[i].value;
++ return 0;
++ }
++ }
++
++ return -1;
++}
++
++void
++delete_op_lock (void *p1, void *p2)
++{
++ int i;
++
++ for (i = 0; i < MAX_OP_LOCKS; ++i) {
++ if ((op_locks[i].ptr1 == p1) && (op_locks[i].ptr2 == p2)) {
++ op_locks[i].ptr1 = NULL;
++ return;
++ }
++ }
++}
+
+ /* Callback function section */
+ int
+@@ -2277,17 +2335,51 @@ err:
+ return 0;
+ }
+
++void
++server_special_setxattr (dict_t *dict, char *key, data_t *value, void *ctx)
++{
++ char **lock_str = ctx;
++
++ /* Only return the first match. */
++ if (*lock_str) {
++ return;
++ }
++
++ if (!strcmp(key,"user.glusterfs.lock")) {
++ *lock_str = data_to_str(value);
++ }
++}
+
+ int
+ server_setxattr_resume (call_frame_t *frame, xlator_t *bound_xl)
+ {
+ server_state_t *state = NULL;
++ char *lock_str = NULL;
++ inode_t *inode = NULL;
++ int ret = 0;
+
+ state = CALL_STATE (frame);
+
+ if (state->resolve.op_ret != 0)
+ goto err;
+
++ dict_foreach(state->dict,server_special_setxattr,&lock_str);
++ if (lock_str) {
++ inode = state->loc.inode;
++ gf_log(bound_xl->name,GF_LOG_WARNING,
++ "got lock request for %d from %p",
++ inode->ino, state->conn);
++ ret = store_op_lock(inode,state->conn,inode->gen);
++ if (ret != 0) {
++ gf_log(bound_xl->name,GF_LOG_ERROR,
++ "lock request failed (store_op_lock) for %d",
++ inode->ino);
++ state->resolve.op_ret = -1;
++ state->resolve.op_errno = ENOMEM;
++ goto err;
++ }
++ }
++
+ STACK_WIND (frame, server_setxattr_cbk,
+ bound_xl, bound_xl->fops->setxattr,
+ &state->loc, state->dict, state->flags);
+@@ -2389,12 +2481,25 @@ int
+ server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
+ {
+ server_state_t *state = NULL;
++ inode_t *inode = NULL;
++ uint64_t old_val = 0;
+
+ state = CALL_STATE (frame);
+
+ if (state->resolve.op_ret != 0)
+ goto err;
+
++ inode = state->fd->inode;
++ if (fetch_op_lock(inode,state->conn,&old_val) == 0) {
++ if (old_val != inode->gen) {
++ gf_log(bound_xl->name,GF_LOG_WARNING,
++ "would reject write for %d from %p",
++ inode->ino, state->conn);
++ }
++ delete_op_lock(inode,state->conn);
++ }
++ ++(inode->gen);
++
+ STACK_WIND (frame, server_writev_cbk,
+ bound_xl, bound_xl->fops->writev,
+ state->fd, state->payload_vector, state->payload_count,
12 years, 9 months
Changes to 'master'
by Jeff Darcy
New branch 'master' available with the following commits:
commit 3f5cc30cd9f403cb2848096e67f2f626986b6e9a
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Wed Dec 15 11:23:18 2010 -0500
Initial version of CloudFS for public consumption.
12 years, 9 months