README.md | 19 ++++++++
scripts/filt-cloud.py | 61 +++++++++++++++++++++++++++
scripts/filt-crypto.py | 35 +++++++++++++++
scripts/filt-log-io.py | 33 ++++++++++++++
scripts/volfilter.py | 108 +++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 256 insertions(+)
New commits:
commit 27c56982c5247983c22e00a596f2e48f6384ef4c
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Wed Dec 22 15:22:56 2010 -0500
Add volfile-manipulation scripts.
diff --git a/README.md b/README.md
index 105d5c8..6af15bd 100644
--- a/README.md
+++ b/README.md
@@ -103,6 +103,25 @@ a change globally, you'll need to do the following:
See the CONFIG.txt in each translator's directory for instructions specific
to that translator.
+The .../scripts directory contains some Python scripts that can help automate
+the process of modifying volfiles. Specifically:
+
+* filt-log-io.py: inserts a debug/log-io translator between a protocol/server
+ volume and each of its subvolumes
+
+* filt-crypto.py: inserts an encryption/crypto translator on top of each
+ protocol/client volume. Also disables performance/quick-read, which is
+ incompatible with encryption/crypto.
+
+* filt-cloud.py: replaces a simple translator "stack" (from storage/posix up
+ to whatever is below protocol/server) with one such stack per named tenant,
+ plus a cluster/cloud translator to tie them together.
+
+Running these scripts after each gluster "volume create" or "volume
set"
+command should generate a new volfile with the desired enhancements. They use
+a common volfile parsing/modification library that might be useful for other
+tasks as well.
+
Work List
---------
diff --git a/scripts/filt-cloud.py b/scripts/filt-cloud.py
new file mode 100755
index 0000000..a055556
--- /dev/null
+++ b/scripts/filt-cloud.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+
+# Copyright (c) 2010 Red Hat, Inc.
+#
+# This file is part of CloudFS.
+#
+# CloudFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License *
+# along with CloudFS. If not, see <
http://www.gnu.org/licenses/>.
+
+import copy
+import sys
+import volfilter
+
+def copy_stack (old_xl,suffix):
+ new_name = old_xl.name + "-" + suffix
+ new_xl = volfilter.Translator(new_name)
+ new_xl.type = old_xl.type
+ # The results with normal assignment here are . . . amusing.
+ new_xl.opts = copy.deepcopy(old_xl.opts)
+ for sv in old_xl.subvols:
+ new_xl.subvols.append(copy_stack(sv,suffix))
+ # Patch up the path at the bottom.
+ if new_xl.type == "storage/posix":
+ new_xl.opts["directory"] += ("/" + suffix)
+ return new_xl
+
+if len(sys.argv) < 3:
+ print >> sys.stderr, "Usage: %s orig_volfile tenant..." % sys.argv[0]
+ sys.exit(1)
+
+graph, last = volfilter.load(sys.argv[1])
+
+# Add the pre-authentication subvolume.
+junk_sv = last.subvols[0]
+while junk_sv.type != "storage/posix":
+ junk_sv = junk_sv.subvols[0]
+subvols = [copy_stack(junk_sv,"junk")]
+
+# Add the per-tenant subvolumes.
+for suffix in sys.argv[2:]:
+ subvols.append(copy_stack(last.subvols[0],suffix))
+
+# One cloud to bring them all...
+my_xl = volfilter.Translator(last.subvols[0].name)
+my_xl.type = "cluster/cloud"
+my_xl.subvols = subvols
+
+# ...and splice it in.
+last.subvols = [my_xl]
+
+volfilter.generate(graph,last)
diff --git a/scripts/filt-crypto.py b/scripts/filt-crypto.py
new file mode 100755
index 0000000..6a1f0a6
--- /dev/null
+++ b/scripts/filt-crypto.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010 Red Hat, Inc.
+#
+# This file is part of CloudFS.
+#
+# CloudFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License *
+# along with CloudFS. If not, see <
http://www.gnu.org/licenses/>.
+
+import sys
+import volfilter
+
+if len(sys.argv) != 3:
+ print >> sys.stderr, "Usage: %s orig_volfile key" % sys.argv[0]
+ sys.exit(1)
+
+graph, last = volfilter.load(sys.argv[1])
+# Quick-read bypasses the calls that crypto uses.
+to_do = [xl for xl in graph.itervalues() if xl.type ==
"performance/quick-read"]
+for td in to_do:
+ volfilter.delete(graph,td)
+to_do = [xl for xl in graph.itervalues() if xl.type == "protocol/client"]
+for td in to_do:
+ volfilter.push_filter(graph,td,"encryption/crypto",{"key":sys.argv[2]})
+volfilter.generate(graph,last)
diff --git a/scripts/filt-log-io.py b/scripts/filt-log-io.py
new file mode 100755
index 0000000..ac3ced8
--- /dev/null
+++ b/scripts/filt-log-io.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2010 Red Hat, Inc.
+#
+# This file is part of CloudFS.
+#
+# CloudFS is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License *
+# along with CloudFS. If not, see <
http://www.gnu.org/licenses/>.
+
+import sys
+import volfilter
+
+if len(sys.argv) != 2:
+ print >> sys.stderr, "Usage: %s orig_volfile" % sys.argv[0]
+ sys.exit(1)
+
+graph, last = volfilter.load(sys.argv[1])
+to_do = [xl for xl in graph.itervalues() if xl.type == "protocol/server"]
+for td in to_do:
+ for sv in td.subvols:
+ # TBD: deal with multiple volumes/logs per server
+ volfilter.push_filter(graph,sv,"debug/log-io")
+volfilter.generate(graph,last)
diff --git a/scripts/volfilter.py b/scripts/volfilter.py
new file mode 100755
index 0000000..3fd6250
--- /dev/null
+++ b/scripts/volfilter.py
@@ -0,0 +1,108 @@
+ #!/usr/bin/env python
+
+ # Copyright (c) 2010 Red Hat, Inc.
+ #
+ # This file is part of CloudFS.
+ #
+ # CloudFS is free software: you can redistribute it and/or modify it under the
+ # terms of the GNU Affero General Public License as published by the Free
+ # Software Foundation, either version 3 of the License, or (at your option)
+ # any later version.
+ #
+ # CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY
+ # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ # details.
+ #
+ # You should have received a copy of the GNU Affero General Public License *
+ # along with CloudFS. If not, see <
http://www.gnu.org/licenses/>.
+
+import string
+import sys
+
+class Translator:
+ def __init__ (self, name):
+ self.name = name
+ self.type = ""
+ self.opts = {}
+ self.subvols = []
+ self.dumped = False
+ def __repr__ (self):
+ return "<Translator %s>" % self.name
+
+def load (path):
+ fp = file(path,"r")
+ all_xlators = {}
+ xlator = None
+ last_xlator = None
+ while True:
+ text = fp.readline()
+ if text == "":
+ break
+ text = text.split()
+ if not len(text):
+ continue
+ if text[0] == "volume":
+ if xlator:
+ raise RuntimeError, "nested volume definition"
+ xlator = Translator(text[1])
+ continue
+ if not xlator:
+ raise RuntimeError, "text outside volume definition"
+ if text[0] == "type":
+ xlator.type = text[1]
+ continue
+ if text[0] == "option":
+ xlator.opts[text[1]] = string.join(text[2:])
+ continue
+ if text[0] == "subvolumes":
+ for sv in text[1:]:
+ xlator.subvols.append(all_xlators[sv])
+ continue
+ if text[0] == "end-volume":
+ all_xlators[xlator.name] = xlator
+ last_xlator = xlator
+ xlator = None
+ continue
+ raise RuntimeError, "unrecognized keyword %s" % text[0]
+ if xlator:
+ raise RuntimeError, "unclosed volume definition"
+ return all_xlators, last_xlator
+
+def generate (graph, last):
+ for sv in last.subvols:
+ if not sv.dumped:
+ generate(graph,sv)
+ print ""
+ sv.dumped = True
+ print "volume %s" % last.name
+ print " type %s" % last.type
+ for k, v in last.opts.iteritems():
+ print " option %s %s" % (k, v)
+ if last.subvols:
+ print " subvolumes %s" % string.join(
+ [ sv.name for sv in last.subvols ])
+ print "end-volume"
+
+def push_filter (graph, old_xl, filt_type, opts={}):
+ suffix = string.split(old_xl.type,"/")[1]
+ new_xl = Translator(old_xl.name+"-"+suffix)
+ new_xl.type = old_xl.type
+ new_xl.opts = old_xl.opts
+ new_xl.subvols = old_xl.subvols
+ graph[new_xl.name] = new_xl
+ old_xl.type = filt_type
+ old_xl.opts = opts
+ old_xl.subvols = [new_xl]
+
+def delete (graph, victim):
+ if len(victim.subvols) != 1:
+ raise RuntimeError, "attempt to delete non-unary translator"
+ for xl in graph.itervalues():
+ while xl.subvols.count(victim):
+ i = xl.subvols.index(victim)
+ xl.subvols[i] = victim.subvols[0]
+
+if __name__ == "__main__":
+ graph, last = load(sys.argv[1])
+ generate(graph,last)