README.md | 19 ++++++++ scripts/filt-cloud.py | 61 +++++++++++++++++++++++++++ scripts/filt-crypto.py | 35 +++++++++++++++ scripts/filt-log-io.py | 33 ++++++++++++++ scripts/volfilter.py | 108 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 256 insertions(+)

New commits: commit 27c56982c5247983c22e00a596f2e48f6384ef4c Author: Jeff Darcy jdarcy@redhat.com Date: Wed Dec 22 15:22:56 2010 -0500

Add volfile-manipulation scripts.

diff --git a/README.md b/README.md index 105d5c8..6af15bd 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,25 @@ a change globally, you'll need to do the following: See the CONFIG.txt in each translator's directory for instructions specific to that translator.

+The .../scripts directory contains some Python scripts that can help automate +the process of modifying volfiles. Specifically: + +* filt-log-io.py: inserts a debug/log-io translator between a protocol/server + volume and each of its subvolumes + +* filt-crypto.py: inserts an encryption/crypto translator on top of each + protocol/client volume. Also disables performance/quick-read, which is + incompatible with encryption/crypto. + +* filt-cloud.py: replaces a simple translator "stack" (from storage/posix up + to whatever is below protocol/server) with one such stack per named tenant, + plus a cluster/cloud translator to tie them together. + +Running these scripts after each gluster "volume create" or "volume set" +command should generate a new volfile with the desired enhancements. They use +a common volfile parsing/modification library that might be useful for other +tasks as well. + Work List ---------

diff --git a/scripts/filt-cloud.py b/scripts/filt-cloud.py new file mode 100755 index 0000000..a055556 --- /dev/null +++ b/scripts/filt-cloud.py @@ -0,0 +1,61 @@ +#!/usr/bin/python + +# Copyright (c) 2010 Red Hat, Inc. +# +# This file is part of CloudFS. +# +# CloudFS is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License * +# along with CloudFS. If not, see http://www.gnu.org/licenses/. + +import copy +import sys +import volfilter + +def copy_stack (old_xl,suffix): + new_name = old_xl.name + "-" + suffix + new_xl = volfilter.Translator(new_name) + new_xl.type = old_xl.type + # The results with normal assignment here are . . . amusing. + new_xl.opts = copy.deepcopy(old_xl.opts) + for sv in old_xl.subvols: + new_xl.subvols.append(copy_stack(sv,suffix)) + # Patch up the path at the bottom. + if new_xl.type == "storage/posix": + new_xl.opts["directory"] += ("/" + suffix) + return new_xl + +if len(sys.argv) < 3: + print >> sys.stderr, "Usage: %s orig_volfile tenant..." % sys.argv[0] + sys.exit(1) + +graph, last = volfilter.load(sys.argv[1]) + +# Add the pre-authentication subvolume. +junk_sv = last.subvols[0] +while junk_sv.type != "storage/posix": + junk_sv = junk_sv.subvols[0] +subvols = [copy_stack(junk_sv,"junk")] + +# Add the per-tenant subvolumes. +for suffix in sys.argv[2:]: + subvols.append(copy_stack(last.subvols[0],suffix)) + +# One cloud to bring them all... +my_xl = volfilter.Translator(last.subvols[0].name) +my_xl.type = "cluster/cloud" +my_xl.subvols = subvols + +# ...and splice it in. +last.subvols = [my_xl] + +volfilter.generate(graph,last) diff --git a/scripts/filt-crypto.py b/scripts/filt-crypto.py new file mode 100755 index 0000000..6a1f0a6 --- /dev/null +++ b/scripts/filt-crypto.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# Copyright (c) 2010 Red Hat, Inc. +# +# This file is part of CloudFS. +# +# CloudFS is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License * +# along with CloudFS. If not, see http://www.gnu.org/licenses/. + +import sys +import volfilter + +if len(sys.argv) != 3: + print >> sys.stderr, "Usage: %s orig_volfile key" % sys.argv[0] + sys.exit(1) + +graph, last = volfilter.load(sys.argv[1]) +# Quick-read bypasses the calls that crypto uses. +to_do = [xl for xl in graph.itervalues() if xl.type == "performance/quick-read"] +for td in to_do: + volfilter.delete(graph,td) +to_do = [xl for xl in graph.itervalues() if xl.type == "protocol/client"] +for td in to_do: + volfilter.push_filter(graph,td,"encryption/crypto",{"key":sys.argv[2]}) +volfilter.generate(graph,last) diff --git a/scripts/filt-log-io.py b/scripts/filt-log-io.py new file mode 100755 index 0000000..ac3ced8 --- /dev/null +++ b/scripts/filt-log-io.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +# Copyright (c) 2010 Red Hat, Inc. +# +# This file is part of CloudFS. +# +# CloudFS is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License * +# along with CloudFS. If not, see http://www.gnu.org/licenses/. + +import sys +import volfilter + +if len(sys.argv) != 2: + print >> sys.stderr, "Usage: %s orig_volfile" % sys.argv[0] + sys.exit(1) + +graph, last = volfilter.load(sys.argv[1]) +to_do = [xl for xl in graph.itervalues() if xl.type == "protocol/server"] +for td in to_do: + for sv in td.subvols: + # TBD: deal with multiple volumes/logs per server + volfilter.push_filter(graph,sv,"debug/log-io") +volfilter.generate(graph,last) diff --git a/scripts/volfilter.py b/scripts/volfilter.py new file mode 100755 index 0000000..3fd6250 --- /dev/null +++ b/scripts/volfilter.py @@ -0,0 +1,108 @@ + #!/usr/bin/env python + + # Copyright (c) 2010 Red Hat, Inc. + # + # This file is part of CloudFS. + # + # CloudFS is free software: you can redistribute it and/or modify it under the + # terms of the GNU Affero General Public License as published by the Free + # Software Foundation, either version 3 of the License, or (at your option) + # any later version. + # + # CloudFS is distributed in the hope that it will be useful, but WITHOUT ANY + # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + # details. + # + # You should have received a copy of the GNU Affero General Public License * + # along with CloudFS. If not, see http://www.gnu.org/licenses/. + +import string +import sys + +class Translator: + def __init__ (self, name): + self.name = name + self.type = "" + self.opts = {} + self.subvols = [] + self.dumped = False + def __repr__ (self): + return "<Translator %s>" % self.name + +def load (path): + fp = file(path,"r") + all_xlators = {} + xlator = None + last_xlator = None + while True: + text = fp.readline() + if text == "": + break + text = text.split() + if not len(text): + continue + if text[0] == "volume": + if xlator: + raise RuntimeError, "nested volume definition" + xlator = Translator(text[1]) + continue + if not xlator: + raise RuntimeError, "text outside volume definition" + if text[0] == "type": + xlator.type = text[1] + continue + if text[0] == "option": + xlator.opts[text[1]] = string.join(text[2:]) + continue + if text[0] == "subvolumes": + for sv in text[1:]: + xlator.subvols.append(all_xlators[sv]) + continue + if text[0] == "end-volume": + all_xlators[xlator.name] = xlator + last_xlator = xlator + xlator = None + continue + raise RuntimeError, "unrecognized keyword %s" % text[0] + if xlator: + raise RuntimeError, "unclosed volume definition" + return all_xlators, last_xlator + +def generate (graph, last): + for sv in last.subvols: + if not sv.dumped: + generate(graph,sv) + print "" + sv.dumped = True + print "volume %s" % last.name + print " type %s" % last.type + for k, v in last.opts.iteritems(): + print " option %s %s" % (k, v) + if last.subvols: + print " subvolumes %s" % string.join( + [ sv.name for sv in last.subvols ]) + print "end-volume" + +def push_filter (graph, old_xl, filt_type, opts={}): + suffix = string.split(old_xl.type,"/")[1] + new_xl = Translator(old_xl.name+"-"+suffix) + new_xl.type = old_xl.type + new_xl.opts = old_xl.opts + new_xl.subvols = old_xl.subvols + graph[new_xl.name] = new_xl + old_xl.type = filt_type + old_xl.opts = opts + old_xl.subvols = [new_xl] + +def delete (graph, victim): + if len(victim.subvols) != 1: + raise RuntimeError, "attempt to delete non-unary translator" + for xl in graph.itervalues(): + while xl.subvols.count(victim): + i = xl.subvols.index(victim) + xl.subvols[i] = victim.subvols[0] + +if __name__ == "__main__": + graph, last = load(sys.argv[1]) + generate(graph,last)