ToDo | 4 -- scripts/cfs_add_directory.py | 18 +++++++++---- scripts/cfs_add_node.py | 7 ++++- scripts/cfs_add_tenant.py | 15 ++++++++--- scripts/cfs_add_volume.py | 14 ++++++---- scripts/cfs_delete_tenant.py | 15 ++++++++--- scripts/cfs_enable_tenant.py | 17 +++++++++--- scripts/cfs_list_tenants.py | 14 ++++++++++ scripts/cfs_list_vols.py | 16 +++++++++++ scripts/cfs_paths.py | 6 ++-- scripts/cfs_rm_volume.py | 55 +++++++++++++++++++++++++++++++++++++++++ scripts/cfs_start_volume.py | 30 +++++++++++++++++----- scripts/cfs_stop_volume.py | 14 +++++++++- scripts/cfs_utils.py | 9 +++++- scripts/cloudfsd.py | 9 ++++++ scripts/views/rm_vol_done.html | 15 +++++++++++ scripts/views/volumes.html | 3 +- 17 files changed, 223 insertions(+), 38 deletions(-)

New commits: commit 04d9caaab692c3ca819e1b3038099c81eee058a5 Author: Jeff Darcy jdarcy@redhat.com Date: Thu May 12 15:42:11 2011 -0400

Added CLI programs to list volumes/tenants.

diff --git a/ToDo b/ToDo index 696245b..2fbbcb9 100644 --- a/ToDo +++ b/ToDo @@ -1,5 +1,5 @@ = High Priority = -Add CLI equivalents to list nodes/volumes/tenants +(nothing left)

= Medium Priority = SSL diff --git a/scripts/cfs_list_tenants.py b/scripts/cfs_list_tenants.py new file mode 100755 index 0000000..3446418 --- /dev/null +++ b/scripts/cfs_list_tenants.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +import cfs_utils + +db_obj = cfs_utils.open_db() + +for tenant in [t for t in db_obj.keys() if t.startswith("tv_")]: + print "%s:" % t[3:] + vol_list = db_obj[tenant] + if vol_list == "": + print " (no volumes enabled)" + else: + for vol in vol_list.split(","): + print " %s enabled" % vol diff --git a/scripts/cfs_list_vols.py b/scripts/cfs_list_vols.py new file mode 100755 index 0000000..f26f3b8 --- /dev/null +++ b/scripts/cfs_list_vols.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import cfs_utils + +brick_list = cfs_utils.get_bricks() +vol_list = brick_list.keys()[:] +vol_list.sort() +db_obj = cfs_utils.open_db() + +for vol in vol_list: + if db_obj.has_key("vt_"+vol): + print "Volume %s (CloudFS):" % vol + else: + print "Volume %s (GlusterFS):" % vol + for brick in brick_list[vol]: + print " %s" % brick

commit 6a1ba723db3dee6496350f5a9e6f59741645ee6b Author: Jeff Darcy jdarcy@redhat.com Date: Thu May 12 15:26:18 2011 -0400

Fixed CLI versions of most functions.

diff --git a/ToDo b/ToDo index fec4353..696245b 100644 --- a/ToDo +++ b/ToDo @@ -1,5 +1,5 @@ = High Priority = -Make sure CLI equivalents work +Add CLI equivalents to list nodes/volumes/tenants

= Medium Priority = SSL diff --git a/scripts/cfs_add_directory.py b/scripts/cfs_add_directory.py old mode 100644 new mode 100755 index 508f501..1c9f0b9 --- a/scripts/cfs_add_directory.py +++ b/scripts/cfs_add_directory.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python

import os import re @@ -78,12 +79,16 @@ def add_local (path): xp_list = [] paths_by_node[k[3:]] = set(xp_list) # Add the user's paths. + members = cfs_utils.get_members() for p in expand(path): parts = string.split(p,":") if len(parts) != 2: - return "add_local(%s) rejected %s on %s" %( + return "add_local(%s) rejected %s on %s (no node)" %( path, p, socket.gethostname()) node, dir = parts + if node not in members: + return "add_local(%s) rejected %s on %s (bad node)" %( + path, p, socket.gethostname()) if paths_by_node.has_key(node): paths_by_node[node].add(dir) else: @@ -108,13 +113,16 @@ def run_common (path): for line in url_obj: scratch[1].append(line) blob.append(scratch) - return template("add_dir_done.html",path=path,blob=blob); + return blob

def run_www (): path = request.forms.get("path") - return run_common(path) + blob = run_common(path) + return template("add_dir_done.html",path=path,blob=blob);

if __name__ == "__main__": path = sys.argv[1] - #run_common(tn_name,tn_pw) - print add_local(path) + blob = run_common(path) + print "Directory %s added." % path + cfs_utils.print_blob(blob) + diff --git a/scripts/cfs_add_node.py b/scripts/cfs_add_node.py old mode 100644 new mode 100755 index 497db36..b9bb7c4 --- a/scripts/cfs_add_node.py +++ b/scripts/cfs_add_node.py @@ -50,4 +50,9 @@ def run_www (): return run_common(node_name)

if __name__ == "__main__": - run_common(sys.argv[1]) + if len(sys.argv) != 2: + print >> sys.stderr, "Usage: %s node_name_or_addr" % sys.argv[0] + sys.exit(1) + name = sys.argv[1] + run_common(name) + print "Node %s added." % name diff --git a/scripts/cfs_add_tenant.py b/scripts/cfs_add_tenant.py old mode 100644 new mode 100755 index d36837e..d460367 --- a/scripts/cfs_add_tenant.py +++ b/scripts/cfs_add_tenant.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python

import os import re @@ -34,15 +35,21 @@ def run_common (tn_name, tn_pw): for line in url_obj: scratch[1].append(line) blob.append(scratch) - return template("tn_act_done.html",name=tn_name,action="added", - blob=blob); + return blob

def run_www (): tn_name = request.forms.get("tn_name") tn_pw = request.forms.get("tn_pw") - return run_common(tn_name,tn_pw) + blob = run_common(tn_name,tn_pw) + return template("tn_act_done.html",name=tn_name,action="added", + blob=blob);

if __name__ == "__main__": + if len(sys.argv) != 3: + print >> sys.stderr, "Usage: %s name password" % sys.argv[0] + sys.exit(1) tn_name = sys.argv[1] tn_pw = sys.argv[2] - run_common(tn_name,tn_pw) + blob = run_common(tn_name,tn_pw) + print "Tenant %s added." % tn_name + cfs_utils.print_blob(blob) diff --git a/scripts/cfs_add_volume.py b/scripts/cfs_add_volume.py old mode 100644 new mode 100755 index 4af663b..32bdc60 --- a/scripts/cfs_add_volume.py +++ b/scripts/cfs_add_volume.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python

import os import re @@ -23,10 +24,10 @@ def run_common (vname, vtype, vcount, bricks): if vtype != "plain": cmd = "%s %s %s" % (cmd, vtype, vcount) cmd = "%s %s" % (cmd, string.join(bricks)) + print cmd sts = cfs_utils.run_cmd("gluster",cmd).wait() if sts: - return template("add_vol_fail.html", name=vname, - action="gluster", status=sts) + return [["gluster",["command failed with %d"%sts]]] blob = [] for node in cfs_utils.get_nodes_for_vol(vname): scratch = [node,[]] @@ -40,7 +41,7 @@ def run_common (vname, vtype, vcount, bricks): for line in url_obj: scratch[1].append(line) blob.append(scratch) - return template("add_vol_done.html",name=vname,blob=blob); + return blob

def run_www (): volume_id = request.forms.get("VOLUMEID") @@ -50,12 +51,15 @@ def run_www (): for prop in request.forms.iterkeys(): if prop.startswith("VOLUME_"): brick_list.append(prop[7:]) - return run_common(volume_id,volume_type,replica_or_stripe_count, + blob = run_common(volume_id,volume_type,replica_or_stripe_count, brick_list) + return template("add_vol_done.html",name=volume_id,blob=blob);

if __name__ == "__main__": volume_id = sys.argv[1] volume_type = sys.argv[2] replica_or_stripe_count = sys.argv[3] brick_list = sys.argv[4:] - run_common(volume_id,volume_type,replica_or_stripe_count,brick_list) + blob = run_common(volume_id,volume_type,replica_or_stripe_count,brick_list) + print "Volume %s added." % volume_id + cfs_utils.print_blob(blob) diff --git a/scripts/cfs_delete_tenant.py b/scripts/cfs_delete_tenant.py old mode 100644 new mode 100755 index ebbd35d..92c11d7 --- a/scripts/cfs_delete_tenant.py +++ b/scripts/cfs_delete_tenant.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python

import os import re @@ -31,12 +32,18 @@ def run_common (tn_name): for line in url_obj: scratch[1].append(line) blob.append(scratch) - return template("tn_act_done.html",name=tn_name,action="deleted", - blob=blob); + return blob

def run_www (tn_name): - return run_common(tn_name) + blob = run_common(tn_name) + return template("tn_act_done.html",name=tn_name,action="deleted", + blob=blob);

if __name__ == "__main__": + if len(sys.argv) != 2: + print >> sys.stderr, "Usage: %s name" % sys.argv[0] + sys.exit(1) tn_name = sys.argv[1] - run_common(tn_name) + blob = run_common(tn_name) + print "Tenant %s deleted." % tn_name + cfs_utils.print_blob(blob) diff --git a/scripts/cfs_enable_tenant.py b/scripts/cfs_enable_tenant.py old mode 100644 new mode 100755 index 3855546..146c785 --- a/scripts/cfs_enable_tenant.py +++ b/scripts/cfs_enable_tenant.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python

import os import re @@ -48,8 +49,7 @@ def run_common (tn_name, vol_list): for line in url_obj: scratch[1].append(line) blob.append(scratch) - return template("tn_act_done.html",name=tn_name,action="enabled", - blob=blob) + return blob

def run_www (tn_name): vol_list = [] @@ -57,8 +57,17 @@ def run_www (tn_name): print prop if prop.startswith("VOLUME_"): vol_list.append(prop[7:]) - return run_common(tn_name,vol_list) + blob = run_common(tn_name,vol_list) + return template("tn_act_done.html",name=tn_name,action="enabled", + blob=blob)

if __name__ == "__main__": - run_common(sys.argv[1],sys.argv[2:]) + if len(sys.argv) < 2: + print >> sys.stderr, "Usage: %s name [volume...]" % sys.argv[0] + sys.exit(1) + name = sys.argv[1] + vols = sys.argv[2:] + blob = run_common(name,vols) + print "Volumes %s enabled for %s." % (string.join(vols,","), name) + cfs_utils.print_blob(blob) diff --git a/scripts/cfs_paths.py b/scripts/cfs_paths.py index c497fe7..bf47ebd 100644 --- a/scripts/cfs_paths.py +++ b/scripts/cfs_paths.py @@ -8,6 +8,6 @@ log_dir = "/var/log/cloudfs" pid_dir = "/var/run/cloudfs" idle_subdir = os.path.join(pid_dir,".idle_ports") used_subdir = os.path.join(pid_dir,".used_ports") -volfile_re = re.compile("[^.]+.(.*).bricks-") +volfile_re = re.compile("(?P<vol>[^.]+).(?P<node>.+).(?P<path>[^.]+).vol") CLOUDFSD_PORT = 8080

diff --git a/scripts/cfs_rm_volume.py b/scripts/cfs_rm_volume.py old mode 100644 new mode 100755 index c871f65..bfab9b2 --- a/scripts/cfs_rm_volume.py +++ b/scripts/cfs_rm_volume.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python

import os import re @@ -17,7 +18,7 @@ def rm_local (vname): db_obj["vt_"+vname] = "" return "rm_local(%s) OK on %s" % (vname, socket.gethostname())

-def run_www (vname): +def run_common (vname): # TBD: all sorts of input-validity checking nodes_for_vol = cfs_utils.get_nodes_for_vol(vname) cmd = "volume delete %s" % vname @@ -37,9 +38,18 @@ def run_www (vname): for line in url_obj: scratch[1].append(line) blob.append(scratch) + return blob + +def run_www (vname): + blob = run_common(vname) return template("rm_vol_done.html",name=vname,blob=blob);

if __name__ == "__main__": + if len(sys.argv) != 2: + print >> sys.stderr, "Usage: %s volume_name" % sys.argv[0] + sys.exit(0) name = sys.argv[1] - run_www(name) + blob = run_common(name) + print "Volume %s removed." % name + cfs_utils.print_blob(blob)

diff --git a/scripts/cfs_start_volume.py b/scripts/cfs_start_volume.py old mode 100644 new mode 100755 index 031735a..11c992b --- a/scripts/cfs_start_volume.py +++ b/scripts/cfs_start_volume.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python

import glob import os @@ -54,7 +55,7 @@ def scan_gfs_volfiles(vol_name): for vf in glob.iglob(my_glob): m = cfs_paths.volfile_re.match(os.path.basename(vf)) if m: - this_host = m.groups(1)[0] + this_host = m.group("node") this_addr = socket.getaddrinfo(this_host, 0)[0][4][0] for addr in my_addrs: if this_addr == addr[4][0]: @@ -167,7 +168,7 @@ def start_local (vol_name): return "start_local(%s) returned %d on %s\n" % ( vol_name, retcode, socket.gethostname())

-def run_www (vol_name): +def run_common (vol_name): node_list = cfs_utils.get_nodes_for_vol(vol_name) blob = [] for node in node_list: @@ -181,7 +182,18 @@ def run_www (vol_name): for line in url_obj: scratch[1].append(line) blob.append(scratch) + return blob + +def run_www (vol_name): + blob = run_common(vol_name) return template("start_done.html",name=vol_name,blob=blob)

if __name__ == "__main__": - run_www(sys.argv[1]) + if len(sys.argv) != 2: + print >> sys.stderr, "Usage: %s volume_name" % sys.argv[0] + sys.exit(1) + name = sys.argv[1] + blob = run_common(name) + print "Volume %s started." % name + cfs_utils.print_blob(blob) + diff --git a/scripts/cfs_stop_volume.py b/scripts/cfs_stop_volume.py old mode 100644 new mode 100755 index 0851d6b..b93b438 --- a/scripts/cfs_stop_volume.py +++ b/scripts/cfs_stop_volume.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python

import fileinput import glob @@ -42,7 +43,7 @@ def stop_local (vol_name): vol_name, retcode, socket.gethostname())

-def run_www (vol_name): +def run_common (vol_name): node_list = cfs_utils.get_nodes_for_vol(vol_name) blob = [] for node in node_list: @@ -56,7 +57,18 @@ def run_www (vol_name): for line in url_obj: scratch[1].append(line) blob.append(scratch) + return blob + +def run_www (vol_name): + blob = run_common(vol_name) return template("stop_done.html",name=vol_name,blob=blob) if __name__ == "__main__": + if len(sys.argv) != 2: + print >> sys.stderr, "Usage: %s volume_name" % sys.argv[0] + sys.exit(1) + name = sys.argv[1] + blob = run_common(name) + print "Volume %s stopped." % name + cfs_utils.print_blob(blob) run_www(sys.argv[1]) diff --git a/scripts/cfs_utils.py b/scripts/cfs_utils.py index 9655b58..39a36e0 100644 --- a/scripts/cfs_utils.py +++ b/scripts/cfs_utils.py @@ -151,7 +151,7 @@ def get_nodes_for_vol (vol_name): for vf in glob.iglob(my_glob): m = cfs_paths.volfile_re.match(os.path.basename(vf)) if m: - node_list.add(m.groups(1)[0]) + node_list.add(m.group("node")) return node_list

# Open our configuration database. @@ -161,3 +161,10 @@ def get_nodes_for_vol (vol_name): def open_db (): db_path = os.path.join(cfs_paths.info_dir,"config.db") return dbm.open(db_path,"c",0600) + +# Print a "blob" of [node, [line1, line2]] tuples/lists for CLI/debugging. +def print_blob (blob): + for node, text in blob: + print "= %s =" % node + for line in text: + print line

commit dae24a26c655bdb57a38f95f72d7b92ccdb88b6d Author: Jeff Darcy jdarcy@redhat.com Date: Thu May 12 14:12:51 2011 -0400

Interface to remove volumes.

diff --git a/ToDo b/ToDo index 24fa991..fec4353 100644 --- a/ToDo +++ b/ToDo @@ -1,5 +1,4 @@ = High Priority = -Interface to remove volumes Make sure CLI equivalents work

= Medium Priority = diff --git a/scripts/cfs_rm_volume.py b/scripts/cfs_rm_volume.py new file mode 100644 index 0000000..c871f65 --- /dev/null +++ b/scripts/cfs_rm_volume.py @@ -0,0 +1,45 @@ + +import os +import re +import socket +import string +import sys +import urllib +import urllib2 + +from bottle import request, template +import cfs_paths +import cfs_utils + +def rm_local (vname): + db_obj = cfs_utils.open_db() + # Can't actually delete, but clearing is functionally equivalent. + db_obj["vt_"+vname] = "" + return "rm_local(%s) OK on %s" % (vname, socket.gethostname()) + +def run_www (vname): + # TBD: all sorts of input-validity checking + nodes_for_vol = cfs_utils.get_nodes_for_vol(vname) + cmd = "volume delete %s" % vname + kid = cfs_utils.run_cmd("gluster",cmd) + blob = [["gluster",kid.communicate("y\n")]] + sts = kid.wait() + if sts: + return template("rm_vol_done.html", name=vname, blob=blob) + for node in nodes_for_vol: + scratch = [node,[]] + if socket.gethostbyname(node) in cfs_utils.local_addrs: + url_obj = [rm_local(vname)] + else: + url = "http://%s:%d/volumes/%s/rm_local" % ( + node, cfs_paths.CLOUDFSD_PORT, vname) + url_obj = urllib2.urlopen(url) + for line in url_obj: + scratch[1].append(line) + blob.append(scratch) + return template("rm_vol_done.html",name=vname,blob=blob); + +if __name__ == "__main__": + name = sys.argv[1] + run_www(name) + diff --git a/scripts/cloudfsd.py b/scripts/cloudfsd.py index 2cc7259..8255a44 100755 --- a/scripts/cloudfsd.py +++ b/scripts/cloudfsd.py @@ -13,6 +13,7 @@ import cfs_utils import cfs_add_node import cfs_add_directory import cfs_add_volume +import cfs_rm_volume import cfs_start_volume import cfs_stop_volume import cfs_add_tenant @@ -68,6 +69,14 @@ def add_vol_local (): vname = request.forms.get("vname") return cfs_add_volume.add_local(vname)

+@route("volumes/:name/remove") +def rm_volume (name): + return cfs_rm_volume.run_www(name) + +@route("/volumes/:name/rm_local") +def rm_vol_local (name): + return cfs_rm_volume.rm_local(name) + @route("/volumes/:vol_name/start") def start_volume(vol_name): return cfs_start_volume.run_www(vol_name) diff --git a/scripts/views/rm_vol_done.html b/scripts/views/rm_vol_done.html new file mode 100644 index 0000000..cede10d --- /dev/null +++ b/scripts/views/rm_vol_done.html @@ -0,0 +1,15 @@ +<html><head> +<meta http-equiv="pragma" content="no-cache"> +</head><body> +%for node, output in blob: + <p><b>{{node}}</b></p> + <pre> + %for line in output: + {{line}} + %end + </pre> +%end +<p>Volume {{name}} deleted.</p> +<p><a href="/volumes">Back to volume configuration</a></p> +<p><a href="/cfgmain">Back to main menu</a></p> +</body></html> diff --git a/scripts/views/volumes.html b/scripts/views/volumes.html index 7595052..cc54f92 100644 --- a/scripts/views/volumes.html +++ b/scripts/views/volumes.html @@ -14,7 +14,8 @@ <p><b>{{vol_name}}</b> <a href="/volumes/{{vol_name}}/tenants">tenants</a> <a href="/volumes/{{vol_name}}/start">start</a> - <a href="/volumes/{{vol_name}}/stop">stop</a></p> + <a href="/volumes/{{vol_name}}/stop">stop</a> + <a href="/volumes/{{vol_name}}/remove">remove</a></p> <ul> %for brick in brick_list: <li>{{brick}}</li>

commit 81fc70f17bddfa72c77487351927aae382075a6c Author: Jeff Darcy jdarcy@redhat.com Date: Thu May 12 13:35:26 2011 -0400

Move log/pid/port directories.

diff --git a/ToDo b/ToDo index db4cf21..24fa991 100644 --- a/ToDo +++ b/ToDo @@ -1,5 +1,4 @@ = High Priority = -Create log/pid directories, move port links to /var/run Interface to remove volumes Make sure CLI equivalents work

diff --git a/scripts/cfs_paths.py b/scripts/cfs_paths.py index 27e11b0..c497fe7 100644 --- a/scripts/cfs_paths.py +++ b/scripts/cfs_paths.py @@ -4,10 +4,10 @@ import os

gfs_dir = "/var/lib/glusterd" info_dir = "/var/lib/cloudfs" -idle_subdir = os.path.join(info_dir,".idle_ports") -used_subdir = os.path.join(info_dir,".used_ports") log_dir = "/var/log/cloudfs" pid_dir = "/var/run/cloudfs" +idle_subdir = os.path.join(pid_dir,".idle_ports") +used_subdir = os.path.join(pid_dir,".used_ports") volfile_re = re.compile("[^.]+.(.*).bricks-") CLOUDFSD_PORT = 8080

diff --git a/scripts/cfs_start_volume.py b/scripts/cfs_start_volume.py index c0b8ce7..031735a 100644 --- a/scripts/cfs_start_volume.py +++ b/scripts/cfs_start_volume.py @@ -17,12 +17,18 @@ import cfs_utils # Make sure the volume directory exists and has the right stuff in it. def check_volume_directory(vol_name): if not os.path.exists(cfs_paths.info_dir): - os.mkdir(cfs_paths.info_dir) - os.mkdir(cfs_paths.idle_subdir) + os.mkdir(cfs_paths.info_dir,0700) + if not os.path.exists(cfs_paths.log_dir): + os.mkdir(cfs_paths.log_dir,0700) + if not os.path.exists(cfs_paths.pid_dir): + os.mkdir(cfs_paths.pid_dir,0700) + if not os.path.exists(cfs_paths.idle_subdir): + os.mkdir(cfs_paths.idle_subdir,0700) for i in range(24010, 24030): fp = open("%s/%d" % (cfs_paths.idle_subdir, i), "w") fp.close() - os.mkdir(cfs_paths.used_subdir) + if not os.path.exists(cfs_paths.used_subdir): + os.mkdir(cfs_paths.used_subdir,0700)

vol_dir = "%s/%s" % (cfs_paths.info_dir, vol_name) if not os.path.exists(vol_dir):