server/update-master-directory-list | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-)
New commits: commit 6733901175ba722737c985366bc8979e22d0a5cd Author: Matt Domsch Matt_Domsch@dell.com Date: Fri Apr 29 10:19:19 2011 -0500
umdl: lookup category early, pass it down
diff --git a/server/update-master-directory-list b/server/update-master-directory-list index 8bada71..a25a99d 100755 --- a/server/update-master-directory-list +++ b/server/update-master-directory-list @@ -368,8 +368,7 @@ def sync_category_directories(category, category_directories): make_repomd_file_details(dir) ageFileDetails()
-def parse_rsync_listing(cname, f): - category = Category.byName(cname) +def parse_rsync_listing(category, f): category_directories = {} for line in f: line.strip() @@ -387,7 +386,7 @@ def parse_rsync_listing(cname, f):
-def sync_directories_using_rsync(rsyncpath, cname, extra_rsync_args=None): +def sync_directories_using_rsync(rsyncpath, category, extra_rsync_args=None): cmd = "rsync --temp-dir=/tmp -r --exclude=.snapshot --exclude='*.~tmp~'" if extra_rsync_args is not None: cmd += ' ' + extra_rsync_args @@ -396,23 +395,22 @@ def sync_directories_using_rsync(rsyncpath, cname, extra_rsync_args=None): devnull = open('/dev/null', 'r') p = Popen(cmd, shell=True, stdin=devnull, stdout=PIPE, bufsize=1, close_fds=True) except: - print "Category %s: Unable to run %s" % (cname, args) + print "Category %s: Unable to run %s" % (category.name, args) devnull.close() return - parse_rsync_listing(cname, p.stdout) + parse_rsync_listing(category, p.stdout) devnull.close()
-def sync_directories_from_file(filename, cname): +def sync_directories_from_file(filename, category): f = open(filename, 'r') - parse_rsync_listing(cname, f) + parse_rsync_listing(category, f) f.close()
-def sync_directories_from_directory(directory, cname, excludes=[]): +def sync_directories_from_directory(directory, category, excludes=[]): global unreadable_dirs # drop trailing slashes from path directory = directory.rstrip('/') - category = Category.byName(cname) category_directories = {} topdir = category.topdir topdirName = topdir.name @@ -510,14 +508,14 @@ def main(): continue
if i['type'] == 'rsync': - sync_directories_using_rsync(i['url'], cname, options) + sync_directories_using_rsync(i['url'], category, options)
if i['type'] == 'file': - sync_directories_from_file(i['url'], cname) + sync_directories_from_file(i['url'], category)
if i['type'] == 'directory': excludes = i.get('excludes', []) - sync_directories_from_directory(i['path'], cname, excludes) + sync_directories_from_directory(i['path'], category, excludes)
remove_pidfile(pidfile) print "Ending umdl %s" % (datetime.datetime.utcnow().isoformat())
commit 24b5c76e5b3c92ca2a62d61fae34749e0cb0c1f9 Author: Matt Domsch Matt_Domsch@dell.com Date: Fri Apr 29 10:15:26 2011 -0500
catch errors in umdl.master_directories earlier, avoids later failures
diff --git a/server/update-master-directory-list b/server/update-master-directory-list index a7e7f9e..8bada71 100755 --- a/server/update-master-directory-list +++ b/server/update-master-directory-list @@ -498,15 +498,26 @@ def main(): except KeyError: options = None
+ cname = i['category'] + try: + category = Category.byName(cname) + except SQLObjectNotFound: + print 'Error: umdl.master_directories Category %s does not exist in the database, skipping' % (cname) + continue + + if category.product is None: + print 'Error: umdl.master_directories Category %s has null Product, skipping' % (cname) + continue + if i['type'] == 'rsync': - sync_directories_using_rsync(i['url'], i['category'], options) + sync_directories_using_rsync(i['url'], cname, options)
if i['type'] == 'file': - sync_directories_from_file(i['url'], i['category']) + sync_directories_from_file(i['url'], cname)
if i['type'] == 'directory': excludes = i.get('excludes', []) - sync_directories_from_directory(i['path'], i['category'], excludes) + sync_directories_from_directory(i['path'], cname, excludes)
remove_pidfile(pidfile) print "Ending umdl %s" % (datetime.datetime.utcnow().isoformat())