Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
commit 9b930ab244072cd03220f404aeac47304681137b Author: Matt Domsch Matt_Domsch@dell.com Date: Mon Jun 17 20:46:27 2013 -0500
fix run_rsync() in umdl and crawler
The crawler was failing to properly mark up-to-date or not when using rsync. It turns out that the rsync results from a fairly complete mirror can be nearly 100MB itself, which was sitting in memory (and could cause memory problems). So, let's write that to a safe temporary file.
Fixing this requires a corresponding change in umdl.
And as tempfile.SpooledTemporaryFile only appeared in python 2.6, and coindentally that's what RHEL6 has, fix up the spec to require python 2.6 or above.
Furthermore, the crawler was setting all_files outside a loop, instead of _inside_ the loop where it belonged. That's a bug. And it was later testing all_files inside a per-file loop, instead of after all those files had been processed - wasteful.
mirrormanager.spec.in | 9 +---- server/crawler_perhost | 55 ++++++++++++++++++---------------- server/mirrormanager/lib.py | 30 ++++++++++++------- server/update-master-directory-list | 4 ++- 4 files changed, 53 insertions(+), 45 deletions(-)
diff --git a/mirrormanager.spec.in b/mirrormanager.spec.in index 7433642..0faef59 100644 --- a/mirrormanager.spec.in +++ b/mirrormanager.spec.in @@ -9,17 +9,12 @@ License: MIT and GPLv2 URL: http://fedorahosted.org/mirrormanager Source0: https://fedorahosted.org/releases/m/i/%%7Bname%7D/%%7Bname%7D-%%7Bversion%7D... BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -BuildRequires: python +# for tempfile.SpooledTemporaryFile at least. RHEL6 has python 2.6 anyhow. +BuildRequires: python >= 2.6 Requires: TurboGears, python-IPy, python-GeoIP, wget, yum, psmisc, python-dns, python-fedora >= 0.3.14, python-fedora-turbogears, py-radix, mod_wsgi, python-TurboMail Requires: supervisor Requires(pre): shadow-utils
-%define py_ver %(echo `python -c "import sys; print sys.version[:3]"`) -%if "%{py_ver}" < "2.5" -# for md5, sha1, sha256, which are native to python 2.5 and higher -Requires: python-hashlib -%endif - %description MirrorManager tracks all the content provided on a master mirror server, and that of all public and private mirrors of that content. diff --git a/server/crawler_perhost b/server/crawler_perhost index 0a4a554..2e3e84c 100755 --- a/server/crawler_perhost +++ b/server/crawler_perhost @@ -19,6 +19,7 @@ import threading import urlgrabber import hashlib from smtplib import SMTP +import traceback
from turbogears.database import PackageHub hostid = None @@ -151,21 +152,6 @@ class TryLater(Exception): pass class ForbiddenExpected(Exception): pass
-def get_rsync_dir(url): - result, output = run_rsync(url, '--no-motd') - - if result is False: - return None - - listing = [] - - for line in output: - listing.append(line) - - output.close() - return listing - - def get_ftp_dir(hoststate, url, readable, i=0): if i > 1: raise TryLater() @@ -464,18 +450,26 @@ def try_percategory(trydirs, url, host_category_dirs, hc, host, categoryPrefixLe if not url.endswith('/'): url += '/'
- listing = get_rsync_dir(url) + result, listing = run_rsync(url, '--no-motd') + if not result: + return None
# put the rsync listing in a dict for easy access - for line in listing: + while True: + line = listing.readline() + if not line: break fields = line.split() - rsync[fields[4]] = {'mode': fields[0], 'size': fields[1], 'date': fields[2], 'time': fields[3]} + try: + rsync[fields[4]] = {'mode': fields[0], 'size': fields[1], 'date': fields[2], 'time': fields[3]} + except IndexError: + sys.stderr.write("invalid rsync line: %s\n" % line) + sys.stderr.flush()
prefix = "" - all_files = True
# for all directories in this category for d in trydirs: + all_files = True # the rsync listing is missing the category part of the url # remove if from the ones we are comparing it with name = d.name[categoryPrefixLen:] @@ -487,21 +481,30 @@ def try_percategory(trydirs, url, host_category_dirs, hc, host, categoryPrefixLe key = name + "/" + file try: if rsync[key]['size'] != d.files[file]['size']: + sys.stdout.write('rsync: file size mismatch %s %s != %s\n' % (file, d.files[file]['size'], rsync[key]['size'])) + sys.stdout.flush() all_files = False break - except: + except KeyError: # file is not in the rsync listing all_files = False break + except Exception, e: # something else went wrong + all_files = False + exception_msg = "Exception caught in try_percategory()\n" + exception_msg += traceback.format_exc(e) + sys.stderr.write(exception_msg) + sys.stderr.flush() + break
- if all_files is False: - host_category_dirs[(hc, d)] = False - else: - host_category_dirs[(hc, d)] = True - host_category_dirs = add_parents(host_category_dirs, hc, d) + if all_files is False: + host_category_dirs[(hc, d)] = False + else: + print d.name + host_category_dirs[(hc, d)] = True + host_category_dirs = add_parents(host_category_dirs, hc, d)
if len(host_category_dirs) > 0: sync_hcds(host, host_category_dirs) - print prefix + " updated" return True
mark_not_up2date(host, "No host category directories found. Check that your Host Category URLs are correct.") diff --git a/server/mirrormanager/lib.py b/server/mirrormanager/lib.py index 1f9db85..7776dde 100644 --- a/server/mirrormanager/lib.py +++ b/server/mirrormanager/lib.py @@ -1,5 +1,7 @@ -import types, os -from subprocess import Popen, PIPE +import os +import subprocess +import tempfile +import types
def createErrorString(tg_errors): """ @@ -89,19 +91,25 @@ def append_value_to_cache(cache, key, value): return cache
def run_rsync(rsyncpath, extra_rsync_args=None): - output = None + tmpfile = tempfile.SpooledTemporaryFile() result = True cmd = "rsync --temp-dir=/tmp -r --exclude=.snapshot --exclude='*.~tmp~'" if extra_rsync_args is not None: cmd += ' ' + extra_rsync_args cmd += ' ' + rsyncpath try: - devnull = open('/dev/null', 'r') - print cmd - p = Popen(cmd, shell=True, stdin=devnull, stdout=PIPE, bufsize=1, close_fds=True) - output = p.stdout - except: - output = "Unable to run " + args + devnull = open('/dev/null', 'rw') + sys.stderr.write('invoking %s\n' % cmd) + sys.stderr.flush() + p = subprocess.Popen(cmd, shell=True, stdin=devnull, + stdout=tmpfile, stderr=devnull, close_fds=True, bufsize=-1) + p.wait() + except Exception, e: + msg = "Exception invoking rsync:\n" + msg += traceback.format_exc(e) + sys.stderr.write(msg+'\n') + sys.stderr.flush() result = False - devnull.close() - return (result, output) + tmpfile.flush() + tmpfile.seek(0) + return (result, tmpfile) diff --git a/server/update-master-directory-list b/server/update-master-directory-list index 4057532..d380ffc 100755 --- a/server/update-master-directory-list +++ b/server/update-master-directory-list @@ -419,7 +419,9 @@ def sync_category_directories(category, category_directories, directory_info): def parse_rsync_listing(category, f): topdir_name = category.topdir.name category_directories = {} - for line in f: + while True: + line = f.readline() + if not line: break line.strip() l = line.split() if line.startswith('d') and len(l) == 5 and len(l[0]) == 10: # good guess it's a directory line
mirrormanager-commits@lists.fedorahosted.org