[mirrormanager] master: crawler_perhost: ignore SIGHUP more cleanly (f763d73)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit f763d7395807370d66ff8cf1447a61e50d01409d
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 12 13:04:07 2013 -0500
crawler_perhost: ignore SIGHUP more cleanly
>---------------------------------------------------------------
server/crawler_perhost | 5 +----
1 files changed, 1 insertions(+), 4 deletions(-)
diff --git a/server/crawler_perhost b/server/crawler_perhost
index cfabc13..e2261b8 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -613,9 +613,6 @@ def sigusr1_handler(signum, frame):
pass
os._exit(1)
-def sighup_handler(signum, frame):
- pass
-
def select_host_categories_to_scan(options):
result = []
host = Host.get(options.hostid)
@@ -656,7 +653,7 @@ def per_host(host, options):
timer.start()
signal.signal(signal.SIGUSR1, sigusr1_handler)
- signal.signal(signal.SIGHUP, sighup_handler)
+ signal.signal(signal.SIGHUP, signal.SIG_IGN)
hoststate = hostState(http_debuglevel=http_debuglevel, ftp_debuglevel=ftp_debuglevel)
10 years, 10 months
[mirrormanager] master: crawler_perhost: use logging module (5a56d5e)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 5a56d5e301b441ae19645409263981a46ebdd433
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 12 13:01:00 2013 -0500
crawler_perhost: use logging module
instead of logging to stdout manually, which gets tripped up by the
logrotate daemon sometimes, use common logging library code instead of
doing it all ourselves.
>---------------------------------------------------------------
server/crawler_perhost | 151 +++++++++++++++++-------------------------------
1 files changed, 53 insertions(+), 98 deletions(-)
diff --git a/server/crawler_perhost b/server/crawler_perhost
index 28e91da..cfabc13 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -8,6 +8,7 @@ import ftplib
from ftplib import FTP
import hashlib
import httplib
+import logging
from optparse import OptionParser
import os
import sys
@@ -28,7 +29,7 @@ from mirrormanager.lib import run_rsync
from turbogears.database import PackageHub
hostid = None
-logfile = sys.stdout
+logger = sys.stdout
################################################
# overrides for httplib because we're
@@ -119,7 +120,7 @@ class hostState:
def check_ftp_dir_callback(self, line):
if self.ftp_debuglevel > 0:
- print >>logfile, line
+ logger.info(line)
self.ftp_dir_results.append(line)
def ftp_dir(self, url):
@@ -180,7 +181,7 @@ def get_ftp_dir(hoststate, url, readable, i=0):
if str(e).startswith('500'): # Oops
raise TryLater()
else:
- print >>logfile, "unknown permanent error %s on %s" % (e, url)
+ logger.error("unknown permanent error %s on %s" % (e, url))
raise
except ftplib.error_temp, e:
# Returned by Boston University when directory does not exist
@@ -188,13 +189,13 @@ def get_ftp_dir(hoststate, url, readable, i=0):
return []
# Returned by Princeton University when cannot log in due to connection restrictions
if str(e).startswith('421'):
- print >>logfile, "Connections Exceeded %s" % url
+ logger.info("Connections Exceeded %s" % url)
raise TryLater()
if str(e).startswith('425'):
- print >>logfile, "Failed to establish connection on %s" % url
+ logger.info("Failed to establish connection on %s" % url)
raise TryLater()
else:
- print >>logfile, "unknown error %s on %s" % (e, url)
+ logger.error("unknown error %s on %s" % (e, url))
raise
except (EOFError, socket.error):
hoststate.close_ftp(url)
@@ -302,7 +303,7 @@ def check_head(hoststate, url, filedata, recursion, readable, retry=0):
elif status >= 500:
raise HTTP500()
- print >>logfile, "status = %s" % status
+ logger.info("status = %s" % status)
raise HTTPUnknown()
def sync_hcds(host, host_category_dirs):
@@ -334,11 +335,10 @@ def sync_hcds(host, host_category_dirs):
hcd.up2date=up2date
hcd.sync()
if up2date == False:
- print >>logfile, "Directory %s is not up-to-date on this host." % d.name
+ logger.info("Directory %s is not up-to-date on this host." % d.name)
else:
- print >>logfile, d.name
+ logger.info(d.name)
- logfile.flush()
current_hcds[hcd] = True
@@ -417,7 +417,6 @@ def try_perfile(d, hoststate, url):
graburl = "%s/%s" % (url, file)
try:
exists = check_url(hoststate, graburl, d.files[file], 0, d.readable)
- #print >>logfile, "%s %s" % (exists, graburl)
if exists == False:
return False
except TryLater:
@@ -459,8 +458,7 @@ def try_percategory(trydirs, url, host_category_dirs, hc, host, categoryPrefixLe
result, listing = run_rsync(url, '--no-motd')
if result > 0:
- print >>logfile, 'rsync returned exit code %d' % result
- logfile.flush()
+ logger.info('rsync returned exit code %d' % result)
# put the rsync listing in a dict for easy access
while True:
@@ -470,9 +468,7 @@ def try_percategory(trydirs, url, host_category_dirs, hc, host, categoryPrefixLe
try:
rsync[fields[4]] = {'mode': fields[0], 'size': fields[1], 'date': fields[2], 'time': fields[3]}
except IndexError:
- if options.debug_rsync:
- print >>logfile, "invalid rsync line: %s\n" % line
- logfile.flush()
+ logger.debug("invalid rsync line: %s\n" % line)
# for all directories in this category
for d in trydirs:
@@ -490,25 +486,18 @@ def try_percategory(trydirs, url, host_category_dirs, hc, host, categoryPrefixLe
else:
key = os.path.join(name, filename)
try:
- if options.debug_rsync:
- print >>logfile, 'trying with key %s' % key
- logfile.flush()
+ logger.debug('trying with key %s' % key)
if rsync[key]['size'] != d.files[filename]['size'] and not rsync[key]['mode'].startswith('l'): # ignore symlink size differences
- if options.debug_rsync:
- print >>logfile, 'rsync: file size mismatch %s %s != %s\n' % (filename, d.files[filename]['size'], rsync[key]['size'])
- logfile.flush()
+ logger.debug('rsync: file size mismatch %s %s != %s\n' % (filename, d.files[filename]['size'], rsync[key]['size']))
all_files = False
break
except KeyError, e: # file is not in the rsync listing
msg = 'Missing remote file %s\n' % key
- print >>logfile, msg
- logfile.flush()
+ logger.debug(msg)
break
except Exception, e: # something else went wrong
exception_msg = "Exception caught in try_percategory()\n"
- exception_msg += traceback.format_exc(e)
- print >>logfile, exception_msg
- logfile.flush()
+ logger.exception(exception_msg)
all_files = False
break
@@ -537,7 +526,6 @@ def try_perdir(d, hoststate, url):
return None
if len(listing) == 0:
- #print >>logfile, 'FALSE %s' % url
return False
for line in listing:
@@ -562,8 +550,7 @@ mypid = None
def dienow():
global must_dienow
must_dienow = True
- print >>logfile, "Host %d timeout expired, killing this process. Better luck next time." % hostid
- logfile.flush()
+ logger.error("Host %d timeout expired, killing this process. Better luck next time." % hostid)
os.kill(mypid, signal.SIGUSR1)
os._exit(0)
@@ -598,10 +585,9 @@ Date: %s
config.get('crawler.admin_mail_to'),
msg)
except Exception, e:
- print "Error sending email"
- print e
- print "Email message follows:"
- print msg
+ logger.exception("Error sending email")
+ logger.debug("Email message follows:")
+ logger.debug(msg)
try:
smtp.quit()
@@ -612,11 +598,9 @@ def mark_not_up2date(exc, host, reason="Unknown", mail=True):
host.set_not_up2date()
host.lastCrawled = datetime.utcnow()
msg = "Host marked not up2date: %s" % reason
- print >> logfile, msg
- logfile.flush()
+ logger.warning(msg)
if exc is not None:
- print >> logfile, exc[0], exc[1], exc[2]
- logfile.flush()
+ logger.debug("%s %s %s" % (exc[0], exc[1], exc[2]))
if mail:
send_email(host, msg, exc)
@@ -627,25 +611,10 @@ def sigusr1_handler(signum, frame):
mark_not_up2date(None, Host.get(hostid), "Crawler timed out before completing. Host is likely overloaded.")
except:
pass
- logfile.flush()
os._exit(1)
-reopenLogs = False
-def reopen_logs():
- global logfile
- if logfile is not None and logfile != sys.stdout:
- fname = logfile.name
- logfile.flush()
- # it's a logfile, don't care if it's fsynced to disk or not...
- logfile.close()
- logfile = open(fname, 'a+b')
-
def sighup_handler(signum, frame):
- if signum == signal.SIGHUP:
- global reopenLogs
- reopenLogs = True
- signal.signal(signal.SIGHUP, sighup_handler)
-
+ pass
def select_host_categories_to_scan(options):
result = []
@@ -657,8 +626,7 @@ def select_host_categories_to_scan(options):
hc = HostCategory.selectBy(host=host, category=category)[0]
result.append(hc)
except SQLObjectNotFound:
- print >>logfile, "Category '%s' specified on the command line is not found on this host, ignoring.\n" % c
- logfile.flush()
+ logger.warning("Category '%s' specified on the command line is not found on this host, ignoring.\n" % c)
continue
else:
@@ -679,9 +647,8 @@ def per_host(host, options):
return
http_debuglevel = 0
ftp_debuglevel = 0
- if options.debug_http:
+ if options.debug:
http_debuglevel = 2
- if options.debug_ftp:
ftp_debuglevel = 2
if options.timeout_minutes > 0:
@@ -719,13 +686,7 @@ def per_host(host, options):
for d in trydirs:
if must_dienow:
- logfile.flush()
- sys.exit(1)
-
- global reopenLogs
- if reopenLogs:
- reopenLogs = False
- reopen_logs()
+ return 1
if not d.readable:
continue
@@ -744,30 +705,27 @@ def per_host(host, options):
if has_all_files == False:
- print >>logfile, "Not up2date: %s" % (d.name)
+ logger.warning("Not up2date: %s" % (d.name))
host_category_dirs[(hc, d)] = False
elif has_all_files == True:
host_category_dirs[(hc, d)] = True
- print >>logfile, url
+ logger.info(url)
# make sure our parent dirs appear on the list too
host_category_dirs = add_parents(host_category_dirs, hc, d)
else:
# could be a dir with no files, or an unreadable dir.
# defer decision on this dir, let a child decide.
pass
- logfile.flush()
except TryLater:
timer.cancel()
msg = "Server load exceeded - try later"
- print >> logfile, msg
- logfile.flush()
- sys.exit(1)
+ logger.error(msg)
+ return 1
except:
timer.cancel()
mark_not_up2date(sys.exc_info(), host, "Unknown exception raised. This is a bug in the MM crawler.")
- logfile.flush()
- sys.exit(1)
+ return 1
hoststate.close()
if options.timeout_minutes > 0:
@@ -790,18 +748,6 @@ def main():
dest="hostid", type='int', default=None,
help="Crawl a single host at site")
- parser.add_option("--debug-http",
- action="store_true", dest="debug_http", default=False,
- help="Dump HTTP headers for each transaction")
-
- parser.add_option("--debug-ftp",
- action="store_true", dest="debug_ftp", default=False,
- help="Dump FTP headers for each transaction")
-
- parser.add_option("--debug-rsync",
- action="store_true", dest="debug_rsync", default=False,
- help="Be verbose about actions taken while using rsync")
-
parser.add_option("--include-private",
action="store_true", dest="include_private", default=False,
help="Include hosts marked 'private' in the crawl")
@@ -818,6 +764,9 @@ def main():
parser.add_option("--category",
dest="categories", action="append", default=[],
help="Category to scan (default=all), can be repeated")
+ parser.add_option("--debug-http", "--debug-ftp", "--debug-rsync",
+ dest="debug", action="store_true", default=False,
+ help="Enable debug logging")
(options, args) = parser.parse_args()
@@ -826,34 +775,40 @@ def main():
global hub
global __connection__
global hostid
- global logfile
+ global logger
hub = PackageHub("mirrormanager")
__connection__ = hub
os.chdir('/tmp')
+ global logger
if options.logfile is not None:
- logfile = open(options.logfile, "a+b")
+ fmt ='%(asctime)s %(message)s'
+ datefmt ='%m/%d/%Y %I:%M:%S %p'
+ formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
+ logger = logging.getLogger('crawler_perhost')
+ handler = logging.handlers.WatchedFileHandler(options.logfile, "a+b")
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+ if options.debug:
+ logger.setLevel(logging.DEBUG)
+ # fixme - http and FTP debugging is still getting printed to stdout
+ # from the httplib module
try:
host = Host.get(options.hostid)
except:
- print >>logfile, "Host %s not found." % options.hostid
- logfile.flush()
- sys.exit(1)
+ logger.error("Host %s not found." % options.hostid)
+ return 1
hostid = host.id
- print >>logfile, "Starting crawl %s" % (datetime.utcnow().isoformat())
- logfile.flush()
-
- per_host(host.id, options)
+ logger.info("Starting crawl")
- print >>logfile, "Ending crawl %s" % (datetime.utcnow().isoformat())
- logfile.flush()
- if logfile != sys.stdout: logfile.close()
+ rc = per_host(host.id, options)
+ logger.info("Ending crawl")
+ return rc
if __name__ == "__main__":
rc = main()
- logfile.flush()
sys.exit(rc)
10 years, 10 months
[mirrormanager] master: bump crawler timeout to 120 minutes; 90m isn't enough anymore (ce1ab60)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit ce1ab608a2a99859bb9866a87488dfd1e137e329
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 12 00:22:06 2013 -0500
bump crawler timeout to 120 minutes; 90m isn't enough anymore
>---------------------------------------------------------------
server/crawler_perhost | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/server/crawler_perhost b/server/crawler_perhost
index f5beb6b..28e91da 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -807,8 +807,8 @@ def main():
help="Include hosts marked 'private' in the crawl")
parser.add_option("--timeout-minutes", type="int",
- dest="timeout_minutes", default=90,
- help="Minutes to let the crawler run before killed (default=90)")
+ dest="timeout_minutes", default=120,
+ help="Minutes to let the crawler run before killed (default=120)")
parser.add_option("--logfile", type="string", metavar="FILE",
dest="logfile", default=None,
help="write logs to FILE")
10 years, 10 months
[mirrormanager] master: crawler_perhost: add more flush() calls (282a999)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 282a9995b375b2c3acd7f4c447a5e5653a5693a8
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 12 00:06:48 2013 -0500
crawler_perhost: add more flush() calls
>---------------------------------------------------------------
server/crawler_perhost | 4 ++++
1 files changed, 4 insertions(+), 0 deletions(-)
diff --git a/server/crawler_perhost b/server/crawler_perhost
index 58cb531..f5beb6b 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -338,6 +338,7 @@ def sync_hcds(host, host_category_dirs):
else:
print >>logfile, d.name
+ logfile.flush()
current_hcds[hcd] = True
@@ -612,8 +613,10 @@ def mark_not_up2date(exc, host, reason="Unknown", mail=True):
host.lastCrawled = datetime.utcnow()
msg = "Host marked not up2date: %s" % reason
print >> logfile, msg
+ logfile.flush()
if exc is not None:
print >> logfile, exc[0], exc[1], exc[2]
+ logfile.flush()
if mail:
send_email(host, msg, exc)
@@ -752,6 +755,7 @@ def per_host(host, options):
# could be a dir with no files, or an unreadable dir.
# defer decision on this dir, let a child decide.
pass
+ logfile.flush()
except TryLater:
timer.cancel()
msg = "Server load exceeded - try later"
10 years, 10 months
[mirrormanager] master: crawler: import datetime as dt to avoid name conflict (0fb84b6)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 0fb84b6cac0056a10dac71142836ad4baffc64b6
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Jul 11 23:11:21 2013 -0500
crawler: import datetime as dt to avoid name conflict
>---------------------------------------------------------------
server/crawler | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/server/crawler b/server/crawler
index 50e2a5c..fc100ff 100755
--- a/server/crawler
+++ b/server/crawler
@@ -3,7 +3,7 @@ __requires__ = 'TurboGears[future]'
import pkg_resources
pkg_resources.require("TurboGears")
-import datetime
+import datetime as dt
from optparse import OptionParser
import os
from subprocess import Popen
@@ -62,11 +62,11 @@ class ForkingMaster:
def start_time(self, pid, hostid):
self.timings[pid] = dict()
- self.timings[pid]['start'] = datetime.datetime.utcnow()
+ self.timings[pid]['start'] = dt.datetime.utcnow()
self.timings[pid]['hostid'] = hostid
def stop_time(self, pid):
- self.timings[pid]['stop'] = datetime.datetime.utcnow()
+ self.timings[pid]['stop'] = dt.datetime.utcnow()
diff = self.timings[pid]['stop'] - self.timings[pid]['start']
host = Host.get(self.timings[pid]['hostid'])
10 years, 10 months
[mirrormanager] master: crawler: report run times, cleanup imports (f3a9bc0)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit f3a9bc05de14bdf2d0280df8be335c8aead1caf7
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Jul 11 22:43:55 2013 -0500
crawler: report run times, cleanup imports
>---------------------------------------------------------------
server/crawler | 31 ++++++++++++++++++++++++++-----
1 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/server/crawler b/server/crawler
index 3cb3753..50e2a5c 100755
--- a/server/crawler
+++ b/server/crawler
@@ -3,13 +3,18 @@ __requires__ = 'TurboGears[future]'
import pkg_resources
pkg_resources.require("TurboGears")
+import datetime
+from optparse import OptionParser
+import os
+from subprocess import Popen
+import sys
+
from sqlobject import *
-import sys, os
import turbogears
+
+sys.path.append('/usr/share/mirrormanager/server')
from mirrormanager.model import *
from mirrormanager.lib import manage_pidfile, remove_pidfile
-from optparse import OptionParser
-from subprocess import Popen
pidfile='/var/run/mirrormanager/crawler.pid'
@@ -21,6 +26,7 @@ class ForkingMaster:
self.active_children = None
self.max_children = max_children
self.devnull = open('/dev/null', 'rw')
+ self.timings = {}
def collect_children(self):
"""Internal routine to wait for died children."""
@@ -37,12 +43,14 @@ class ForkingMaster:
pid = None
if not pid: break
self.active_children.remove(pid)
+ self.stop_time(pid)
- def process_request(self, command, args):
+ def process_request(self, command, args, hostid):
"""Fork a new subprocess to process the request."""
self.collect_children()
print "Starting crawler %s" % args
p = Popen(args, executable=command, stdin=self.devnull, stdout=self.devnull, stderr=self.devnull, close_fds=True)
+ self.start_time(p.pid, hostid)
# Parent process
if self.active_children is None:
self.active_children = []
@@ -52,6 +60,19 @@ class ForkingMaster:
self.max_children = 0
self.collect_children()
+ def start_time(self, pid, hostid):
+ self.timings[pid] = dict()
+ self.timings[pid]['start'] = datetime.datetime.utcnow()
+ self.timings[pid]['hostid'] = hostid
+
+ def stop_time(self, pid):
+ self.timings[pid]['stop'] = datetime.datetime.utcnow()
+
+ diff = self.timings[pid]['stop'] - self.timings[pid]['start']
+ host = Host.get(self.timings[pid]['hostid'])
+ sys.stdout.write('Host name %s crawl time %s\n' % (host.name, str(diff)))
+ sys.stdout.flush()
+ del self.timings[pid]
def doit(options, include_private=False, threads=10, config=None, logdir=None):
master = ForkingMaster(max_children=threads)
@@ -74,7 +95,7 @@ def doit(options, include_private=False, threads=10, config=None, logdir=None):
logfilename = os.path.join(logdir, str(h.id) + '.log')
args.append('--logfile=%s' % logfilename)
- master.process_request(command, args)
+ master.process_request(command, args, h.id)
master.wait_for_completion()
10 years, 10 months
[mirrormanager] master's head updated: crawler_perhost: fix traceback by moving host definition higher (c0c8eaf)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
Branch 'master' now includes:
709e10d SCHEMA CHANGE: add class Zone, many-to-many with Hosts
a287438 include update_1_4_0 in schema updates
7519bda export and use zone.hosts information
4d1ae10 SCHEMA CHANGE: add class Schema
eb01186 create Schema table on update
333a7a5 s/zone/location/g everywhere
43cc891 s/zone/location/g in schema_updates too
085da93 SCHEMA CHANGE: add FileGroups to model
f16fc94 SCHEMA CHANGE: drop class Schema, add cols to Site
447d0a6 mirrorlist_server: return hosts in netblocks in ascending netblock size order
aae2ac8 remove dead code
4ff7a71 handle malformed pidfiles
f72f20e remove pushd bashism from Makefile
c3304d9 dev.cfg: add ubuntu to umdl
3a15ddc add ubuntu initialization code
31c955c remove start-mirrormanager.py, use mirrormanager.wsgi instead
4ff484c SCHEMA CHANGE: add Version.sortorder and Host.dnsCountryHost
6266d22 adminview: add [Add] buttons to object types
d3abedc controllers: update Site, Host, SimpleDbObject, add Location and LocationHost controllers
b16b28a fix Makefile chdir/cd fix again
120b4ba spec: fix changelog order
37afb6d fix update_1_4_0 to compile
d18eb52 LICENSES: fix zebra directory note
fcdc6a5 fix move-to-archive
76d93b3 add move-devel-to-release
c818164 SCHEMA CHANGE: add Version.codename, fix Version.sortorder
35c9c6e scratch item off TODO
9166f5c add Category creation to adminview
9e7b9c5 expose CategoryController, strip user-provided Directory name
26dbcdc allow longer User user_name fields, to prepare for OpenID
77363a2 umdl: add Debian/Ubuntu version detection
0d1a872 fix move-devel-to-release
fee3e61 Merge branch 'v1.4' of ssh+git://fedorahosted.org/git/mirrormanager into v1.4
31453bc always initialize Version.sortorder = id, change later if you wish
1712040 Revert "always initialize Version.sortorder = id, change later if you wish"
7f3dc9c SCHEMA CHANGE: add field to hide Products from publiclist chooser
6156fb6 SCHEMA CHANGE: add Category.GeoDNSDomain field
131d6ee umdl: handle CentOS
241ee8a remove obsolete categories from example report_mirror.conf
e94f6fb create /var/lock and /var/run if necessary, per https://bugzilla.redhat.com/show_bug.cgi?id=656623
35bc757 add tmpfiles.d conf file, remove previous %ghost dirs. Fixes BZ656623
c479db8 use Product.publiclist
c8739ab Makefile: create /etc/tmpfiles.d
530d747 setup.py: require TurboGears, due to multiple versions of CherryPy
cea4aab initial attempt to export data as DNS zones
5b29597 controllers: expose Category GeoDNSDomain field
b2257f9 dns: move default output directory to a module variable
fd48293 add errorcheck CLI
56bdab2 supress some messages, wordsmith others.
eba044f minor fixups to controllers, Location stuff untested
19ca121 add destroySelf() to Location
54f349b fix adminview locationhost deletion
dedc018 add boringlocationhostform
a09828b fix mod_wsgi requirements
4853836 add/fix location info on adminview and host web pages
112fe43 host web text cleanups
96fe95c adminview: don't include a link to a Location - it's not linkable
74304ee controllers: minor cleanups
3911127 model: use @staticmethod for selectFieldOptions()
dfe430e fix adminview Categories [Add] button
499adb4 rename dns.py to avoid conflicts with system libs
72754b1 fix import errors in zonecreator
43d8418 finish deleting dns.py
184c7d2 fix zonecreator to not crash
a1a5e00 fix CategoryController.create
7432771 move lookupCategory into Category as staticmethod
2baf77d store DNS metadata in the database too
c5f9e83 drop CategoryDnsMX as unneeded, set default SOA values
33bc667 mirrorlist_server: speed up host netblock lookups using radix trees for the common case of no match
b13f624 fix previous commit
254c185 remove DNS SOA stuff, plan to include resulting per-category zonefile via $INCLUDE
bbeacdd report number of sites and hosts in adminview
9c4eaf0 errorcheck: add ability to send email, other small fixups
622764e fix errorcheck to actually run under turbomail v2
84e27f5 errorcheck: no FQDN warning for private hosts
192dfc4 model: don't let report_mirror checkins create HostCategory objects
4e522f3 errorcheck: supress a couple warnings for now
b23a4e5 errorcheck: supress messages for inactive Hosts
3417651 repomap: development/15 should get named repositories
3a7162a repomap: exclude non-Everything directories from releases/ from getting named repos
042fc74 grab internet2 netblocks from all routers and aggregate results
ed97273 mirrorlist_server: fix host netblocks lookup
4780437 add missing comma
ab0e395 fix schema updates, add update-schema tool
fe3754a remove UPGRADING.txt - we have a new method now
d4408b7 we don't need this sqlobject patch as we aren't using tg-admin upgrade anymore
fcd3d55 use global
2c4ac04 fix typo
79df6c0 ip -> clientIP
56bdcb1 fix adminview statistics
b734157 shorten adminview by dropping directories, repositories, and netblocks long lists
6a65f14 create host_location table in the database on upgrade
79c72e2 add uniqueness index to host_location
24b5c76 catch errors in umdl.master_directories earlier, avoids later failures
6733901 umdl: lookup category early, pass it down
249c59c create HostLocation intermediate table in model
fb05506 errorcheck: add tests; fix fails-to-run bugs
502f065 mirrorlist_server: on invalid requested repo, return repo=%s&arch=%s
b23cc6f Add dep on supervisor. Use supervisor to run mirrorlist_server.py
e181906 supervisor .conf -> .ini
5e49f7d add comments to README.rpm_setup
87e2349 cleanups to mirrorlist-server directory
9804298 fix pidfile creation
7f76e40 excise dnsCountryHost
bfa67b0 SCHEMA CHANGE: add Country and HostCountry tables, use in zonecreator
cadc807 use os.path.dirname rather than splits and joins
6ca60f5 mark child dirs unreadable if their parent is unreadable
d1024a4 umdl: use set() instead of dict for unreadable_dirs
be1cc6b SCHEMA CHANGE: add username/password fields to SiteToSite, to facilitate keeping rsync secrets files in MM
ecbf226 SCHEMA CHANGE: add NetblockCountry table to allow overriding of GeoIP data
9ef4097 crawler: don't keep probing bitflip-locked directories
b837454 updated export compliance text from spot
c6d9928 mirrorlist_server: add netblock country cache lookup
f035637 controllers: add NetblockCountry simple interface
740c8dc mirrorlist_server: unify calls to IPy.IP(client_ip) to one place
5b00b84 mirrorlist_server: early exit from tree_lookup() if more results aren't needed
6b59eb6 mirrorlist_server: fix radix tree lookups
ba082b1 add update-database-schema, add update_countries()
6f0fa9c s/fedoraproject.org/example.com/
444f904 update README
5475353 Two fixes for TG-1.1.x: Set the default template engine to kid explicitly and remove unneeded sqlalchemy.dburi:/// for using jsonfas auth.
5d0d1c6 SCHEMA CHANGE: add HostPeerAsn table and plumbing
3bddb76 SCHEMA CHANGE: add name fields to HostNetblock and HostPeerAsn tables
4ef5d1c Merge branch 'v1.4' of ssh+git://fedorahosted.org/git/mirrormanager into v1.4
51f0300 add /catwalk controller
db54975 remove old adminview, replace with catwalk at /adminview instead
74b0cf1 s/Toolbox/Home/ and fix url
fdf97c5 fix typo preventing compilation
a877b38 fix make rpm
d6ae156 fix adminview typo
2e8d7db fix url quoting
34b6829 fix update-database-schema runtime failures
834966d fix catwalk by subclassing rather than copying
3bdc3ab fix update-database-schema countries
8808bb1 manually create file_detail_file_group intermediate table
1ba32f5 fix toolbox graphics 404s
57c1a50 update umdl nuke_gone_directories()
bfcf5eb umdl: use OptionParser
3814c07 umdl: add --delete-directories option
ddd0edb require python-fedora-turbogears
5dc1a3d add length to SiteToSite username field
a76e301 require TG<1.2, now that 1.1.3 is in Fedora
20a99b9 Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4
a2c734b explicitly select kid engine on most templates
9a42b35 force catwalk template to use kid also
8a58299 make catwalk-session dir
c4f051d add length to Directory.files PickleCol
48ca8fc fix host.kid template at new() time when values=None
989fa06 fix hand-crafted SQL statements
121192c add cron examples to docs
bb685ca use find -print0 | xargs -0 - filenames can have spaces
560509d add missing import
59efab7 greatly speed up mirrorlist generation
83a0f52 add append_value_to_cache() to lib.py
fb8e907 mirrorlist.py: use append_value_to_cache now in lib
27c9034 model: cleanup Directory caching and aging
2f3141e model: add setup_directory_category_cache() to speed up umdl
6dc4f30 SCHEMA UPDATE: add Version name+product unique index
81e4544 umdl: major speedups
7d944fe fix cron for umdl requiring -c now
bc10ba8 SCHEMA CHANGE: set length on Version.name so index works
2377fdc setup_directory_category_cache moved to model
f82a3f9 automatically update Version.name length on schema update
f041881 SCHEMA CHANGE: delete unused HostCategory columns
d4b3de0 SCHEMA CHANGE: force host.bandwidth_int to be not null, integer
d5ee1a4 SCHEMA CHANGE: HostPeerASN.asn NOT NULL
279925c mirrorlist.py: use append_value_to_cache
edf787a SCHEMA CHANGE: Version.sortorder NOT NULL
30a99cf Merge branch 'v1.4' of ssh+git://fedorahosted.org/git/mirrormanager into v1.4
67444da umdl: fix exception catching
0c9a23d fix arguments to SODatabaseIndex()
b3266f3 update-database-schema: fix setting bandwidth_int
a99e0a7 SCHEMA CHANGE: nuke Host.bandwidth finally
b2e3fa7 another CamelCase fix
50ee140 SCHEMA CHANGE: delete unused Directory fields from the database
c8f0fc5 handle SODatabaseIndex() arg changes across SQLObject versions
27de416 SCHEMA CHANGE: nuke unused HCD lastCrawled and files columns
35fa3cb MM crawler (Fedora Infrastructure ticket #3268)
c4d0a34 crawler: add --category option to target just one category
0b83c2e crawler: remove useless print that's also causing a traceback because stdout is closed
a727125 crawler: listify database queries
1c2b171 SCHEMA CHANGE: add Host.max_connections, expose in metalink
234f225 ensure max_connections >= 1
bad9390 missed a comma
14e9c6a move maps to a separate subpackage
58653d4 schema_updates: fix host_category_dir table name
c3bbc1b Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4
209cb4f schema_updates: fix camelcase
4494c17 update TODO a little
9bf8c2d don't include embargoed countries in the map
1760909 generate-worldmap: require TurboGears[future]
5d61bb4 crawler: handle zero-length files
d448684 Upgrade apache config files to support both httpd 2.2 and 2.4
ac72488 flush logfile before exiting to be sure our messages hit the disk
a392148 umdl: speed up short_filelist(), keep only most recent 3 files
3931a0f don't add an extra / to mirrorlist URLs for files in the topdir
ef899f2 fix typo
df4b050 update TODO
4ddb80a note RFC1918 addresses aren't allowed
bfc91e3 begin cleanup of initial user creation
5644129 add README.mysql
40bdd33 change crawler timeout from 60 minutes to 90 minutes
d98150c mirrorlist_server.py: output the exception and the args being passed when we hit a bad request response
368fadd generate-publiclist: handle bandwidth values not being integers as expected
d5f6c2d add report_mirror --exclude-from= option to exclude directories
51d7e1a force kid templates by default
d5c516f remove cut-and-paste error from NetblockCountry
50168ac fix indentation causing startup failure
460cca4 add metalink field to requests, update to current $dist
3798031 mirrorlist_server: renamed argument
6fd1b28 Add syslog logging of each client request and the mirror type they are returned
6a0bfb5 mirrorlist_server: fix item list type
43b0b19 add .gitignore
8ca39be use more recent copy of GPLv2 license text
b3bcda0 use more recent copy of GPLv2 license text for zebra-dump-parser too
dc276a3 drop ancient obsolete ChangeLog file
88ede1c update server_tester to more closely match mirrorlist_server_test
ad13219 remove duplicate mirrorlist_server test code
4382325 remove obsolete mirrorlist_client.py
b50b59e mirrorlist_server: prepend syslog message with "mirrorlist"
25c8759 Revert "force kid templates by default"
8bb3693 web ui: for unclassified errors, print the whole traceback
f04ae4e convert ASN value to int
197b4d2 hook up /host_peer_asn URL
fe51aba host.kid: clean up Peer ASN list presentation
d2c3daa don't include obsolete sqlobject-history directory in the RPM
ec1c479 speed up welcome page rendering for sysadmins
5a27e0f add metalink/HTTP to TODO
f320d6c add schema doc generation to Makefile
62fe3f7 add theme-ing to TODO
47ce31f add initialize-database script, document it in README.mysql
6592df0 create Arch entries in initialize-database
cc4051b create new Arches with primaryArch set
85d6987 initial import of add/mv-instrepos - NEEDS WORK
486df36 add spec changelog, begin pre-releases
d017f91 handle CentOS mirrorlist parameters
d781cc4 2 fixes to schema updates
5a83e95 use hashlib instead of deprecated sha
1e74c9c handle bad weight values in weighted_shuffle
7f0cf8f Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4
e8527c0 force 1 <= bandwidth_int <= 100000 in cache creation
26cab4f handle multiple Repositories pointing at the same Directory
10b5add Merge branch 'v1.4' of ssh+git://fedorahosted.org/git/mirrormanager into v1.4
67b5bd5 keep 1 old copy of /var/lib/mirrormanager in /var/lib/mirrormanager/old/
aa7caa0 log mirrorlist_server stderr by supervisord
fff6005 Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4
ba77761 move-devel-to-release: add required Category option
4ad599f mirrorlist.py: fix shrink()
aa3ffb9 mirrorlist.py: fix file_details_cache()
e995677 model.py: fix RLIKE usage
8d8d59a mirrorlist_server: reload caches in a separate thread
bae5a85 mirrorlist_server: catch SIGTERM and cleanup
ca363f9 mirrorlist.py: list()ify selects
9c312c7 mirrorlist.py: fix broken logic that lead to tracebacks
e215af0 generate-publiclist: bandwidth_int must be >=1, if None set to 1
3b87a3f add import_centos; initial draft
13e07db add distros/ as placeholder for distro-specific code
0c4fb13 bump spec
03354d7 fix permissions on import_centos
05dedb6 remove history_dir from .egg-info
f034e70 mirrorlist-server apache conf: add /static alias
82bb1ec fix typo in mirrorlist_client.wsgi
b88aca9 schema update: fix upstream_site_id column name
d956dd8 fix setting host.max_connections not null
f7b6886 model: fix GROUP BY clause for postgres use
fb02391 mirrorlist_server: remove _ordered_netblocks() usage
69b7e03 fix rlike syntax - it was backwards
e5baa9f move update-static-content into its own cronjob
573018b cron: add update-static-content daily job
2d711e3 publiclist: remove comment about hourly generation
d671bc2 bump release
e736171 Fix sorting results by preferred netblock size
93a2f9a write tracebacks to stderr
8473cc6 remove explicit calls to del
e713adf mirrorlist_server: restart syscalls rather than just interrupting them
5d077bd bump spec
0774e43 factor out the rsync command to lib.py
956aa6e use rsync to crawl a host if available
263ab97 fix typo
7748f2e add induce-stress script
6c0c17d Merge branch 'v1.4' of ssh+git://fedorahosted.org/git/mirrormanager into v1.4
2c4ce73 add mirrorlist-server/test/* to the packages
bdd3b3b Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4
92d83e5 Merge remote-tracking branch 'origin/crawler-rsync' into v1.4
906f13c bump spec
c53c9b6 mirrorlist_server:lots of bug fixes
b2c55b8 mirrorlist_server: attempt to fix traceback from threading.start()
069e34c mirrorlist_server: don't use print, use sys.stdout.write()
27d8348 bump spec
4e862cd bump release
f34ec13 update copyright dates throughout
f358ed7 switch to xz compression, bump RELEASE_DATE
6793d3a fix xz
d41be10 s/print/sys.stdout.write/ in generate-publiclist
32567dd fix metalinks
eb288c3 fix maxconnections again
bfd83f2 mirrorlist_server: reduce memory usage considerably
7fb0cb6 clean up syntax a bit
6bec9d4 fix umdl failure
b2f6924 umdl: return the right type of object
d03bebd umdl: don't make duplicate Repository objects
d20e2e7 umdl: s/print/sys.stdout.write/
3dc150a use sorted() where possible in prep for python3
19aae22 umdl: fix nuke_gone_directories, enable --delete-directories
742dfc7 Fix very nasty bug where files would be put in the incorrect Directory
c224b12 umdl: fix collision on variable name options
a894284 one more killall removal
7126cd4 end pidfile with a newline
941c0b7 sighup supervisord to reopen its log file after rotate
7a0bc8b bump release to 1.4.1
9b930ab fix run_rsync() in umdl and crawler
9d0a0e8 fix lib.py imports
61eb2ed crawler: symlink fix, printing fix
4fee834 add to TODO
2c99120 set mirrorlist WSGI processes=45, threads=1
af5dcda model.py: fix sqlbuilder imports
f7332fe add count-mirrorlist-hits script
80dd827 model.py: fix user_sites() variable name
2aaefb0 nuke unused adminview.kid
20ede25 hide Admin View link for non-sysadmins
14d5b46 future-proof mv-instrepos
72c8e07 don't create install repos for arch content that does not exist
5908add let the database return publiclist_arches() in the right order
a5fa741 fix publiclist Architectures header spanning
d2b88a1 mv-instrepos: bug fixes
dda3291 crawler: fixes
ff8b74f run_rsync(): return the rsync exit code, log nonzero exits
f4c85f7 remove set-but-unused variables
c2c89ac netblock_country_cache key should be an IP() object, not a string
4286727 cleanup imports
e912501 move-devel-to-release: Don't traceback if a Directory doesn't exist yet
88ee40d model: add backref Category->Repositories
6a91ccd fix move-devel-to-release
c556827 umdl: add debug message when creating a Repository
a24b663 fix for "expected a bool or an int"
e1048d5 https://fedorahosted.org/fedora-infrastructure/ticket/3880
c0c8eaf crawler_perhost: fix traceback by moving host definition higher
10 years, 10 months
[mirrormanager] master: crawler_perhost: fix traceback by moving host definition higher (c0c8eaf)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit c0c8eaf450fbe0a2300e35c678a0fdca0bcd1f67
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Jul 11 16:25:18 2013 -0500
crawler_perhost: fix traceback by moving host definition higher
>---------------------------------------------------------------
server/crawler_perhost | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/crawler_perhost b/server/crawler_perhost
index b546bab..58cb531 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -646,11 +646,11 @@ def sighup_handler(signum, frame):
def select_host_categories_to_scan(options):
result = []
+ host = Host.get(options.hostid)
if len(options.categories):
for c in options.categories:
try:
category = Category.byName(c)
- host = Host.get(options.hostid)
hc = HostCategory.selectBy(host=host, category=category)[0]
result.append(hc)
except SQLObjectNotFound:
10 years, 10 months