TODO | 2 -
server/crawler | 7 +++--
server/crawler_perhost | 68 ++++++++++++++++++++++++++++++++++++++++++++-----
server/dev.cfg | 5 +++
4 files changed, 72 insertions(+), 10 deletions(-)
New commits:
commit f5da33ec1bf29b65f6a2f80e583e181cf4a295ef
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat May 1 09:34:57 2010 -0400
crawler: send email when marking a whole host not up2date
This does not send mail to each individual admin yet. It also
requires prod.cfg to be updated. See dev.cfg for the crawler.*
options.
diff --git a/server/crawler_perhost b/server/crawler_perhost
index 881fc4b..eda28aa 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -6,6 +6,7 @@ pkg_resources.require("TurboGears")
from sqlobject import *
import sys, os, signal, socket
import turbogears
+from turbogears import config
from mirrormanager.model import *
from datetime import datetime
import ftplib
@@ -16,6 +17,7 @@ from optparse import OptionParser
import threading
import urlgrabber
import hashlib
+from smtplib import SMTP
from turbogears.database import PackageHub
hostid = None
@@ -454,17 +456,62 @@ def dienow():
os.kill(mypid, signal.SIGUSR1)
os._exit(0)
-def mark_not_up2date(host, reason="Unknown"):
+def send_email(host, report_str, exc):
+ SMTP_DATE_FORMAT = "%a, %d %b %Y %H:%M:%S %z"
+ msg = """From: %s
+To: %s
+Subject: %s MirrorManager crawler report
+Date: %s
+
+""" % (config.get('crawler.mail_from'),
+ config.get('crawler.admin_mail_to'),
+ host.name,
+ time.strftime(SMTP_DATE_FORMAT))
+
+ msg += report_str + '\n'
+ msg += 'Log can be found at %s/%s.log\n' %
(config.get('crawler.logdir'), str(host.id))
+ if exc is not None:
+ msg += "Exception info: type %s; value %s\n" % (exc[0], exc[1])
+ msg += str(exc[2])
+ try:
+ smtp = SMTP(config.get('crawler.smtp_host'),
+ config.get('crawler.smtp_port'))
+
+ username = config.get('crawler.smtp_username')
+ password = config.get('crawler.smtp_password')
+
+ if username and password:
+ smtp.login(username, password)
+
+ smtp.sendmail(config.get('crawler.smtp_from'),
+ config.get('crawler.admin_mail_to'),
+ msg)
+ except Exception, e:
+ print "Error sending email"
+ print e
+ print "Email message follows:"
+ print msg
+
+ try:
+ smtp.quit()
+ except:
+ pass
+
+def mark_not_up2date(exc, host, reason="Unknown", mail=True):
host.set_not_up2date()
host.lastCrawled = datetime.utcnow()
- print >> logfile, "Host marked not up2date: %s" % reason
-
+ msg = "Host marked not up2date: %s" % reason
+ print >> logfile, msg
+ if exc is not None:
+ print >> logfile, exc[0], exc[1], exc[2]
+ if mail:
+ send_email(host, msg, exc)
def sigusr1_handler(signum, frame):
if signum == signal.SIGUSR1:
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
try:
- mark_not_up2date(Host.get(hostid), "Crawler timed out before completing.
Host is likely overloaded.")
+ mark_not_up2date(None, Host.get(hostid), "Crawler timed out before
completing. Host is likely overloaded.")
except:
pass
os._exit(1)
@@ -561,7 +608,7 @@ def per_host(host, options=None):
pass
except:
timer.cancel()
- mark_not_up2date(host, "Unknown exception raised. This is a bug in
the MM crawler.")
+ mark_not_up2date(sys.exc_info(), host, "Unknown exception raised.
This is a bug in the MM crawler.")
sys.exit(1)
hoststate.close()
@@ -571,7 +618,7 @@ def per_host(host, options=None):
if len(host_category_dirs) > 0:
sync_hcds(host, host_category_dirs)
else:
- mark_not_up2date(host, "No host category directories found. Check that your
Host Category URLs are correct.")
+ mark_not_up2date(None, host, "No host category directories found. Check
that your Host Category URLs are correct.")
def main():
global mypid
diff --git a/server/dev.cfg b/server/dev.cfg
index a02cd89..962453d 100644
--- a/server/dev.cfg
+++ b/server/dev.cfg
@@ -62,6 +62,11 @@ mrr.repos = ''' {
'updates-testing-source-f%s':'rawhide-source'
} '''
+crawler.mail_from = 'mirror-admin(a)fedoraproject.org'
+crawler.admin_mail_to = 'mirror-admin(a)fedoraproject.org'
+crawler.smtp_host = 'localhost'
+crawler.smtp_port = 25
+crawler.logdir = 'https://admin.fedoraproject.org/mirrormanager/crawler'
# This is where all of your settings go for your development environment
# Settings that are the same for both development and production
commit 80c22e55482fbc83cdedcaf0437e4c0004782e73
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat May 1 08:30:53 2010 -0400
crawler: crawl private i2 hosts that also say i2_clients are ok
diff --git a/TODO b/TODO
index 6c93a5b..19041ed 100644
--- a/TODO
+++ b/TODO
@@ -17,8 +17,6 @@
* cronjob to mark private mirrors as not-up2date, some amount of time
after the content has changed but they haven't run report_mirror.
-* can the crawler check private, but serving I2 clients, mirrors?
-
* metalinks for multiple CDs/files
- Just like torrents, we should have a single metalink that downloads
all the CDs in a set via one URL.
diff --git a/server/crawler b/server/crawler
index eaabaa0..3cb3753 100755
--- a/server/crawler
+++ b/server/crawler
@@ -63,9 +63,12 @@ def doit(options, include_private=False, threads=10, config=None,
logdir=None):
commonargs.append('--canary')
for h in Host.select():
- if not h.is_active() or (not include_private and h.is_private()):
+ if not h.is_active():
continue
-
+ if not include_private and h.is_private():
+ if not (h.internet2 and h.internet2_clients):
+ continue
+
args = commonargs + ['--hostid=%s' % h.id]
if logdir is not None:
logfilename = os.path.join(logdir, str(h.id) + '.log')
commit 7c92f194683059a3b70903d81a933dcac34168a1
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat May 1 08:21:44 2010 -0400
crawler: implement canary mode
diff --git a/server/crawler_perhost b/server/crawler_perhost
index ebc3fa6..881fc4b 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -487,6 +487,11 @@ def sighup_handler(signum, frame):
def per_host(host, options=None):
+ """Canary mode looks for 2 things:
+ directory.path ends in 'iso' or directory.path ends in 'repodata'.
In
+ this case it checks for availability of each of the files in those
+ directories.
+ """
global must_dienow
host = Host.get(host)
done_host=False
@@ -530,6 +535,10 @@ def per_host(host, options=None):
if not d.readable:
continue
+ if options.canary:
+ if not (d.name.endswith('/repodata') or
d.name.endswith('/iso')):
+ continue
+
dirname = d.name[categoryPrefixLen:]
url = '%s/%s' % (categoryUrl, dirname)