Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/repomap.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
New commits:
commit 3a7162aa0b5b189f94aa4a47cc9dfdc597fca2ed
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Feb 12 08:01:09 2011 -0600
repomap: exclude non-Everything directories from releases/ from getting named repos
diff --git a/server/mirrormanager/repomap.py b/server/mirrormanager/repomap.py
index 3f17b13..74390a7 100644
--- a/server/mirrormanager/repomap.py
+++ b/server/mirrormanager/repomap.py
@@ -162,8 +162,9 @@ def repo_prefix(path, category, ver):
elif isFedoraLinux or isFedoraSecondary or isFedoraArchive:
if isReleases or isDevelopment:
+ if isReleases and not isEverything: prefix = None
# fedora-
- if isDebug:
+ elif isDebug:
prefix = u'fedora-debug-%s' % version
elif isSource:
prefix = u'fedora-source-%s' % version
13 years, 2 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/repomap.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
New commits:
commit 3417651b632f97659c2c9ef22f2dc1e06e92f10e
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Feb 12 00:49:17 2011 -0600
repomap: development/15 should get named repositories
Now that Fedora does early branching, and both development/15 and
development/rawhide trees exist, it's correct for development/15/*/
repositories to have the proper fedora-15 prefix, instead of being
handled by repository redirects to rawhide as we've done in the past
but which is now incorrect as rawhide lives on independent of 15/.
diff --git a/server/mirrormanager/repomap.py b/server/mirrormanager/repomap.py
index 27665b6..3f17b13 100644
--- a/server/mirrormanager/repomap.py
+++ b/server/mirrormanager/repomap.py
@@ -162,10 +162,8 @@ def repo_prefix(path, category, ver):
elif isFedoraLinux or isFedoraSecondary or isFedoraArchive:
if isReleases or isDevelopment:
- if not isEverything:
- prefix = None
# fedora-
- elif isDebug:
+ if isDebug:
prefix = u'fedora-debug-%s' % version
elif isSource:
prefix = u'fedora-source-%s' % version
13 years, 2 months
Branch 'v1.4' - server/errorcheck
by Matt Domsch
server/errorcheck | 2 --
1 file changed, 2 deletions(-)
New commits:
commit b23a4e59c3108655fc8d102b137ec288619f8ce5
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Feb 4 09:27:59 2011 -0600
errorcheck: supress messages for inactive Hosts
diff --git a/server/errorcheck b/server/errorcheck
index a811114..82355da 100755
--- a/server/errorcheck
+++ b/server/errorcheck
@@ -161,8 +161,6 @@ def check_host(sr, host):
return
if not host.is_active():
- msg = u"Host %s is not active - ignoring." % (host.name)
- sr.messages.append(msg)
return
ips = name_to_ips(host.name)
13 years, 2 months
Branch 'v1.4' - 2 commits - server/errorcheck server/mirrormanager
by Matt Domsch
server/errorcheck | 29 ++++++++++++++++-------------
server/mirrormanager/model.py | 4 +++-
2 files changed, 19 insertions(+), 14 deletions(-)
New commits:
commit 4e522f345ba808524488948b93e7e3282ea198c0
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Feb 4 08:56:33 2011 -0600
errorcheck: supress a couple warnings for now
diff --git a/server/errorcheck b/server/errorcheck
index be7d5fe..a811114 100755
--- a/server/errorcheck
+++ b/server/errorcheck
@@ -213,19 +213,22 @@ def check_host(sr, host):
sr.messages.append(msg)
always_up2date = False
- for hc in host.categories:
- if len(hc.urls) == 0:
- msg = u"Host %s Category %s has no URLs. Please add a URL to the content your mirror carries, or if not carried, delete the Category." % (host.name, hc.category.name)
- sr.messages.append(msg)
-
- if hc.always_up2date: always_up2date = True
- if len(hc.dirs) == 0 and not hc.always_up2date:
- msg = u"Host %s Category %s has no up-to-date directories. " % (host.name, hc.category.name)
- if len(hc.urls) > 0:
- msg = msg + u"Check that your Category URL points to %s. " % (hc.category.topdir.name)
-
- msg = msg + u"Consult the crawler logs and check your report_mirror.conf."
- sr.messages.append(msg)
+
+ # hide this for now, until checkins are fixed
+ if 0:
+ for hc in host.categories:
+ if len(hc.urls) == 0:
+ msg = u"Host %s Category %s has no URLs. Please add a URL to the content your mirror carries, or if not carried, delete the Category." % (host.name, hc.category.name)
+ sr.messages.append(msg)
+
+ if hc.always_up2date: always_up2date = True
+ if len(hc.dirs) == 0 and not hc.always_up2date:
+ msg = u"Host %s Category %s has no up-to-date directories. " % (host.name, hc.category.name)
+ if len(hc.urls) > 0:
+ msg = msg + u"Check that your Category URL points to %s. " % (hc.category.topdir.name)
+
+ msg = msg + u"Consult the crawler logs and check your report_mirror.conf."
+ sr.messages.append(msg)
if host.lastCheckedIn is None and not always_up2date:
msg = u"Host %s has never checked in with report_mirror." % (host.name)
commit 192dfc4ea4041818d22ec0ac990390b0b23e1d23
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Feb 4 08:55:56 2011 -0600
model: don't let report_mirror checkins create HostCategory objects
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index cd4817f..8838302 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -259,7 +259,9 @@ class Host(SQLObject):
if hc.count() > 0:
hc = hc[0]
else:
- hc = HostCategory(host=self, category=category)
+ # don't let report_mirror create HostCategories
+ # it must be done through the web UI
+ continue
marked_up2date = 0
deleted = 0
13 years, 2 months
Branch 'v1.4' - server/errorcheck
by Matt Domsch
server/errorcheck | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
New commits:
commit 84e27f57568239bf400fd38f1d7c7b7d23af7e07
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Feb 4 08:50:49 2011 -0600
errorcheck: no FQDN warning for private hosts
diff --git a/server/errorcheck b/server/errorcheck
index ac80431..be7d5fe 100755
--- a/server/errorcheck
+++ b/server/errorcheck
@@ -166,7 +166,7 @@ def check_host(sr, host):
return
ips = name_to_ips(host.name)
- if len(ips) == 0:
+ if len(ips) == 0 && not host.is_private():
msg = u"Host name %s not found in DNS. Please use a FQDN for your host name. You will also need to make the corresponding change in your report_mirror.conf file." % (host.name)
sr.messages.append(msg)
else:
13 years, 2 months
Branch 'v1.4' - server/errorcheck
by Matt Domsch
server/errorcheck | 62 +++++++++++++++++++++++++++++++++---------------------
1 file changed, 39 insertions(+), 23 deletions(-)
New commits:
commit 622764e1a450d2889ba6f6d92765d58af698c4ec
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Feb 4 00:20:49 2011 -0600
fix errorcheck to actually run under turbomail v2
diff --git a/server/errorcheck b/server/errorcheck
index 59c0f73..ac80431 100755
--- a/server/errorcheck
+++ b/server/errorcheck
@@ -22,6 +22,7 @@ import dns.resolver
from IPy import IP
from fedora.client.fas2 import AccountSystem
import turbomail
+from turbomail.startup import start_extension, shutdown_extension
from turbogears.database import PackageHub
from turbogears import config
@@ -42,10 +43,10 @@ class SiteResult(object):
self.messages = []
self.admin_emails = []
-def send_mail(sender, to, subject, body):
- message = turbomail.Message(sender, to, subject, plain=body)
+def send_mail(sender, to, subject, body, cc=[]):
+ message = turbomail.Message(sender, to, subject, plain=body, cc=cc)
try:
- turbomail.send(message)
+ turbomail.enqueue(message)
except turbomail.MailNotEnabledException, e:
print "Unable to send mail: %s" % (str(e))
except Exception, e:
@@ -219,7 +220,11 @@ def check_host(sr, host):
if hc.always_up2date: always_up2date = True
if len(hc.dirs) == 0 and not hc.always_up2date:
- msg = u"Host %s Category %s has no up-to-date directories. Consult the crawler logs and check your report_mirror.conf." % (host.name, hc.category.name)
+ msg = u"Host %s Category %s has no up-to-date directories. " % (host.name, hc.category.name)
+ if len(hc.urls) > 0:
+ msg = msg + u"Check that your Category URL points to %s. " % (hc.category.topdir.name)
+
+ msg = msg + u"Consult the crawler logs and check your report_mirror.conf."
sr.messages.append(msg)
if host.lastCheckedIn is None and not always_up2date:
@@ -229,7 +234,7 @@ def check_host(sr, host):
else:
msg += u" Public mirrors are encouraged to run report_mirror."
sr.messages.append(msg)
- else:
+ elif host.lastCheckedIn is not None:
if host.lastCheckedIn < (dt.datetime.utcnow() - one_day):
msg = u"Host %s has not checked in with report_mirror in 24 hours." % (host.name)
if host.is_private():
@@ -297,9 +302,9 @@ def setup_netblocks(netblocks_file):
def check_site(site):
if site.private and not options.private:
- return
+ return None
if not site.admin_active or not site.user_active:
- return
+ return None
sr = SiteResult(site)
sr.admin_emails = []
@@ -325,35 +330,40 @@ def check_site(site):
return sr
-def send_site_emails(sr):
+def send_site_emails(sr, cc=[]):
if len(sr.messages) == 0: return
if len(sr.admin_emails) == 0: return
- projectname = config.get(mirrormanager.project.name)
- sender = config.get(crawler.mail_from)
+ projectname = config.get('mirrormanager.project.name', u'')
+ sender = config.get('mirrormanager.mail_from')
+ if sender is None:
+ print "unable to send mail: mirrormanager.mail_from is not set"
+ return
+
subject = projectname + u' MirrorManager report for ' + sr.site.name
+ url = config.get('base_url_filter.base_url') + turbogears.url('/')
+ siteurl = url + 'site/' + str(sr.site.id)
body = u'''\
This is an automated report from the %(project)s MirrorManager at %(url)s.
-Site: %(site)s
+Site: %(site)s %(siteurl)s
+''' % dict(project=projectname, url=url, site=sr.site.name, siteurl=siteurl)
- ''' % dict(project=projectname, url=turbogears.url('/'), site=sr.site.name)
-
- for m in messages:
- body = body + u'\n'
+ for m in sr.messages:
+ body = body + m + u'\n'
to = sr.admin_emails
- send_mail(sender, to, subject, body)
+ send_mail(sender, to, subject, body, cc=cc)
def doit():
+ global options
global internet2_tree
global global_tree
- global options
internet2_tree = setup_netblocks(internet2_netblocks_file)
global_tree = setup_netblocks(global_netblocks_file)
for site in Site.select():
sr = check_site(site)
- if options.email: send_site_emails(sr)
+ if sr is not None and options.email: send_site_emails(sr, options.cc)
options = None
@@ -379,6 +389,9 @@ def main():
parser.add_option("-e", "--email",
dest="email", action="store_true", default=False,
help="Send emails to Site admins")
+ parser.add_option("-cc",
+ dest="cc", default=[],
+ help="Email address to include on cc:")
(options, args) = parser.parse_args()
@@ -395,8 +408,6 @@ def main():
sys.exit(1)
- turbomail.interface.start(turbogears.config)
-
global outfile
outfile = codecs.open(options.outfile, encoding='utf-8', mode='w')
@@ -405,10 +416,15 @@ def main():
hub = PackageHub("mirrormanager")
__connection__ = hub
- doit(options)
- outfile.close()
+ if options.email:
+ start_extension()
+
+ doit()
- turbomail.interface.stop()
+ if options.email:
+ shutdown_extension()
+
+ outfile.close()
remove_pidfile(pidfile)
13 years, 2 months
Branch 'v1.4' - server/dev.cfg server/errorcheck
by Matt Domsch
server/dev.cfg | 6 ++++
server/errorcheck | 71 +++++++++++++++++++++++++++++++++++++++++++++++-------
2 files changed, 68 insertions(+), 9 deletions(-)
New commits:
commit 9c4eaf0d7852985810dfbd85bf776ccf83d196b5
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Feb 1 00:04:01 2011 -0600
errorcheck: add ability to send email, other small fixups
diff --git a/server/dev.cfg b/server/dev.cfg
index 0c92d13..99280db 100644
--- a/server/dev.cfg
+++ b/server/dev.cfg
@@ -70,6 +70,12 @@ mrr.repos = ''' {
'updates-testing-source-f%s':'rawhide-source'
} '''
+mail.on = True
+mail.transport = 'smtp'
+mail.smtp.server = 'localhost'
+mail.smtp.debug = True
+mail.utf8qp.on = True
+
crawler.mail_from = 'mirror-admin(a)fedoraproject.org'
crawler.admin_mail_to = 'mirror-admin(a)fedoraproject.org'
crawler.smtp_host = 'localhost'
diff --git a/server/errorcheck b/server/errorcheck
index 046c62d..59c0f73 100755
--- a/server/errorcheck
+++ b/server/errorcheck
@@ -21,7 +21,7 @@ from optparse import OptionParser
import dns.resolver
from IPy import IP
from fedora.client.fas2 import AccountSystem
-
+import turbomail
from turbogears.database import PackageHub
from turbogears import config
@@ -42,6 +42,16 @@ class SiteResult(object):
self.messages = []
self.admin_emails = []
+def send_mail(sender, to, subject, body):
+ message = turbomail.Message(sender, to, subject, plain=body)
+ try:
+ turbomail.send(message)
+ except turbomail.MailNotEnabledException, e:
+ print "Unable to send mail: %s" % (str(e))
+ except Exception, e:
+ print "Exception thrown when trying to send mail: %s" % (str(e))
+
+
def name_to_ips(name):
namestotry=[]
@@ -228,12 +238,18 @@ def check_host(sr, host):
msg += u" Public mirrors are encouraged to run report_mirror."
sr.messages.append(msg)
+
+ if host.is_private() and len(host.netblocks) == 0 and (host.asn is None or not host.asn_clients):
+ msg = u"Host %s is private has no netblocks or ASN set or asn_clients set. Clients will not automatically be directed to your mirror." % (host.name)
+ sr.messages.append(msg)
+
+
AS = None
-def users_to_emails(userlist):
+def users_accountinfo(userlist):
global options
global AS
- result = []
+ result = {}
if AS is None:
if turbogears.config.get('identity.provider') == 'jsonfas2':
from fedora.client.fas2 import AccountSystem
@@ -241,7 +257,7 @@ def users_to_emails(userlist):
for user in userlist:
person = AS.person_by_username(user)
if person:
- result.append(person['email'])
+ result[user] = person
return result
def site_admins(site):
@@ -286,7 +302,12 @@ def check_site(site):
return
sr = SiteResult(site)
- sr.admin_emails = users_to_emails(site_admins(site))
+ sr.admin_emails = []
+ for username, person in users_accountinfo(site_admins(site)).iteritems():
+ sr.admin_emails.append(person['email'])
+ if person['status'] == u'inactive':
+ msg = u'Site Admin Username %s is inactive in FAS' % (username)
+ sr.messages.append(msg)
if len(site.hosts) == 0:
@@ -301,6 +322,27 @@ def check_site(site):
for msg in sr.messages:
outfile.write("\t%s\n" % msg)
outfile.write("\n")
+ return sr
+
+
+def send_site_emails(sr):
+ if len(sr.messages) == 0: return
+ if len(sr.admin_emails) == 0: return
+ projectname = config.get(mirrormanager.project.name)
+ sender = config.get(crawler.mail_from)
+ subject = projectname + u' MirrorManager report for ' + sr.site.name
+ body = u'''\
+This is an automated report from the %(project)s MirrorManager at %(url)s.
+Site: %(site)s
+
+ ''' % dict(project=projectname, url=turbogears.url('/'), site=sr.site.name)
+
+ for m in messages:
+ body = body + u'\n'
+
+ to = sr.admin_emails
+ send_mail(sender, to, subject, body)
+
def doit():
global internet2_tree
@@ -310,7 +352,9 @@ def doit():
global_tree = setup_netblocks(global_netblocks_file)
for site in Site.select():
- check_site(site)
+ sr = check_site(site)
+ if options.email: send_site_emails(sr)
+
options = None
@@ -327,12 +371,15 @@ def main():
dest="faspassword", default=None,
help="Fedora Account System password for queries")
parser.add_option("--private",
- dest="private", default=None,
+ dest="private", action="store_true", default=None,
help="include private mirrors")
parser.add_option("-o", "--output",
dest="outfile", default=None,
help="FILE to write to")
-
+ parser.add_option("-e", "--email",
+ dest="email", action="store_true", default=False,
+ help="Send emails to Site admins")
+
(options, args) = parser.parse_args()
@@ -347,6 +394,9 @@ def main():
print "Must specify an output file.\n"
sys.exit(1)
+
+ turbomail.interface.start(turbogears.config)
+
global outfile
outfile = codecs.open(options.outfile, encoding='utf-8', mode='w')
@@ -355,8 +405,11 @@ def main():
hub = PackageHub("mirrormanager")
__connection__ = hub
- doit()
+ doit(options)
outfile.close()
+
+ turbomail.interface.stop()
+
remove_pidfile(pidfile)
13 years, 2 months