Branch 'v1.4' - 2 commits - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 5 ++---
server/mirrormanager/templates/adminview.kid | 2 +-
2 files changed, 3 insertions(+), 4 deletions(-)
New commits:
commit 74304ee53f8cc47a5bf54205842cf4b0195191f4
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 15:34:44 2011 -0600
controllers: minor cleanups
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 20f11f7..b387a84 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -866,7 +866,7 @@ class CategoryFields(widgets.WidgetsList):
canonicalhost = widgets.TextField(validator=validators.UnicodeString, attrs=dict(size='30'))
topdirPath = widgets.TextField(validator=validators.UnicodeString, attrs=dict(size='30'))
publiclist = widgets.CheckBox()
- GeoDNSDomain = widgets.TextField(validator=validators.UnicodeString, attrs=dict(size='30'))
+ GeoDNSDomain = widgets.TextField(label="GeoDNS Domain", validator=validators.UnicodeString, attrs=dict(size='30'))
category_form = widgets.TableForm(fields=CategoryFields(), submit_text="Create Category")
@@ -1132,7 +1132,6 @@ class LocationController(SimpleDbObjectController):
# LocationHosts
#########################################################
class LocationHostFields(widgets.WidgetsList):
- locationName = widgets.Label(label="Location Name")
hostid = widgets.TextField(validator=validators.Int, label="Host ID")
location_host_form = widgets.TableForm(fields=LocationHostFields(), submit_text="Create Location / Host Mapping")
@@ -1149,7 +1148,7 @@ class LocationHostController(controllers.Controller, identity.SecureResource, co
raise redirect("/adminview")
submit_action = "/locationhost/create?locationid=%s" % locationid
- return dict(form=location_host_form, values=None, action=submit_action, page_title="New Location / Host", location=location)
+ return dict(form=location_host_form, values=None, action=submit_action, page_title="New Location / Host", location=location, locationName=location.name)
@expose(template="mirrormanager.templates.boringlocationhostform")
@error_handler(new)
commit 96fe95cb487eef45b235a6a49844923929fe3ce9
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 15:34:15 2011 -0600
adminview: don't include a link to a Location - it's not linkable
diff --git a/server/mirrormanager/templates/adminview.kid b/server/mirrormanager/templates/adminview.kid
index f1aa4b3..4f5a506 100644
--- a/server/mirrormanager/templates/adminview.kid
+++ b/server/mirrormanager/templates/adminview.kid
@@ -111,7 +111,7 @@ Nothing to see here, move along.
<div id="Locations">
<h3>Locations <a href="${tg.url('/location/new')}">[Add]</a></h3>
<ul>
- <li py:for="l in locations"> <a href="${tg.url('/location/'+str(l.id)+'/')}"><span py:replace="l.name">Location</span></a>
+ <li py:for="l in locations"><span py:replace="l.name">Location</span>
<a href="${tg.url('/locationhost/new?locationid=' + str(l.id))}">[Add Host]</a>
<a href="${tg.url('/location/' + str(l.id) + '/delete')}">[Delete]</a>
13 years, 3 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 6 +++---
server/mirrormanager/templates/host.kid | 6 ++++++
2 files changed, 9 insertions(+), 3 deletions(-)
New commits:
commit 112fe43ecc9609085ca37a811c187aead5bbcb44
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 14:10:10 2011 -0600
host web text cleanups
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 6c12db5..20f11f7 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -306,14 +306,14 @@ class HostFields(widgets.WidgetsList):
user_active = widgets.CheckBox("user_active", default=True, help_text="Uncheck this box to temporarily disable this host, it will be removed from public listings.")
country = widgets.TextField(validator=validators.All(validators.Regex(r'^[a-zA-Z][a-zA-Z]$'),validators.NotEmpty),
help_text="2-letter ISO country code" )
- bandwidth_int = widgets.TextField(validator=validators.Int, help_text="* integer megabits/sec, how much bandwidth this host can serve")
+ bandwidth_int = widgets.TextField(label="Bandwidth", validator=validators.Int, help_text="* integer megabits/sec, how much bandwidth this host can serve")
private = widgets.CheckBox(help_text="e.g. not available to the public, an internal private mirror")
internet2 = widgets.CheckBox(help_text="on Internet2")
internet2_clients = widgets.CheckBox(help_text="serves Internet2 clients, even if private")
asn = widgets.TextField("asn", label="ASN", default='', validator=validators.Int, help_text="Autonomous System Number, used in BGP routing tables.")
asn_clients = widgets.CheckBox('asn_clients', label="ASN Clients?", default=True, help_text="Serve all clients from the same ASN. Used for ISPs, companies, or schools, not personal networks.")
- robot_email = widgets.TextField(validator=validators.All(validators.UnicodeString,validators.Email), help_text="email address, will receive notice of upstream content updates")
- dnsCountryHost = widgets.CheckBox(help_text="use host in <country-code>.<domain> DNS entries")
+ robot_email = widgets.TextField(label="Robot email", validator=validators.All(validators.UnicodeString,validators.Email), help_text="email address, will receive notice of upstream content updates")
+ dnsCountryHost = widgets.CheckBox(label="DNS Country Host", help_text="use host in <country-code>.<domain> DNS entries")
comment = widgets.TextField(validator=validators.Any(validators.UnicodeString, validators.Empty), help_text="text, anything else you'd like a public end user to know about your mirror")
host_form = widgets.TableForm(fields=HostFields(),
diff --git a/server/mirrormanager/templates/host.kid b/server/mirrormanager/templates/host.kid
index ee2bdf3..86134ab 100644
--- a/server/mirrormanager/templates/host.kid
+++ b/server/mirrormanager/templates/host.kid
@@ -59,6 +59,12 @@ Last Crawled: ${values.lastCrawled} <a href="${tg.url('/crawler/'+str(values.id
</div>
<h3>Locations</h3>
+Locations are ways to group hosts when netblocks are unwieldy.
+Examples include Amazon EC2 availability zones. Mirrorlist clients
+append location=string to their request URLs to specify a preferred
+location. Setting up locations requires MirrorManager administrator
+privileges.
+
<UL>
<li py:for="l in values.locations">
<span py:replace="l.name">Location Name</span>
13 years, 3 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 29 ++++++++++++++++++++-------
server/mirrormanager/templates/adminview.kid | 8 +++++--
server/mirrormanager/templates/host.kid | 8 +++++++
3 files changed, 36 insertions(+), 9 deletions(-)
New commits:
commit 4853836a9bd1d19a4acd08978132b656f824b8a2
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 13:47:18 2011 -0600
add/fix location info on adminview and host web pages
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index b4e005b..6c12db5 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -1141,8 +1141,9 @@ class LocationHostController(controllers.Controller, identity.SecureResource, co
require = identity.in_group(admin_group)
@expose(template="mirrormanager.templates.boringlocationhostform")
- def new(self, locationid, **kwargs):
+ def new(self, **kwargs):
try:
+ locationid = kwargs['locationid']
location = Location.get(locationid)
except sqlobject.SQLObjectNotFound:
raise redirect("/adminview")
@@ -1153,23 +1154,36 @@ class LocationHostController(controllers.Controller, identity.SecureResource, co
@expose(template="mirrormanager.templates.boringlocationhostform")
@error_handler(new)
@validate(form=location_host_form)
- def create(self, locationid, **kwargs):
+ def create(self, **kwargs):
+ try:
+ locationid = kwargs['locationid']
+ hostid = kwargs['hostid']
+ except:
+ turbogears.flash("Error: Invalid arguments.")
+ raise redirect("/adminview")
+
try:
location = Location.get(locationid)
except sqlobject.SQLObjectNotFound:
turbogears.flash("Error: Location %s does not exist" % locationid)
- raise redirect("/")
+ raise redirect("/adminview")
try:
- hostid = kwargs['hostid']
location.addHost(hostid)
except: # probably sqlite IntegrityError but we can't catch that for some reason...
turbogears.flash("Error:LocationHost already exists")
turbogears.flash("Location/Host created.")
- raise turbogears.redirect("/location/%s" % location.id)
+ raise redirect("/adminview")
@expose(template="mirrormanager.templates.boringlocationhostform")
- def delete(self, locationid, hostid, **kwargs):
+ def delete(self, **kwargs):
+ try:
+ locationid = kwargs['locationid']
+ hostid = kwargs['hostid']
+ except:
+ turbogears.flash("Error: Invalid arguments.")
+ raise redirect("/adminview")
+
try:
location = Location.get(locationid)
except sqlobject.SQLObjectNotFound:
@@ -1180,7 +1194,7 @@ class LocationHostController(controllers.Controller, identity.SecureResource, co
except:
turbogears.flash("Error: Location does not have Host %s" % hostid)
- raise turbogears.redirect("/location/%s" % location.id)
+ raise redirect("/adminview")
class Root(controllers.RootController):
@@ -1206,6 +1220,7 @@ class Root(controllers.RootController):
repository_redirect = RepositoryRedirectController()
country_continent_redirect = CountryContinentRedirectController()
location = LocationController()
+ locationhost = LocationHostController()
from mirrormanager.xmlrpc import XmlrpcController
xmlrpc = XmlrpcController()
diff --git a/server/mirrormanager/templates/adminview.kid b/server/mirrormanager/templates/adminview.kid
index 2669398..f1aa4b3 100644
--- a/server/mirrormanager/templates/adminview.kid
+++ b/server/mirrormanager/templates/adminview.kid
@@ -112,12 +112,16 @@ Nothing to see here, move along.
<h3>Locations <a href="${tg.url('/location/new')}">[Add]</a></h3>
<ul>
<li py:for="l in locations"> <a href="${tg.url('/location/'+str(l.id)+'/')}"><span py:replace="l.name">Location</span></a>
- <a href="${tg.url('/location/%s/delete' % (l.id))}">[Delete]</a>
+ <a href="${tg.url('/locationhost/new?locationid=' + str(l.id))}">[Add Host]</a>
+ <a href="${tg.url('/location/' + str(l.id) + '/delete')}">[Delete]</a>
<UL>
<li py:for="h in l.hosts">
<a href="${tg.url('/host/' + str(h.id))}"><span py:replace="h.name">Host name</span></a>
- <a href="${tg.url('/locationhost/%s/%s/delete' % (l.id,h.id))}">[Delete]</a>
+<?python
+deleteurl = tg.url('/locationhost/delete?locationid=' + str(l.id) + '&hostid=' + str(h.id))
+?>
+ <a href="${deleteurl}">[Delete the mapping]</a>
</li>
</UL>
</li>
diff --git a/server/mirrormanager/templates/host.kid b/server/mirrormanager/templates/host.kid
index bda135e..ee2bdf3 100644
--- a/server/mirrormanager/templates/host.kid
+++ b/server/mirrormanager/templates/host.kid
@@ -57,6 +57,14 @@ Last Crawled: ${values.lastCrawled} <a href="${tg.url('/crawler/'+str(values.id
</li>
</ul>
</div>
+
+<h3>Locations</h3>
+<UL>
+<li py:for="l in values.locations">
+ <span py:replace="l.name">Location Name</span>
+</li>
+</UL>
+
<h3>Countries Allowed</h3>
Some mirrors need to restrict themselves to serving only end
users from their country. If you're one of these, list the
13 years, 3 months
Branch 'v1.4' - server/mirrormanager.wsgi
by Matt Domsch
server/mirrormanager.wsgi | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
New commits:
commit a09828bac5cf90f36fd6b6444387b7b0d806f352
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 10:02:35 2011 -0600
fix mod_wsgi requirements
diff --git a/server/mirrormanager.wsgi b/server/mirrormanager.wsgi
index c056146..0390a1d 100755
--- a/server/mirrormanager.wsgi
+++ b/server/mirrormanager.wsgi
@@ -1,11 +1,16 @@
#!/usr/bin/python
-__requires__ = 'TurboGears[future]'
import sys
sys.stdout = sys.stderr
sys.path.append('/usr/share/mirrormanager/server/')
+if __name__ != "__main__": # mod_wsgi case
+ import __main__
+ __main__.__requires__ = ['TurboGears[future]']
+else:
+ __requires__ = ['TurboGears[future]']
+
import pkg_resources
-pkg_resources.require("TurboGears")
+pkg_resources.require("TurboGears<1.1")
pkg_resources.require("CherryPy<3.0")
import turbogears
13 years, 3 months
Branch 'v1.4' - 4 commits - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 88 +++++++-------
server/mirrormanager/model.py | 6
server/mirrormanager/templates/adminview.kid | 2
server/mirrormanager/templates/boringlocationhostform.kid | 16 ++
4 files changed, 68 insertions(+), 44 deletions(-)
New commits:
commit dedc01805578e879c0582c5fe35ec0e392c34bf7
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 00:21:54 2011 -0600
add boringlocationhostform
diff --git a/server/mirrormanager/templates/boringlocationhostform.kid b/server/mirrormanager/templates/boringlocationhostform.kid
new file mode 100644
index 0000000..edc92f4
--- /dev/null
+++ b/server/mirrormanager/templates/boringlocationhostform.kid
@@ -0,0 +1,16 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://purl.org/kid/ns#"
+ py:extends="'master.kid'">
+<head>
+<meta content="text/html; charset=utf-8" http-equiv="Content-Type" py:replace="''"/>
+<title>${page_title}</title>
+</head>
+<body>
+<p>
+Back to <a href="${tg.url('/location/' + str(location.id))}"><span
+py:replace="location.name">Location Name</span></a>
+</p>
+
+${form(value=values, action=tg.url(action))}
+</body>
+</html>
commit 54f349bbf877900ac4fc116306bea0e8de0ab98a
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 00:21:31 2011 -0600
fix adminview locationhost deletion
diff --git a/server/mirrormanager/templates/adminview.kid b/server/mirrormanager/templates/adminview.kid
index 12bc558..2669398 100644
--- a/server/mirrormanager/templates/adminview.kid
+++ b/server/mirrormanager/templates/adminview.kid
@@ -117,7 +117,7 @@ Nothing to see here, move along.
<UL>
<li py:for="h in l.hosts">
<a href="${tg.url('/host/' + str(h.id))}"><span py:replace="h.name">Host name</span></a>
- <a href="${tg.url('/location/%s/%s/delete' % (l.id,h.id))}">[Delete]</a>
+ <a href="${tg.url('/locationhost/%s/%s/delete' % (l.id,h.id))}">[Delete]</a>
</li>
</UL>
</li>
commit 19ca12147629b74cad373498dd93afa67924d8c2
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 00:21:09 2011 -0600
add destroySelf() to Location
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index bd56890..1fe2b97 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -888,6 +888,12 @@ class Location(SQLObject):
name = UnicodeCol(alternateID=True, length=UnicodeColKeyLength)
hosts = SQLRelatedJoin('Host')
+ def destroySelf(self):
+ """Cascade the delete operation"""
+ for h in self.hosts:
+ self.removeHost(h)
+ SQLObject.destroySelf(self)
+
class FileGroup(SQLObject):
class sqlmeta:
cacheValues = False
commit eba044f5c865b849f0e19c456d038fd1d72eed40
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 17 00:20:32 2011 -0600
minor fixups to controllers, Location stuff untested
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 2f432e7..b4e005b 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -170,14 +170,6 @@ class SiteController(controllers.Controller, identity.SecureResource, content):
site.del_downstream_site(dsite)
raise turbogears.redirect("/site/%s" % site.id)
- @expose(format='json')
- def search(self, name):
- """Called by the LocationHost controller AutoCompleteWidget."""
- if not name: return dict()
- results = Site.select(LIKE(Site.q.name, "%"+name+"%"))
- sites = [ s.name for s in results ]
- return dict(sites=sites)
-
##############################################
class SiteAdminFields(widgets.WidgetsList):
username = widgets.TextField(validator=validators.All(validators.UnicodeString,validators.NotEmpty, help_text="FAS username you wish to have be an admin for this site"))
@@ -309,7 +301,7 @@ class SiteToSiteController(controllers.Controller, identity.SecureResource, cont
##############################################
class HostFields(widgets.WidgetsList):
name = widgets.TextField(validator=validators.All(validators.UnicodeString,validators.NotEmpty), attrs=dict(size='30'), label="Host Name",
- help_text="* Name of server as seen by a public end user")
+ help_text="* FQDN of server as seen by a public end user")
admin_active = widgets.CheckBox("admin_active", default=True, help_text="Uncheck this box to temporarily disable this host, it will be removed from public listings.")
user_active = widgets.CheckBox("user_active", default=True, help_text="Uncheck this box to temporarily disable this host, it will be removed from public listings.")
country = widgets.TextField(validator=validators.All(validators.Regex(r'^[a-zA-Z][a-zA-Z]$'),validators.NotEmpty),
@@ -876,7 +868,7 @@ class CategoryFields(widgets.WidgetsList):
publiclist = widgets.CheckBox()
GeoDNSDomain = widgets.TextField(validator=validators.UnicodeString, attrs=dict(size='30'))
-category_form = Widgets.TableForm(fields=CategoryFields(), submit_text="Create Category")
+category_form = widgets.TableForm(fields=CategoryFields(), submit_text="Create Category")
class CategoryController(SimpleDbObjectController):
page_title="Category"
@@ -905,6 +897,7 @@ class CategoryController(SimpleDbObjectController):
class ArchFields(widgets.WidgetsList):
name = widgets.TextField(validator=validators.UnicodeString, attrs=dict(size='30'))
+ primaryArch = widgets.CheckBox(label="Primary Arch (not secondary)")
arch_form = widgets.TableForm(fields=ArchFields(), submit_text="Create Arch")
@@ -1140,45 +1133,54 @@ class LocationController(SimpleDbObjectController):
#########################################################
class LocationHostFields(widgets.WidgetsList):
locationName = widgets.Label(label="Location Name")
- siteName = widgets.AutoCompleteField(label="Site Name",
- search_controller=tg.url('/site/search'),
- search_param='name', result_name='sites',
- validator=validators.AutoCompleteValidator())
+ hostid = widgets.TextField(validator=validators.Int, label="Host ID")
location_host_form = widgets.TableForm(fields=LocationHostFields(), submit_text="Create Location / Host Mapping")
-class LocationHostController(SimpleDbObjectController):
- page_title = "Location / Host"
- myClass = Location
- url_prefix="locationhosts"
- form = location_host_form
-
- @expose(template="mirrormanager.templates.boringform")
- @validate(form=form)
- @error_handler(SimpleDbObjectController.new)
- def create(self, tg_errors=None, **kwargs):
- SimpleDbObjectController.create(self, tg_errors, **kwargs)
+class LocationHostController(controllers.Controller, identity.SecureResource, content):
+ require = identity.in_group(admin_group)
+
+ @expose(template="mirrormanager.templates.boringlocationhostform")
+ def new(self, locationid, **kwargs):
+ try:
+ location = Location.get(locationid)
+ except sqlobject.SQLObjectNotFound:
+ raise redirect("/adminview")
- @expose(template="mirrormanager.templates.boringform")
- @validate(form=form)
- @error_handler()
- def update(self, obj, tg_errors=None, **kwargs):
- return SimpleDbObjectController.update(self, obj, tg_errors, **kwargs)
+ submit_action = "/locationhost/create?locationid=%s" % locationid
+ return dict(form=location_host_form, values=None, action=submit_action, page_title="New Location / Host", location=location)
+
+ @expose(template="mirrormanager.templates.boringlocationhostform")
+ @error_handler(new)
+ @validate(form=location_host_form)
+ def create(self, locationid, **kwargs):
+ try:
+ location = Location.get(locationid)
+ except sqlobject.SQLObjectNotFound:
+ turbogears.flash("Error: Location %s does not exist" % locationid)
+ raise redirect("/")
- @expose(template="mirrormanager.templates.boringdeleteform")
- def delete(self, obj, tg_errors=None, **kwargs):
- confirmed = kwargs.get('confirmed', None)
- confirm_delete_form = widgets.TableForm(fields=ConfirmDeleteFields(), submit_text="Yes, really delete it!")
- if confirmed:
- turbogears.flash("%s has been deleted." % obj.name)
- obj.destroySelf()
- turbogears.redirect("/adminview")
- else:
- form = confirm_delete_form
- page_title = "Item Deletion"
- submit_action = "/%s/%s/delete?confirmed=1" % (self.url_prefix, obj.id)
- return dict(form=form, values=obj, action=submit_action, page_title=page_title)
+ try:
+ hostid = kwargs['hostid']
+ location.addHost(hostid)
+ except: # probably sqlite IntegrityError but we can't catch that for some reason...
+ turbogears.flash("Error:LocationHost already exists")
+ turbogears.flash("Location/Host created.")
+ raise turbogears.redirect("/location/%s" % location.id)
+ @expose(template="mirrormanager.templates.boringlocationhostform")
+ def delete(self, locationid, hostid, **kwargs):
+ try:
+ location = Location.get(locationid)
+ except sqlobject.SQLObjectNotFound:
+ turbogears.flash("Error: Location %s does not exist" % locationid)
+ raise redirect("/")
+ try:
+ location.removeHost(hostid)
+ except:
+ turbogears.flash("Error: Location does not have Host %s" % hostid)
+
+ raise turbogears.redirect("/location/%s" % location.id)
class Root(controllers.RootController):
13 years, 3 months
Branch 'v1.4' - server/errorcheck
by Matt Domsch
server/errorcheck | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
New commits:
commit 56bdab259519f962078bc8a0aabe7e21f8b0ff9b
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jan 13 23:26:04 2011 -0600
supress some messages, wordsmith others.
diff --git a/server/errorcheck b/server/errorcheck
index f1df0b4..046c62d 100755
--- a/server/errorcheck
+++ b/server/errorcheck
@@ -148,6 +148,12 @@ def open_geoip_databases():
def check_host(sr, host):
if host.is_private() and not options.private:
return
+
+ if not host.is_active():
+ msg = u"Host %s is not active - ignoring." % (host.name)
+ sr.messages.append(msg)
+ return
+
ips = name_to_ips(host.name)
if len(ips) == 0:
msg = u"Host name %s not found in DNS. Please use a FQDN for your host name. You will also need to make the corresponding change in your report_mirror.conf file." % (host.name)
@@ -195,16 +201,18 @@ def check_host(sr, host):
msg = u"Host %s has no content Categories. Please add Host Categories to the content your mirror carries." % (host.name)
sr.messages.append(msg)
+ always_up2date = False
for hc in host.categories:
if len(hc.urls) == 0:
msg = u"Host %s Category %s has no URLs. Please add a URL to the content your mirror carries, or if not carried, delete the Category." % (host.name, hc.category.name)
sr.messages.append(msg)
- if len(hc.dirs) == 0:
+ if hc.always_up2date: always_up2date = True
+ if len(hc.dirs) == 0 and not hc.always_up2date:
msg = u"Host %s Category %s has no up-to-date directories. Consult the crawler logs and check your report_mirror.conf." % (host.name, hc.category.name)
sr.messages.append(msg)
- if host.lastCheckedIn is None:
+ if host.lastCheckedIn is None and not always_up2date:
msg = u"Host %s has never checked in with report_mirror." % (host.name)
if host.is_private():
msg += u" Private mirrors must run report_mirror."
@@ -274,6 +282,9 @@ def setup_netblocks(netblocks_file):
def check_site(site):
if site.private and not options.private:
return
+ if not site.admin_active or not site.user_active:
+ return
+
sr = SiteResult(site)
sr.admin_emails = users_to_emails(site_admins(site))
13 years, 3 months
Branch 'v1.4' - 3 commits - server/errorcheck server/mirrormanager
by Matt Domsch
server/errorcheck | 354 ++++++++++++++++++++++++++++++++++++
server/mirrormanager/controllers.py | 1
server/mirrormanager/dns.py | 3
3 files changed, 357 insertions(+), 1 deletion(-)
New commits:
commit fd482936938aae373dddcde81541608e02cb1f9e
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jan 13 21:54:06 2011 -0600
add errorcheck CLI
diff --git a/server/errorcheck b/server/errorcheck
new file mode 100755
index 0000000..f1df0b4
--- /dev/null
+++ b/server/errorcheck
@@ -0,0 +1,354 @@
+#!/usr/bin/python
+#
+# Copyright 2011 Dell, Inc.
+# by Matt Domsch <Matt_Domsch(a)dell.com>
+# MIT/X11 license
+
+
+__requires__ = 'TurboGears[future]'
+import pkg_resources
+pkg_resources.require("TurboGears")
+
+from sqlobject import *
+import sys
+import datetime as dt
+import codecs
+import radix
+import turbogears
+from mirrormanager.model import *
+from mirrormanager.lib import manage_pidfile, remove_pidfile
+from optparse import OptionParser
+import dns.resolver
+from IPy import IP
+from fedora.client.fas2 import AccountSystem
+
+
+from turbogears.database import PackageHub
+from turbogears import config
+hub = __connection__ = None
+
+
+pidfile='/var/run/mirrormanager/errorcheck.pid'
+internet2_netblocks_file = '/var/lib/mirrormanager/i2_netblocks.txt'
+global_netblocks_file = '/var/lib/mirrormanager/global_netblocks.txt'
+internet2_tree = None
+global_tree = None
+one_day = dt.timedelta(days=1)
+outfile = None
+
+class SiteResult(object):
+ def __init__(self, site):
+ self.site = site
+ self.messages = []
+ self.admin_emails = []
+
+
+def name_to_ips(name):
+ namestotry=[]
+ result=set()
+ records = []
+
+ try:
+ records = dns.resolver.query(name, 'CNAME')
+ except:
+ pass
+ if len(records) > 0:
+ for rdata in records:
+ namestotry.append(rdata.to_text())
+ else:
+ namestotry = [name]
+
+ records = []
+ recordtypes=('A', 'AAAA')
+ for name in namestotry:
+ for r in recordtypes:
+ try:
+ records = dns.resolver.query(name, r)
+ except:
+ pass
+ for rdata in records:
+ try:
+ ip = IP(str(rdata.to_text()))
+ except:
+ raise
+ result.add(ip)
+ return result
+
+def convert_6to4_v4(ip):
+ all_6to4 = ip('2002::/16')
+ if ip.version() != 6 or ip not in all_6to4:
+ return None
+ parts=ip.strnormal().split(':')
+
+ ab = int(parts[1],16)
+ a = (ab >> 8) & 0xff
+ b = ab & 0xff
+ cd = int(parts[2],16)
+ c = (cd >> 8) & 0xff
+ d = cd & 0xff
+
+ v4addr = '%d.%d.%d.%d' % (a,b,c,d)
+ return ip(v4addr)
+
+def convert_teredo_v4(ip):
+ teredo_std = ip('2001::/32')
+ teredo_xp = ip('3ffe:831f::/32')
+ if ip.version() != 6 or (ip not in teredo_std and ip not in teredo_xp):
+ return None
+ parts=ip.strnormal().split(':')
+
+ ab = int(parts[6],16)
+ a = ((ab >> 8) & 0xff) ^ 0xff
+ b = (ab & 0xff) ^ 0xff
+ cd = int(parts[7],16)
+ c = ((cd >> 8) & 0xff) ^ 0xff
+ d = (cd & 0xff) ^ 0xff
+
+ v4addr = '%d.%d.%d.%d' % (a,b,c,d)
+ return ip(v4addr)
+
+gipv4 = None
+gipv6 = None
+
+def lookup_country(ip):
+ clientcountry = None
+ # attempt ipv6, then ipv6 6to4 as ipv4, then teredo, then ipv4
+ try:
+ if ip.version() == 6:
+ if gipv6 is not None:
+ clientcountry = gipv6.country_code_by_addr_v6(ip.strnormal())
+ if clientcountry is None:
+ # try the ipv6-to-ipv4 translation schemes
+ for scheme in (convert_6to4_v4, convert_teredo_v4):
+ result = scheme(ip)
+ if result is not None:
+ ip = result
+ break
+ if ip.version() == 4 and gipv4 is not None:
+ clientcountry = gipv4.country_code_by_addr(ip.strnormal())
+ except:
+ pass
+
+ return clientcountry
+
+
+def open_geoip_databases():
+ global gipv4
+ global gipv6
+ try:
+ gipv4 = GeoIP.open("/usr/share/geoip/geoip.dat", GeoIP.geoip_standard)
+ except:
+ gipv4=None
+ try:
+ gipv6 = GeoIP.open("/usr/share/geoip/geoipv6.dat", GeoIP.geoip_standard)
+ except:
+ gipv6=None
+
+
+def check_host(sr, host):
+ if host.is_private() and not options.private:
+ return
+ ips = name_to_ips(host.name)
+ if len(ips) == 0:
+ msg = u"Host name %s not found in DNS. Please use a FQDN for your host name. You will also need to make the corresponding change in your report_mirror.conf file." % (host.name)
+ sr.messages.append(msg)
+ else:
+ if host.internet2:
+ really_on_i2 = False
+ for ip in ips:
+ asn = lookup_ip_asn(internet2_tree, ip)
+ if asn is not None:
+ really_on_i2 = True
+ break;
+ if not really_on_i2:
+ msg = u"Host %s claims to be on Internet2, but is not in the Internet2 BGP table." % (host.name)
+ sr.messages.append(msg)
+
+ if host.country is None:
+ msg = u"Host %s does not list the country it is in." % (host.name)
+ sr.messages.append(msg)
+ else:
+ really_in_country = False
+ country = None
+ for ip in ips:
+ country = lookup_country(ip)
+ if country is not None and country == host.country:
+ really_in_country = True
+ break
+ if country and not really_in_country:
+ msg = u"Host %s claims to be in country %s, but GeoIP reports them in %s" % (host.name, host.country, country)
+ sr.messages.append(msg)
+
+
+ if host.asn_clients:
+ if host.asn is None:
+ msg = u"Host %s has set asn_clients, but has not set it's ASN." % (host.name)
+ sr.messages.append(msg)
+ else:
+ for ip in ips:
+ asn = lookup_ip_asn(global_tree, ip)
+ if host.asn != asn:
+ msg = u"Host %s has set ASN=%s, but appears in ASN %s in the global BGP table." % (host.name, host.asn, asn)
+ sr.messages.append(msg)
+
+ if len(host.categories) == 0:
+ msg = u"Host %s has no content Categories. Please add Host Categories to the content your mirror carries." % (host.name)
+ sr.messages.append(msg)
+
+ for hc in host.categories:
+ if len(hc.urls) == 0:
+ msg = u"Host %s Category %s has no URLs. Please add a URL to the content your mirror carries, or if not carried, delete the Category." % (host.name, hc.category.name)
+ sr.messages.append(msg)
+
+ if len(hc.dirs) == 0:
+ msg = u"Host %s Category %s has no up-to-date directories. Consult the crawler logs and check your report_mirror.conf." % (host.name, hc.category.name)
+ sr.messages.append(msg)
+
+ if host.lastCheckedIn is None:
+ msg = u"Host %s has never checked in with report_mirror." % (host.name)
+ if host.is_private():
+ msg += u" Private mirrors must run report_mirror."
+ else:
+ msg += u" Public mirrors are encouraged to run report_mirror."
+ sr.messages.append(msg)
+ else:
+ if host.lastCheckedIn < (dt.datetime.utcnow() - one_day):
+ msg = u"Host %s has not checked in with report_mirror in 24 hours." % (host.name)
+ if host.is_private():
+ msg += u" Private mirrors must run report_mirror."
+ else:
+ msg += u" Public mirrors are encouraged to run report_mirror."
+ sr.messages.append(msg)
+
+AS = None
+
+def users_to_emails(userlist):
+ global options
+ global AS
+ result = []
+ if AS is None:
+ if turbogears.config.get('identity.provider') == 'jsonfas2':
+ from fedora.client.fas2 import AccountSystem
+ AS = AccountSystem(username=options.fasuser, password=options.faspassword, cache_session=True)
+ for user in userlist:
+ person = AS.person_by_username(user)
+ if person:
+ result.append(person['email'])
+ return result
+
+def site_admins(site):
+ return [sa.username for sa in site.admins]
+
+def lookup_ip_asn(tree, ip):
+ """ @t is a radix tree
+ @ip is an IPy.IP object which may be contained in an entry in l
+ """
+ node = tree.search_best(str(ip))
+ if node is None:
+ return None
+ return node.data['asn']
+
+def setup_netblocks(netblocks_file):
+
+ tree = radix.Radix()
+ if netblocks_file is not None:
+ try:
+ f = open(netblocks_file, 'r')
+ except:
+ return tree
+ for l in f:
+ try:
+ s = l.split()
+ start, mask = s[0].split('/')
+ mask = int(mask)
+ if mask == 0: continue
+ asn = int(s[1])
+ node = tree.add(s[0])
+ node.data['asn'] = asn
+ except:
+ pass
+ f.close()
+
+ return tree
+
+def check_site(site):
+ if site.private and not options.private:
+ return
+ sr = SiteResult(site)
+ sr.admin_emails = users_to_emails(site_admins(site))
+
+
+ if len(site.hosts) == 0:
+ msg = u"Site %s has no Hosts. Please add one or more Hosts to your Site entry." % (site.name)
+ sr.messages.append(msg)
+
+ for host in site.hosts:
+ check_host(sr, host)
+
+ if len(sr.messages) > 0:
+ outfile.write("==== Site: %s ====\n" % (site.name))
+ for msg in sr.messages:
+ outfile.write("\t%s\n" % msg)
+ outfile.write("\n")
+
+def doit():
+ global internet2_tree
+ global global_tree
+ global options
+ internet2_tree = setup_netblocks(internet2_netblocks_file)
+ global_tree = setup_netblocks(global_netblocks_file)
+
+ for site in Site.select():
+ check_site(site)
+
+options = None
+
+def main():
+ global options
+ parser = OptionParser(usage=sys.argv[0] + " [options]")
+ parser.add_option("-c", "--config",
+ dest="config", default='dev.cfg',
+ help="TurboGears config file to use")
+ parser.add_option("--fasuser",
+ dest="fasuser", default=None,
+ help="Fedora Account System username for queries")
+ parser.add_option("--faspassword",
+ dest="faspassword", default=None,
+ help="Fedora Account System password for queries")
+ parser.add_option("--private",
+ dest="private", default=None,
+ help="include private mirrors")
+ parser.add_option("-o", "--output",
+ dest="outfile", default=None,
+ help="FILE to write to")
+
+
+ (options, args) = parser.parse_args()
+
+ turbogears.update_config(configfile=options.config,
+ modulename="mirrormanager.config")
+
+ if manage_pidfile(pidfile):
+ print "another instance is running, try again later."
+ sys.exit(1)
+
+ if options.outfile is None:
+ print "Must specify an output file.\n"
+ sys.exit(1)
+
+ global outfile
+ outfile = codecs.open(options.outfile, encoding='utf-8', mode='w')
+
+ global hub
+ global __connection__
+ hub = PackageHub("mirrormanager")
+ __connection__ = hub
+
+ doit()
+ outfile.close()
+ remove_pidfile(pidfile)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
+
commit b2257f943b1af267cef591d3acb8dbf9ff959cc7
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 5 22:48:37 2011 -0600
dns: move default output directory to a module variable
diff --git a/server/mirrormanager/dns.py b/server/mirrormanager/dns.py
index 89318b3..239e18d 100644
--- a/server/mirrormanager/dns.py
+++ b/server/mirrormanager/dns.py
@@ -7,6 +7,7 @@ from dns.rdataclass import *
from dns.rdatatype import *
default_ttl = 3600
+default_output_dir = '/var/lib/mirrormanager/dns'
def domainTemplateToName(domain, country):
return re.sub(u'CC', country, domain)
@@ -36,7 +37,7 @@ def generateZoneFiles(directory):
z['zone'].to_file(os.path.join(directory, u'zone-' + z['name']))
def writeZoneFiles():
- output_dir = '/var/lib/mirrormanager/dns'
+ output_dir = default_output_dir
try:
os.makedirs(output_dir)
except OSError, e:
commit 5b29597b52a63a6cc47ceb8d41b0fbeb93e633af
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 5 22:48:15 2011 -0600
controllers: expose Category GeoDNSDomain field
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 326c5eb..2f432e7 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -874,6 +874,7 @@ class CategoryFields(widgets.WidgetsList):
canonicalhost = widgets.TextField(validator=validators.UnicodeString, attrs=dict(size='30'))
topdirPath = widgets.TextField(validator=validators.UnicodeString, attrs=dict(size='30'))
publiclist = widgets.CheckBox()
+ GeoDNSDomain = widgets.TextField(validator=validators.UnicodeString, attrs=dict(size='30'))
category_form = Widgets.TableForm(fields=CategoryFields(), submit_text="Create Category")
13 years, 3 months
Branch 'v1.4' - 4 commits - Makefile server/mirrormanager server/refresh_mirrorlist_cache server/setup.py
by Matt Domsch
Makefile | 1
server/mirrormanager/controllers.py | 1
server/mirrormanager/dns.py | 48 ++++++++++++++++++++++++++
server/mirrormanager/templates/publiclist.kid | 5 ++
server/refresh_mirrorlist_cache | 4 +-
server/setup.py | 2 +
6 files changed, 58 insertions(+), 3 deletions(-)
New commits:
commit cea4aabcd9adbdbb61aa370af9436455d84f43cb
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 5 00:12:23 2011 -0600
initial attempt to export data as DNS zones
diff --git a/server/mirrormanager/dns.py b/server/mirrormanager/dns.py
new file mode 100644
index 0000000..89318b3
--- /dev/null
+++ b/server/mirrormanager/dns.py
@@ -0,0 +1,48 @@
+import os
+import re
+from mirrormanager.model import Category, HostCategory
+import dns.zone
+from dns.exception import DNSException
+from dns.rdataclass import *
+from dns.rdatatype import *
+
+default_ttl = 3600
+
+def domainTemplateToName(domain, country):
+ return re.sub(u'CC', country, domain)
+
+def generateZoneFiles(directory):
+ zones = {}
+
+ for c in Category.select():
+ if c.GeoDNSDomain is None: continue
+ hcs = HostCategory.selectBy(category=c)
+ hosts = [ hc.host for hc in hcs if hc.host.dnsCountryHost ]
+ for h in hosts:
+ if h.country in zones:
+ z = zones[h.country]['zone']
+ else:
+ domainname = domainTemplateToName(c.GeoDNSDomain, h.country)
+ z = dns.zone.Zone(domainname)
+ zones[h.country] = {}
+ zones[h.country]['zone'] = z
+ zones[h.country]['name'] = domainname
+
+ rdataset = z.find_rdataset('@', rdtype=CNAME, create=True)
+ rdata = dns.rdtypes.ANY.CNAME.CNAME(IN, CNAME, h.name)
+ rdataset.add(rdata, ttl=default_ttl)
+
+ for country, z in zones.iteritems():
+ z['zone'].to_file(os.path.join(directory, u'zone-' + z['name']))
+
+def writeZoneFiles():
+ output_dir = '/var/lib/mirrormanager/dns'
+ try:
+ os.makedirs(output_dir)
+ except OSError, e:
+ if e.errno != errno.EEXIST:
+ raise
+ generateZoneFiles(output_dir)
+
+
+__all__ = ['writeZoneFiles']
diff --git a/server/refresh_mirrorlist_cache b/server/refresh_mirrorlist_cache
index 7b17f08..884d69a 100755
--- a/server/refresh_mirrorlist_cache
+++ b/server/refresh_mirrorlist_cache
@@ -9,6 +9,7 @@ import turbogears
from mirrormanager.model import *
from mirrormanager.mirrorlist import populate_all_caches, dump_caches
from optparse import OptionParser
+import mirrormanager.dns
from turbogears.database import PackageHub
hub = __connection__ = None
@@ -16,8 +17,7 @@ hub = __connection__ = None
def doit():
populate_all_caches()
dump_caches()
-
-
+ mirrormanager.dns.writeZoneFiles()
def main():
parser = OptionParser(usage=sys.argv[0] + " [options]")
commit 530d74797036b683642d06acab691cf91e04048f
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 5 00:11:14 2011 -0600
setup.py: require TurboGears, due to multiple versions of CherryPy
diff --git a/server/setup.py b/server/setup.py
index 50e3caa..14ea030 100644
--- a/server/setup.py
+++ b/server/setup.py
@@ -1,3 +1,5 @@
+__requires__ = ['TurboGears']
+
from setuptools import setup, find_packages
from turbogears.finddata import find_package_data
commit c8739ab598fcdda4c5a6abb8a8afbf5eb53095e9
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 5 00:10:42 2011 -0600
Makefile: create /etc/tmpfiles.d
diff --git a/Makefile b/Makefile
index 383808b..48cb550 100644
--- a/Makefile
+++ b/Makefile
@@ -84,6 +84,7 @@ install-server:
mkdir -p -m 0755 $(DESTDIR)/etc/httpd/conf.d
mkdir -p -m 0755 $(DESTDIR)/etc/mirrormanager
mkdir -p -m 0755 $(DESTDIR)/etc/rpmlint
+ mkdir -p -m 0755 $(DESTDIR)/etc/tmpfiles.d
# server/
cp -ra server/ $(DESTDIR)/usr/share/mirrormanager
install -m 0644 server/apache/mirrormanager.conf $(DESTDIR)/etc/httpd/conf.d
commit c479db834bb55b3022bfdb16a7b7589a24f23dfe
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Jan 4 22:16:30 2011 -0600
use Product.publiclist
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index fc8fc2e..326c5eb 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -948,6 +948,7 @@ class EmbargoedCountryController(SimpleDbObjectController):
class ProductFields(widgets.WidgetsList):
name = widgets.TextField(validator=validators.All(validators.UnicodeString, validators.NotEmpty),
attrs=dict(size='30'))
+ publiclist = widgets.CheckBox(label="display in the publiclist chooser", default=True)
product_form = widgets.TableForm(fields=ProductFields(), submit_text="Create Product")
diff --git a/server/mirrormanager/templates/publiclist.kid b/server/mirrormanager/templates/publiclist.kid
index 9db180d..91112d1 100644
--- a/server/mirrormanager/templates/publiclist.kid
+++ b/server/mirrormanager/templates/publiclist.kid
@@ -5,7 +5,10 @@
<table cellspacing="0" class="compact altrows minilist">
<caption><strong>Mirror list filter</strong></caption>
<tr><th>Project</th><th>Version</th><th colspan='0'>Architecture</th></tr>
- <div py:for="p in products">
+<?python
+displayproducts = [p for p in products if p.publiclist]
+?>
+ <div py:for="p in displayproducts">
<?python
displayversions = [v for v in p.versions if v.display]
?>
13 years, 4 months