server/errorcheck | 354 ++++++++++++++++++++++++++++++++++++
server/mirrormanager/controllers.py | 1
server/mirrormanager/dns.py | 3
3 files changed, 357 insertions(+), 1 deletion(-)
New commits:
commit fd482936938aae373dddcde81541608e02cb1f9e
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jan 13 21:54:06 2011 -0600
add errorcheck CLI
diff --git a/server/errorcheck b/server/errorcheck
new file mode 100755
index 0000000..f1df0b4
--- /dev/null
+++ b/server/errorcheck
@@ -0,0 +1,354 @@
+#!/usr/bin/python
+#
+# Copyright 2011 Dell, Inc.
+# by Matt Domsch <Matt_Domsch(a)dell.com>
+# MIT/X11 license
+
+
+__requires__ = 'TurboGears[future]'
+import pkg_resources
+pkg_resources.require("TurboGears")
+
+from sqlobject import *
+import sys
+import datetime as dt
+import codecs
+import radix
+import turbogears
+from mirrormanager.model import *
+from mirrormanager.lib import manage_pidfile, remove_pidfile
+from optparse import OptionParser
+import dns.resolver
+from IPy import IP
+from fedora.client.fas2 import AccountSystem
+
+
+from turbogears.database import PackageHub
+from turbogears import config
+hub = __connection__ = None
+
+
+pidfile='/var/run/mirrormanager/errorcheck.pid'
+internet2_netblocks_file = '/var/lib/mirrormanager/i2_netblocks.txt'
+global_netblocks_file = '/var/lib/mirrormanager/global_netblocks.txt'
+internet2_tree = None
+global_tree = None
+one_day = dt.timedelta(days=1)
+outfile = None
+
+class SiteResult(object):
+ def __init__(self, site):
+ self.site = site
+ self.messages = []
+ self.admin_emails = []
+
+
+def name_to_ips(name):
+ namestotry=[]
+ result=set()
+ records = []
+
+ try:
+ records = dns.resolver.query(name, 'CNAME')
+ except:
+ pass
+ if len(records) > 0:
+ for rdata in records:
+ namestotry.append(rdata.to_text())
+ else:
+ namestotry = [name]
+
+ records = []
+ recordtypes=('A', 'AAAA')
+ for name in namestotry:
+ for r in recordtypes:
+ try:
+ records = dns.resolver.query(name, r)
+ except:
+ pass
+ for rdata in records:
+ try:
+ ip = IP(str(rdata.to_text()))
+ except:
+ raise
+ result.add(ip)
+ return result
+
+def convert_6to4_v4(ip):
+ all_6to4 = ip('2002::/16')
+ if ip.version() != 6 or ip not in all_6to4:
+ return None
+ parts=ip.strnormal().split(':')
+
+ ab = int(parts[1],16)
+ a = (ab >> 8) & 0xff
+ b = ab & 0xff
+ cd = int(parts[2],16)
+ c = (cd >> 8) & 0xff
+ d = cd & 0xff
+
+ v4addr = '%d.%d.%d.%d' % (a,b,c,d)
+ return ip(v4addr)
+
+def convert_teredo_v4(ip):
+ teredo_std = ip('2001::/32')
+ teredo_xp = ip('3ffe:831f::/32')
+ if ip.version() != 6 or (ip not in teredo_std and ip not in teredo_xp):
+ return None
+ parts=ip.strnormal().split(':')
+
+ ab = int(parts[6],16)
+ a = ((ab >> 8) & 0xff) ^ 0xff
+ b = (ab & 0xff) ^ 0xff
+ cd = int(parts[7],16)
+ c = ((cd >> 8) & 0xff) ^ 0xff
+ d = (cd & 0xff) ^ 0xff
+
+ v4addr = '%d.%d.%d.%d' % (a,b,c,d)
+ return ip(v4addr)
+
+gipv4 = None
+gipv6 = None
+
+def lookup_country(ip):
+ clientcountry = None
+ # attempt ipv6, then ipv6 6to4 as ipv4, then teredo, then ipv4
+ try:
+ if ip.version() == 6:
+ if gipv6 is not None:
+ clientcountry = gipv6.country_code_by_addr_v6(ip.strnormal())
+ if clientcountry is None:
+ # try the ipv6-to-ipv4 translation schemes
+ for scheme in (convert_6to4_v4, convert_teredo_v4):
+ result = scheme(ip)
+ if result is not None:
+ ip = result
+ break
+ if ip.version() == 4 and gipv4 is not None:
+ clientcountry = gipv4.country_code_by_addr(ip.strnormal())
+ except:
+ pass
+
+ return clientcountry
+
+
+def open_geoip_databases():
+ global gipv4
+ global gipv6
+ try:
+ gipv4 = GeoIP.open("/usr/share/geoip/geoip.dat", GeoIP.geoip_standard)
+ except:
+ gipv4=None
+ try:
+ gipv6 = GeoIP.open("/usr/share/geoip/geoipv6.dat",
GeoIP.geoip_standard)
+ except:
+ gipv6=None
+
+
+def check_host(sr, host):
+ if host.is_private() and not options.private:
+ return
+ ips = name_to_ips(host.name)
+ if len(ips) == 0:
+ msg = u"Host name %s not found in DNS. Please use a FQDN for your host
name. You will also need to make the corresponding change in your report_mirror.conf
file." % (host.name)
+ sr.messages.append(msg)
+ else:
+ if host.internet2:
+ really_on_i2 = False
+ for ip in ips:
+ asn = lookup_ip_asn(internet2_tree, ip)
+ if asn is not None:
+ really_on_i2 = True
+ break;
+ if not really_on_i2:
+ msg = u"Host %s claims to be on Internet2, but is not in the
Internet2 BGP table." % (host.name)
+ sr.messages.append(msg)
+
+ if host.country is None:
+ msg = u"Host %s does not list the country it is in." % (host.name)
+ sr.messages.append(msg)
+ else:
+ really_in_country = False
+ country = None
+ for ip in ips:
+ country = lookup_country(ip)
+ if country is not None and country == host.country:
+ really_in_country = True
+ break
+ if country and not really_in_country:
+ msg = u"Host %s claims to be in country %s, but GeoIP reports them in
%s" % (host.name, host.country, country)
+ sr.messages.append(msg)
+
+
+ if host.asn_clients:
+ if host.asn is None:
+ msg = u"Host %s has set asn_clients, but has not set it's ASN."
% (host.name)
+ sr.messages.append(msg)
+ else:
+ for ip in ips:
+ asn = lookup_ip_asn(global_tree, ip)
+ if host.asn != asn:
+ msg = u"Host %s has set ASN=%s, but appears in ASN %s in the
global BGP table." % (host.name, host.asn, asn)
+ sr.messages.append(msg)
+
+ if len(host.categories) == 0:
+ msg = u"Host %s has no content Categories. Please add Host Categories to
the content your mirror carries." % (host.name)
+ sr.messages.append(msg)
+
+ for hc in host.categories:
+ if len(hc.urls) == 0:
+ msg = u"Host %s Category %s has no URLs. Please add a URL to the
content your mirror carries, or if not carried, delete the Category." % (host.name,
hc.category.name)
+ sr.messages.append(msg)
+
+ if len(hc.dirs) == 0:
+ msg = u"Host %s Category %s has no up-to-date directories. Consult the
crawler logs and check your report_mirror.conf." % (host.name, hc.category.name)
+ sr.messages.append(msg)
+
+ if host.lastCheckedIn is None:
+ msg = u"Host %s has never checked in with report_mirror." %
(host.name)
+ if host.is_private():
+ msg += u" Private mirrors must run report_mirror."
+ else:
+ msg += u" Public mirrors are encouraged to run report_mirror."
+ sr.messages.append(msg)
+ else:
+ if host.lastCheckedIn < (dt.datetime.utcnow() - one_day):
+ msg = u"Host %s has not checked in with report_mirror in 24 hours."
% (host.name)
+ if host.is_private():
+ msg += u" Private mirrors must run report_mirror."
+ else:
+ msg += u" Public mirrors are encouraged to run report_mirror."
+ sr.messages.append(msg)
+
+AS = None
+
+def users_to_emails(userlist):
+ global options
+ global AS
+ result = []
+ if AS is None:
+ if turbogears.config.get('identity.provider') == 'jsonfas2':
+ from fedora.client.fas2 import AccountSystem
+ AS = AccountSystem(username=options.fasuser, password=options.faspassword,
cache_session=True)
+ for user in userlist:
+ person = AS.person_by_username(user)
+ if person:
+ result.append(person['email'])
+ return result
+
+def site_admins(site):
+ return [sa.username for sa in site.admins]
+
+def lookup_ip_asn(tree, ip):
+ """ @t is a radix tree
+ @ip is an IPy.IP object which may be contained in an entry in l
+ """
+ node = tree.search_best(str(ip))
+ if node is None:
+ return None
+ return node.data['asn']
+
+def setup_netblocks(netblocks_file):
+
+ tree = radix.Radix()
+ if netblocks_file is not None:
+ try:
+ f = open(netblocks_file, 'r')
+ except:
+ return tree
+ for l in f:
+ try:
+ s = l.split()
+ start, mask = s[0].split('/')
+ mask = int(mask)
+ if mask == 0: continue
+ asn = int(s[1])
+ node = tree.add(s[0])
+ node.data['asn'] = asn
+ except:
+ pass
+ f.close()
+
+ return tree
+
+def check_site(site):
+ if site.private and not options.private:
+ return
+ sr = SiteResult(site)
+ sr.admin_emails = users_to_emails(site_admins(site))
+
+
+ if len(site.hosts) == 0:
+ msg = u"Site %s has no Hosts. Please add one or more Hosts to your Site
entry." % (site.name)
+ sr.messages.append(msg)
+
+ for host in site.hosts:
+ check_host(sr, host)
+
+ if len(sr.messages) > 0:
+ outfile.write("==== Site: %s ====\n" % (site.name))
+ for msg in sr.messages:
+ outfile.write("\t%s\n" % msg)
+ outfile.write("\n")
+
+def doit():
+ global internet2_tree
+ global global_tree
+ global options
+ internet2_tree = setup_netblocks(internet2_netblocks_file)
+ global_tree = setup_netblocks(global_netblocks_file)
+
+ for site in Site.select():
+ check_site(site)
+
+options = None
+
+def main():
+ global options
+ parser = OptionParser(usage=sys.argv[0] + " [options]")
+ parser.add_option("-c", "--config",
+ dest="config", default='dev.cfg',
+ help="TurboGears config file to use")
+ parser.add_option("--fasuser",
+ dest="fasuser", default=None,
+ help="Fedora Account System username for queries")
+ parser.add_option("--faspassword",
+ dest="faspassword", default=None,
+ help="Fedora Account System password for queries")
+ parser.add_option("--private",
+ dest="private", default=None,
+ help="include private mirrors")
+ parser.add_option("-o", "--output",
+ dest="outfile", default=None,
+ help="FILE to write to")
+
+
+ (options, args) = parser.parse_args()
+
+ turbogears.update_config(configfile=options.config,
+ modulename="mirrormanager.config")
+
+ if manage_pidfile(pidfile):
+ print "another instance is running, try again later."
+ sys.exit(1)
+
+ if options.outfile is None:
+ print "Must specify an output file.\n"
+ sys.exit(1)
+
+ global outfile
+ outfile = codecs.open(options.outfile, encoding='utf-8', mode='w')
+
+ global hub
+ global __connection__
+ hub = PackageHub("mirrormanager")
+ __connection__ = hub
+
+ doit()
+ outfile.close()
+ remove_pidfile(pidfile)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
+
commit b2257f943b1af267cef591d3acb8dbf9ff959cc7
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 5 22:48:37 2011 -0600
dns: move default output directory to a module variable
diff --git a/server/mirrormanager/dns.py b/server/mirrormanager/dns.py
index 89318b3..239e18d 100644
--- a/server/mirrormanager/dns.py
+++ b/server/mirrormanager/dns.py
@@ -7,6 +7,7 @@ from dns.rdataclass import *
from dns.rdatatype import *
default_ttl = 3600
+default_output_dir = '/var/lib/mirrormanager/dns'
def domainTemplateToName(domain, country):
return re.sub(u'CC', country, domain)
@@ -36,7 +37,7 @@ def generateZoneFiles(directory):
z['zone'].to_file(os.path.join(directory, u'zone-' +
z['name']))
def writeZoneFiles():
- output_dir = '/var/lib/mirrormanager/dns'
+ output_dir = default_output_dir
try:
os.makedirs(output_dir)
except OSError, e:
commit 5b29597b52a63a6cc47ceb8d41b0fbeb93e633af
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 5 22:48:15 2011 -0600
controllers: expose Category GeoDNSDomain field
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 326c5eb..2f432e7 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -874,6 +874,7 @@ class CategoryFields(widgets.WidgetsList):
canonicalhost = widgets.TextField(validator=validators.UnicodeString,
attrs=dict(size='30'))
topdirPath = widgets.TextField(validator=validators.UnicodeString,
attrs=dict(size='30'))
publiclist = widgets.CheckBox()
+ GeoDNSDomain = widgets.TextField(validator=validators.UnicodeString,
attrs=dict(size='30'))
category_form = Widgets.TableForm(fields=CategoryFields(), submit_text="Create
Category")