Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 6 ++++--
server/mirrormanager/model.py | 2 ++
server/mirrormanager/schema_updates/__init__.py | 10 +++++++++-
3 files changed, 15 insertions(+), 3 deletions(-)
New commits:
commit 3bddb76c6014cb2089ba6a9756701444c9eb4547
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Jul 30 14:40:53 2011 -0500
SCHEMA CHANGE: add name fields to HostNetblock and HostPeerAsn tables
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 7636786..886d39d 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -642,6 +642,7 @@ class HostAclIPController(HostListitemController):
class HostNetblockFields(widgets.WidgetsList):
netblock = widgets.TextField(validator=validators.All(validators.UnicodeString,validators.NotEmpty))
+ name = widgets.TextField(validator=validators.UnicodeString)
host_netblock_form = widgets.TableForm(fields=HostNetblockFields(),
submit_text="Create Host Netblock")
@@ -673,11 +674,12 @@ class HostNetblockController(HostListitemController):
# also accept DNS hostnames
pass
- HostNetblock(host=host, netblock=kwargs['netblock'])
+ HostNetblock(host=host, netblock=kwargs['netblock'], name=kwargs['name'])
class HostPeerAsnFields(widgets.WidgetsList):
asn = widgets.TextField(validator=validators.All(validators.Int,validators.NotEmpty))
+ name = widgets.TextField(validator=validators.UnicodeString)
host_peer_asn_form = widgets.TableForm(fields=HostPeerAsnFields(),
submit_text="Create Host Peer ASN")
@@ -693,7 +695,7 @@ class HostPeerAsnController(HostListitemController):
return dict(values=v, host=v.host)
def do_create(self, host, kwargs):
- HostPeerAsn(host=host, asn=kwargs['asn'])
+ HostPeerAsn(host=host, asn=kwargs['asn'], name=kwargs['name'])
class HostCountryAllowedFields(widgets.WidgetsList):
country = widgets.TextField(validator=validators.Regex(r'^[a-zA-Z][a-zA-Z]$'),
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 6ca8fc9..67c08da 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -577,6 +577,7 @@ class HostNetblock(SQLObject):
cacheValues = False
host = ForeignKey('Host')
netblock = StringCol()
+ name = UnicodeCol()
def my_site(self):
return self.host.my_site()
@@ -586,6 +587,7 @@ class HostPeerAsn(SQLObject):
cacheValues = False
host = ForeignKey('Host')
asn = IntCol()
+ name = UnicodeCol()
idx = DatabaseIndex('host', 'asn', unique=True)
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index ab0cbf7..8224b9c 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -2,7 +2,7 @@ from sqlobject import *
from sqlobject.sqlbuilder import *
from turbogears import identity, config
from mirrormanager.model import *
-from sqlobject import SQLObject, BoolCol, IntCol
+from sqlobject import SQLObject, BoolCol, IntCol, UnicodeCol, DatabaseIndex
from oldtables import *
import GeoIP
@@ -55,6 +55,10 @@ def change_tables():
OldSiteToSite.sqlmeta.addColumn(DatabaseIndex("username_idx", 'upstream_site', 'username', unique=True), changeSchema=True)
changes['sitetosite.username_password'] = True
+ if 'name' not in OldHostNetblock.sqlmeta.columns:
+ OldHostNetblock.sqlmeta.addColumn(UnicodeCol("name", default=None), changeSchema=True)
+ changes['hostnetblock.name'] = True
+
def update_countries():
db_countries = set(c for c.code in Country.select())
geoip_countries = set(GeoIP.country_codes)
@@ -88,6 +92,10 @@ def fill_new_columns():
s.username = None
s.password = None
+ if changes.get('hostnetblock.name'):
+ for n in HostNetblock.select():
+ n.name = None
+
update_countries()
12 years, 8 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 20 ++++++++++++++++++++
server/mirrormanager/mirrorlist.py | 16 ++++++++++------
server/mirrormanager/model.py | 8 ++++++++
server/mirrormanager/schema_updates/__init__.py | 1 +
server/mirrormanager/templates/host.kid | 20 ++++++++++++++++++++
5 files changed, 59 insertions(+), 6 deletions(-)
New commits:
commit 5d0d1c6069dac03945393c1ce5deeec5e120fe32
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Jul 30 14:30:56 2011 -0500
SCHEMA CHANGE: add HostPeerAsn table and plumbing
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index d0f2ffb..7636786 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -675,6 +675,26 @@ class HostNetblockController(HostListitemController):
HostNetblock(host=host, netblock=kwargs['netblock'])
+
+class HostPeerAsnFields(widgets.WidgetsList):
+ asn = widgets.TextField(validator=validators.All(validators.Int,validators.NotEmpty))
+
+host_peer_asn_form = widgets.TableForm(fields=HostPeerAsnFields(),
+ submit_text="Create Host Peer ASN")
+
+class HostPeerAsnController(HostListitemController):
+ require = identity.in_group(admin_group)
+ submit_action_prefix="/host_peer_asn"
+ page_title = "New Host Peer ASN"
+ form = host_peer_asn_form
+
+ def do_get(self, id):
+ v = HostPeerAsn.get(id)
+ return dict(values=v, host=v.host)
+
+ def do_create(self, host, kwargs):
+ HostPeerAsn(host=host, asn=kwargs['asn'])
+
class HostCountryAllowedFields(widgets.WidgetsList):
country = widgets.TextField(validator=validators.Regex(r'^[a-zA-Z][a-zA-Z]$'),
help_text="2-letter ISO country code")
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 4baa2b0..112a245 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -212,13 +212,17 @@ def host_country_cache():
def asn_host_cache():
cache = {}
for host in Host.selectBy(asn_clients=True):
- if host.asn is None:
- continue
- if host.asn not in cache:
- cache[host.asn] = [host.id]
- else:
- cache[host.asn].append(host.id)
+ if host.asn is not None:
+ if host.asn not in cache:
+ cache[host.asn] = [host.id]
+ else:
+ cache[host.asn].append[host.id]
+ for peer_asn in host.peer_asns:
+ if peer_asn.asn not in cache:
+ cache[peer_asn.asn] = [host.id]
+ else:
+ cache[peer_asn.asn].append(host.id)
return cache
def repository_redirect_cache():
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index f8daecd..6ca8fc9 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -201,6 +201,7 @@ class Host(SQLObject):
exclusive_dirs = MultipleJoin('DirectoryExclusiveHost')
locations = SQLRelatedJoin('Location')
countries = SQLRelatedJoin('Country')
+ peer_asns = MultipleJoin('HostPeerAsn')
def destroySelf(self):
"""Cascade the delete operation"""
@@ -580,6 +581,13 @@ class HostNetblock(SQLObject):
def my_site(self):
return self.host.my_site()
+class HostPeerAsn(SQLObject):
+ class sqlmeta:
+ cacheValues = False
+ host = ForeignKey('Host')
+ asn = IntCol()
+ idx = DatabaseIndex('host', 'asn', unique=True)
+
class HostStats(SQLObject):
class sqlmeta:
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index ba1cf29..ab0cbf7 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -21,6 +21,7 @@ def change_tables():
Country.createTable(ifNotExists=True)
HostCountry.createTable(ifNotExists=True)
NetblockCountry.createTable(ifNotExists=True)
+ HostPeerAsn.createTable(ifNotExists=True)
if 'emailOnDrop' not in OldSite.sqlmeta.columns and \
'emailOnAdd' not in OldSite.sqlmeta.columns:
diff --git a/server/mirrormanager/templates/host.kid b/server/mirrormanager/templates/host.kid
index 86134ab..69a086d 100644
--- a/server/mirrormanager/templates/host.kid
+++ b/server/mirrormanager/templates/host.kid
@@ -58,6 +58,26 @@ Last Crawled: ${values.lastCrawled} <a href="${tg.url('/crawler/'+str(values.id
</ul>
</div>
+<h3>Peer ASNs</h3>
+ Peer ASNs are used to guide an end user on nearby networks
+ to our mirror. For example, a university
+ might list their peer ASNs, and the mirrorlist CGI
+ would return the university-local mirror rather than
+ a country-local mirror. You must be in the
+ MirrorManager administrators group in order to
+ create new entries here.<br/>
+
+<div py:if="admin_group in tg.identity.groups">
+<a href="${tg.url('/host_peer_asn/0/new?hostid=' + str(values.id))}">[Add]</a><br/>
+</div>
+ <ul>
+ <li py:for="a in values.peer_asns">
+ <span py:replace="a.asn">ASN</span><a
+ href="${tg.url('/host_peer_asn/' + str(a.id) + '/delete')}">[Delete]</a>
+ </li>
+ </ul>
+</div>
+
<h3>Locations</h3>
Locations are ways to group hosts when netblocks are unwieldy.
Examples include Amazon EC2 availability zones. Mirrorlist clients
12 years, 8 months
Branch 'v1.4' - 2 commits - server/dev-fas.cfg server/mirrormanager server/update-database-schema
by Matt Domsch
server/dev-fas.cfg | 1
server/mirrormanager/config/app.cfg | 6 ++--
server/mirrormanager/schema_updates/__init__.py | 12 ++++++--
server/update-database-schema | 35 ++++++++++++++++++++++++
4 files changed, 47 insertions(+), 7 deletions(-)
New commits:
commit 5475353e44cb3672725f0f19d21625479171dd6c
Author: Toshio Kuratomi <toshio(a)fedoraproject.org>
Date: Wed Jul 20 19:59:57 2011 -0700
Two fixes for TG-1.1.x: Set the default template engine to kid
explicitly and remove unneeded sqlalchemy.dburi:/// for using jsonfas
auth.
diff --git a/server/dev-fas.cfg b/server/dev-fas.cfg
index 71b4f6d..3c0414b 100644
--- a/server/dev-fas.cfg
+++ b/server/dev-fas.cfg
@@ -45,7 +45,6 @@ tg.strict_parameters = True
##############################
# Fedora Account System config
-sqlalchemy.dburi="sqlite:///"
identity.provider='jsonfas2'
identity.saprovider.model.visit="fedora.accounts.tgfas.VisitIdentity"
visit.manager="jsonfas2"
diff --git a/server/mirrormanager/config/app.cfg b/server/mirrormanager/config/app.cfg
index a09f610..e6f68bc 100644
--- a/server/mirrormanager/config/app.cfg
+++ b/server/mirrormanager/config/app.cfg
@@ -10,14 +10,14 @@
# which view (template engine) to use if one is not specified in the
# template name
-# tg.defaultview = "kid"
+tg.defaultview = "kid"
# The following kid settings determine the settings used by the kid serializer.
# One of (html|xml|json)
-# kid.outputformat="html"
+kid.outputformat="html"
-# kid.encoding="utf-8"
+kid.encoding="utf-8"
# The sitetemplate is used for overall styling of a site that
# includes multiple TurboGears applications
commit ba082b1b52f1bfa3e53665c572195fa3a949e1e3
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Jul 9 23:33:46 2011 -0500
add update-database-schema, add update_countries()
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index c164091..ba1cf29 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -4,6 +4,7 @@ from turbogears import identity, config
from mirrormanager.model import *
from sqlobject import SQLObject, BoolCol, IntCol
from oldtables import *
+import GeoIP
from turbogears.database import PackageHub
hub = PackageHub("mirrormanager")
@@ -53,6 +54,12 @@ def change_tables():
OldSiteToSite.sqlmeta.addColumn(DatabaseIndex("username_idx", 'upstream_site', 'username', unique=True), changeSchema=True)
changes['sitetosite.username_password'] = True
+def update_countries():
+ db_countries = set(c for c.code in Country.select())
+ geoip_countries = set(GeoIP.country_codes)
+ diff = geoip_countries.difference(db_countries)
+ for cc in diff:
+ Country(code=cc)
def fill_new_columns():
global changes
@@ -80,10 +87,9 @@ def fill_new_columns():
s.username = None
s.password = None
+ update_countries()
+
def update():
- """Fills newly created database columns with information.
- Run this after using tg-admin sql upgrade.
- """
change_tables()
fill_new_columns()
diff --git a/server/update-database-schema b/server/update-database-schema
new file mode 100755
index 0000000..045a7c8
--- /dev/null
+++ b/server/update-database-schema
@@ -0,0 +1,35 @@
+#!/usr/bin/python
+__requires__ = 'TurboGears[future]'
+import pkg_resources
+pkg_resources.require("TurboGears")
+
+import turbogears
+from optparse import OptionParser
+import mirrormanager.schema_updates
+
+from turbogears.database import PackageHub
+hub = __connection__ = None
+
+def doit():
+ mirrormanager.schema_updates.update()
+
+
+def main():
+ parser = OptionParser(usage=sys.argv[0] + " [options]")
+ parser.add_option("-c", "--config",
+ dest="config", default='dev.cfg',
+ help="TurboGears config file to use")
+
+ (options, args) = parser.parse_args()
+
+ turbogears.update_config(configfile=options.config,
+ modulename="mirrormanager.config")
+ global hub
+ global __connection__
+ hub = PackageHub("mirrormanager")
+ __connection__ = hub
+
+ doit()
+
+if __name__ == "__main__":
+ sys.exit(main())
12 years, 9 months
server/dev-fas.cfg server/mirrormanager
by Toshio くらとみ
server/dev-fas.cfg | 1 -
server/mirrormanager/config/app.cfg | 6 +++---
2 files changed, 3 insertions(+), 4 deletions(-)
New commits:
commit b65e1e2b28b6534e2b90e2d35e7f7da4f75c6dbb
Author: Toshio Kuratomi <toshio(a)fedoraproject.org>
Date: Wed Jul 20 19:59:57 2011 -0700
Two fixes for TG-1.1.x: Set the default template engine to kid
explicitly and remove unneeded sqlalchemy.dburi:/// for using jsonfas
auth.
diff --git a/server/dev-fas.cfg b/server/dev-fas.cfg
index 71b4f6d..3c0414b 100644
--- a/server/dev-fas.cfg
+++ b/server/dev-fas.cfg
@@ -45,7 +45,6 @@ tg.strict_parameters = True
##############################
# Fedora Account System config
-sqlalchemy.dburi="sqlite:///"
identity.provider='jsonfas2'
identity.saprovider.model.visit="fedora.accounts.tgfas.VisitIdentity"
visit.manager="jsonfas2"
diff --git a/server/mirrormanager/config/app.cfg b/server/mirrormanager/config/app.cfg
index a09f610..e6f68bc 100644
--- a/server/mirrormanager/config/app.cfg
+++ b/server/mirrormanager/config/app.cfg
@@ -10,14 +10,14 @@
# which view (template engine) to use if one is not specified in the
# template name
-# tg.defaultview = "kid"
+tg.defaultview = "kid"
# The following kid settings determine the settings used by the kid serializer.
# One of (html|xml|json)
-# kid.outputformat="html"
+kid.outputformat="html"
-# kid.encoding="utf-8"
+kid.encoding="utf-8"
# The sitetemplate is used for overall styling of a site that
# includes multiple TurboGears applications
12 years, 9 months
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py server/mirrormanager
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 16 ++++++++--------
server/mirrormanager/mirrorlist.py | 4 +++-
2 files changed, 11 insertions(+), 9 deletions(-)
New commits:
commit 6b59eb6faa106720f49cdd0c06066ee1cf739a6c
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Jul 9 22:56:19 2011 -0500
mirrorlist_server: fix radix tree lookups
don't overload use of tree data field 'hosts' with content that isn't
a list of hosts. Code reuse is good, but make it obvious, and there's
no point overloading the data field as it's already a dict, so named
elements are good.
extract data from netblock_country_cache correctly. It's supposed to
be an iterable, because it's handed as result.extend(foo), while we
were putting it into a plain element, not an iterable. Whoops.
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 947e48d..0c066f4 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -185,7 +185,7 @@ def metalink(cache, directory, file, hosts_and_urls):
doc += '</metalink>\n'
return ('metalink', 200, doc)
-def tree_lookup(tree, ip, maxResults=None):
+def tree_lookup(tree, key, field, maxResults=None):
# fast lookup in the tree; if present, find all the matching values by deleting the found one and searching again
# this is safe w/o copying the tree again only because this is the only place the tree is used, and
# we'll get a new copy of the tree from our parent the next time it fork()s.
@@ -194,7 +194,7 @@ def tree_lookup(tree, ip, maxResults=None):
return result
node = tree.search_best(ip.strNormal())
while node is not None:
- result.extend(node.data['hosts'])
+ result.extend(node.data[field])
prefix = node.prefix
if maxResults is None or len(result) < maxResults:
tree.delete(prefix)
@@ -293,7 +293,7 @@ def do_country(kwargs, cache, clientCountry, requested_countries, header):
def do_netblocks(kwargs, cache, header):
hostresults = set()
if not kwargs.has_key('netblock') or kwargs['netblock'] == "1":
- hosts = tree_lookup(host_netblocks_tree, kwargs['IP'])
+ hosts = tree_lookup(host_netblocks_tree, kwargs['IP'], 'hosts')
for hostid in hosts:
if hostid in cache['byHostId']:
hostresults.add(hostid)
@@ -399,7 +399,7 @@ def client_ip_to_country(ip):
return None
# lookup in the cache first
- result = tree_lookup(netblock_country_tree, ip, maxResults=1)
+ result = tree_lookup(netblock_country_tree, ip, 'country', maxResults=1)
if len(result) > 0:
clientCountry = result[0]
return clientCountry
@@ -618,11 +618,11 @@ def do_mirrorlist(kwargs):
d = dict(message=header, resulttype='mirrorlist', returncode=200, results=host_url_list)
return d
-def setup_cache_tree(cache):
+def setup_cache_tree(cache, field):
tree = radix.Radix()
for k, v in cache.iteritems():
node = tree.add(k.strNormal())
- node.data['hosts'] = v
+ node.data[field] = v
return tree
def setup_netblocks(netblocks_file):
@@ -714,8 +714,8 @@ def read_caches():
internet2_tree = setup_netblocks(internet2_netblocks_file)
global_tree = setup_netblocks(global_netblocks_file)
- host_netblocks_tree = setup_cache_tree(host_netblock_cache)
- netblock_country_tree = setup_cache_tree(netblock_country_cache)
+ host_netblocks_tree = setup_cache_tree(host_netblock_cache, 'hosts')
+ netblock_country_tree = setup_cache_tree(netblock_country_cache, 'country')
def errordoc(metalink, message):
if metalink:
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 2ecab3e..4baa2b0 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -274,7 +274,9 @@ def netblock_country_cache():
ip = IP(n.netblock)
except:
continue
- cache[ip.strnormal()] = n.country
+ # guaranteed to be unique by database constraints
+ cache[ip.strNormal()] = (n.country)
+
return cache
12 years, 9 months
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
New commits:
commit 5b00b84789e4c40605fae640cafe7362e49156b1
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Jul 9 19:49:32 2011 -0500
mirrorlist_server: early exit from tree_lookup() if more results aren't needed
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index f11966f..947e48d 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -185,8 +185,8 @@ def metalink(cache, directory, file, hosts_and_urls):
doc += '</metalink>\n'
return ('metalink', 200, doc)
-def tree_lookup(tree, ip):
- # fast lookup in the tree; if present, find all the netblocks by deleting the found one and searching again
+def tree_lookup(tree, ip, maxResults=None):
+ # fast lookup in the tree; if present, find all the matching values by deleting the found one and searching again
# this is safe w/o copying the tree again only because this is the only place the tree is used, and
# we'll get a new copy of the tree from our parent the next time it fork()s.
result = []
@@ -196,13 +196,13 @@ def tree_lookup(tree, ip):
while node is not None:
result.extend(node.data['hosts'])
prefix = node.prefix
- tree.delete(prefix)
- node = tree.search_best(ip.strNormal())
+ if maxResults is None or len(result) < maxResults:
+ tree.delete(prefix)
+ node = tree.search_best(ip.strNormal())
+ else:
+ break
return result
-def client_netblocks(ip):
- return tree_lookup(host_netblocks_tree, ip)
-
def trim_by_client_country(s, clientCountry):
if clientCountry is None:
return s
@@ -293,7 +293,7 @@ def do_country(kwargs, cache, clientCountry, requested_countries, header):
def do_netblocks(kwargs, cache, header):
hostresults = set()
if not kwargs.has_key('netblock') or kwargs['netblock'] == "1":
- hosts = client_netblocks(kwargs['IP'])
+ hosts = tree_lookup(host_netblocks_tree, kwargs['IP'])
for hostid in hosts:
if hostid in cache['byHostId']:
hostresults.add(hostid)
@@ -399,7 +399,7 @@ def client_ip_to_country(ip):
return None
# lookup in the cache first
- result = tree_lookup(netblock_country_tree, ip)
+ result = tree_lookup(netblock_country_tree, ip, maxResults=1)
if len(result) > 0:
clientCountry = result[0]
return clientCountry
12 years, 9 months
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 53 +++++++++++++--------------------
1 file changed, 22 insertions(+), 31 deletions(-)
New commits:
commit 740c8dc2ebb625d9989ffad46c216970ee815c4e
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Jul 9 19:22:33 2011 -0500
mirrorlist_server: unify calls to IPy.IP(client_ip) to one place
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index cdc856d..f11966f 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -73,7 +73,7 @@ def lookup_ip_asn(tree, ip):
""" @t is a radix tree
@ip is an IPy.IP object which may be contained in an entry in l
"""
- node = tree.search_best(str(ip))
+ node = tree.search_best(ip.strNormal())
if node is None:
return None
return node.data['asn']
@@ -185,25 +185,23 @@ def metalink(cache, directory, file, hosts_and_urls):
doc += '</metalink>\n'
return ('metalink', 200, doc)
-def tree_lookup(tree, clientIP):
+def tree_lookup(tree, ip):
# fast lookup in the tree; if present, find all the netblocks by deleting the found one and searching again
# this is safe w/o copying the tree again only because this is the only place the tree is used, and
# we'll get a new copy of the tree from our parent the next time it fork()s.
result = []
- node = tree.search_best(clientIP.strNormal())
+ if ip is None:
+ return result
+ node = tree.search_best(ip.strNormal())
while node is not None:
result.extend(node.data['hosts'])
prefix = node.prefix
tree.delete(prefix)
- node = tree.search_best(clientIP.strNormal())
+ node = tree.search_best(ip.strNormal())
return result
def client_netblocks(ip):
- try:
- clientIP = IP(ip)
- except:
- return []
- result = tree_lookup(host_netblocks_tree, clientIP)
+ return tree_lookup(host_netblocks_tree, ip)
def trim_by_client_country(s, clientCountry):
if clientCountry is None:
@@ -294,9 +292,8 @@ def do_country(kwargs, cache, clientCountry, requested_countries, header):
def do_netblocks(kwargs, cache, header):
hostresults = set()
- client_ip = kwargs['client_ip']
if not kwargs.has_key('netblock') or kwargs['netblock'] == "1":
- hosts = client_netblocks(client_ip)
+ hosts = client_netblocks(kwargs['IP'])
for hostid in hosts:
if hostid in cache['byHostId']:
hostresults.add(hostid)
@@ -305,12 +302,8 @@ def do_netblocks(kwargs, cache, header):
def do_internet2(kwargs, cache, clientCountry, header):
hostresults = set()
- client_ip = kwargs['client_ip']
- if client_ip == 'unknown':
- return (header, hostresults)
- try:
- ip = IP(client_ip)
- except:
+ ip = kwargs['IP']
+ if ip is None:
return (header, hostresults)
asn = lookup_ip_asn(internet2_tree, ip)
if asn is not None:
@@ -322,12 +315,8 @@ def do_internet2(kwargs, cache, clientCountry, header):
def do_asn(kwargs, cache, header):
hostresults = set()
- client_ip = kwargs['client_ip']
- if client_ip == 'unknown':
- return (header, hostresults)
- try:
- ip = IP(client_ip)
- except:
+ ip = kwargs['IP']
+ if ip is None:
return (header, hostresults)
asn = lookup_ip_asn(global_tree, ip)
if asn is not None and asn in asn_host_cache:
@@ -404,11 +393,9 @@ def trim_to_preferred_protocols(hosts_and_urls):
results.append((hostid, url))
return results
-def client_ip_to_country(client_ip):
+def client_ip_to_country(ip):
clientCountry = None
- try:
- ip = IP(client_ip)
- except:
+ if ip is None:
return None
# lookup in the cache first
@@ -501,6 +488,11 @@ def do_mirrorlist(kwargs):
repo_information += "# repo=%s&arch=%s\n" % i
return return_error(kwargs, message=repo_information)
+ # set kwargs['IP'] exactly once
+ try:
+ kwargs['IP'] = IP(kwargs['client_ip'])
+ except:
+ kwargs['IP'] = None
ordered_mirrorlist = cache.get('ordered_mirrorlist', default_ordered_mirrorlist)
done = 0
@@ -532,8 +524,7 @@ def do_mirrorlist(kwargs):
if not ordered_mirrorlist:
done = 1
- client_ip = kwargs['client_ip']
- clientCountry = client_ip_to_country(client_ip)
+ clientCountry = client_ip_to_country(kwargs['IP'])
if clientCountry is None:
print_client_country = "N/A"
@@ -542,7 +533,7 @@ def do_mirrorlist(kwargs):
if debug:
if kwargs.has_key('repo') and kwargs.has_key('arch'):
- print ("IP: " + client_ip +
+ print ("IP: " + (kwargs['IP'] or 'None') +
"; DATE: " + time.strftime("%Y-%m-%d") +
"; COUNTRY: " + print_client_country +
"; REPO: " + kwargs['repo'] +
@@ -550,7 +541,7 @@ def do_mirrorlist(kwargs):
if logfile is not None:
if kwargs.has_key('repo') and kwargs.has_key('arch'):
- logfile.write("IP: " + client_ip +
+ logfile.write("IP: " + (kwargs('IP') or 'None') +
"; DATE: " + time.strftime("%Y-%m-%d") +
"; COUNTRY: " + print_client_country +
"; REPO: " + kwargs['repo'] +
12 years, 9 months