Branch 'v1.4' - server/crawler_perhost
by Matt Domsch
server/crawler_perhost | 52 ++++++++++++++++++++++++++++++++-----------------
1 file changed, 34 insertions(+), 18 deletions(-)
New commits:
commit 9ef40977d5add4babe099e630b6fa9ab72895abc
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed May 25 10:22:19 2011 -0500
crawler: don't keep probing bitflip-locked directories
diff --git a/server/crawler_perhost b/server/crawler_perhost
index 92bf12c..23e251b 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -146,11 +146,11 @@ class hostState:
self.close_ftp(c)
-class TryLater(Exception):
- pass
+class TryLater(Exception): pass
+class ForbiddenExpected(Exception): pass
-def get_ftp_dir(hoststate, url, i=0):
+def get_ftp_dir(hoststate, url, readable, i=0):
if i > 1:
raise TryLater()
@@ -162,11 +162,14 @@ def get_ftp_dir(hoststate, url, i=0):
return []
# Returned by Princeton University when directory isn't readable (pre-bitflip)
if str(e).startswith('553'):
- return None
+ if readable:
+ return []
+ else:
+ raise ForbiddenExpected()
# Returned by ftp2.surplux.net when cannot log in due to connection restrictions
if str(e).startswith('530'):
hoststate.close_ftp(url)
- return get_ftp_dir(hoststate, url, i+1)
+ return get_ftp_dir(hoststate, url, readable, i+1)
if str(e).startswith('500'): # Oops
raise TryLater()
else:
@@ -188,17 +191,19 @@ def get_ftp_dir(hoststate, url, i=0):
raise
except (EOFError, socket.error):
hoststate.close_ftp(url)
- return get_ftp_dir(hoststate, url, i+1)
+ return get_ftp_dir(hoststate, url, readable, i+1)
return hoststate.ftp_dir_results
-def check_ftp_file(hoststate, url, filedata):
+def check_ftp_file(hoststate, url, filedata, readable):
if url.endswith('/'):
url = url[:-1]
try:
- results = get_ftp_dir(hoststate, url)
+ results = get_ftp_dir(hoststate, url, readable)
except TryLater:
raise
+ except ForbiddenExpected:
+ return None
if results is None:
return None
if len(results) == 1:
@@ -207,26 +212,26 @@ def check_ftp_file(hoststate, url, filedata):
return True
return False
-def check_url(hoststate, url, filedata, recursion):
+def check_url(hoststate, url, filedata, recursion, readable):
if url.startswith('http:'):
- return check_head(hoststate, url, filedata, recursion)
+ return check_head(hoststate, url, filedata, recursion, readable)
elif url.startswith('ftp:'):
- return check_ftp_file(hoststate, url, filedata)
+ return check_ftp_file(hoststate, url, filedata, readable)
class HTTPUnknown(Exception): pass
class HTTP500(Exception): pass
-def handle_redirect(hoststate, url, location, filedata, recursion):
+def handle_redirect(hoststate, url, location, filedata, recursion, readable):
if recursion > 10:
raise HTTPUnknown()
if location.startswith('/'):
scheme, netloc, path, query, fragment = urlsplit(url)
location = '%s:%s%s' % (scheme, netloc, location)
- return check_url(hoststate, location, filedata, recursion+1)
+ return check_url(hoststate, location, filedata, recursion+1, readable)
-def check_head(hoststate, url, filedata, recursion, retry=0):
+def check_head(hoststate, url, filedata, recursion, readable, retry=0):
""" Returns tuple:
True - URL exists
False - URL doesn't exist
@@ -275,11 +280,14 @@ def check_head(hoststate, url, filedata, recursion, retry=0):
else:
return False
if status >= 300 and status < 400:
- return handle_redirect(hoststate, url, r.getheader('Location'), filedata, recursion)
+ return handle_redirect(hoststate, url, r.getheader('Location'), filedata, recursion, readable)
elif status >= 400 and status < 500:
if status == 403: # forbidden
# may be a hidden dir still
- return None
+ if readable:
+ return False
+ else:
+ raise ForbiddenExpected()
elif status == 404 or status == 410: # not found / gone
return False
# we don't know
@@ -388,12 +396,14 @@ def try_perfile(d, hoststate, url):
exists = None
graburl = "%s/%s" % (url, file)
try:
- exists = check_url(hoststate, graburl, d.files[file], 0)
+ exists = check_url(hoststate, graburl, d.files[file], 0, d.readable)
#print >>logfile, "%s %s" % (exists, graburl)
if exists == False:
return False
except TryLater:
raise
+ except ForbiddenExpected:
+ return None
except ftplib.all_errors:
hoststate.close_ftp(url)
return None
@@ -422,7 +432,7 @@ def try_perdir(d, hoststate, url):
results = {}
if not url.endswith('/'):
url += '/'
- listing = get_ftp_dir(hoststate, url)
+ listing = get_ftp_dir(hoststate, url, d.readable)
if listing is None:
return None
@@ -606,6 +616,12 @@ def per_host(host, options=None):
# could be a dir with no files, or an unreadable dir.
# defer decision on this dir, let a child decide.
pass
+ except TryLater:
+ timer.cancel()
+ msg = "Server load exceeded - try later"
+ print >> logfile, msg
+ sys.exit(1)
+
except:
timer.cancel()
mark_not_up2date(sys.exc_info(), host, "Unknown exception raised. This is a bug in the MM crawler.")
12 years, 11 months
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py server/mirrormanager
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 5 +++++
server/mirrormanager/mirrorlist.py | 15 ++++++++++++++-
server/mirrormanager/model.py | 13 +++++++++++++
server/mirrormanager/schema_updates/__init__.py | 1 +
4 files changed, 33 insertions(+), 1 deletion(-)
New commits:
commit ecbf226c5cee55150eb14c9c9c9ce7a36be19f26
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue May 24 23:36:47 2011 -0500
SCHEMA CHANGE: add NetblockCountry table to allow overriding of GeoIP data
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index d5d078c..fafe90a 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -66,6 +66,7 @@ internet2_tree = radix.Radix()
global_tree = radix.Radix()
host_netblocks_tree = radix.Radix()
location_cache = {}
+netblock_country_cache = {}
def lookup_ip_asn(tree, ip):
""" @t is a radix tree
@@ -500,6 +501,7 @@ def do_mirrorlist(kwargs):
client_ip = kwargs['client_ip']
clientCountry = None
+ # fixme!! lookup client in netblock_country_cache here first
# attempt IPv6, then IPv6 6to4 as IPv4, then Teredo, then IPv4
try:
ip = IP(client_ip)
@@ -655,6 +657,7 @@ def read_caches():
global hcurl_cache
global asn_host_cache
global location_cache
+ global netblock_country_cache
data = {}
try:
@@ -690,6 +693,8 @@ def read_caches():
asn_host_cache = data['asn_host_cache']
if 'location_cache' in data:
location_cache = data['location_cache']
+ if 'netblock_country_cache' in data:
+ netblock_country_cache = data['netblock_country_cache']
del data
setup_continents()
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 2181188..2ecab3e 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -266,6 +266,18 @@ def location_cache():
cache[location.name] = [host.id for host in location.hosts]
return cache
+def netblock_country_cache():
+ cache = {}
+ for n in NetblockCountry.select():
+ ip = None
+ try:
+ ip = IP(n.netblock)
+ except:
+ continue
+ cache[ip.strnormal()] = n.country
+ return cache
+
+
def populate_all_caches():
populate_host_country_allowed_cache()
populate_netblock_cache()
@@ -285,7 +297,8 @@ def dump_caches():
'file_details_cache':file_details_cache(),
'hcurl_cache':hcurl_cache(),
'asn_host_cache':asn_host_cache(),
- 'location_cache':location_cache()}
+ 'location_cache':location_cache(),
+ 'netblock_country_cache':netblock_country_cache()}
try:
f = open('/var/lib/mirrormanager/mirrorlist_cache.pkl', 'w')
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 17576c8..f8daecd 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -928,6 +928,19 @@ class HostCountry(SQLObject):
country = ForeignKey('Country')
hlidx = DatabaseIndex('host', 'country', unique=True)
+class NetblockCountry(SQLObject):
+ class sqlmeta:
+ cacheValues = False
+ netblock = UnicodeCol(alternateID=True, notNone=True, length=UnicodeColKeyLength)
+ country = UnicodeCol(notNone=True)
+
+ def _set_country(self, country):
+ self._SO_set_country(country.upper())
+
+ def _set_continent(self, continent):
+ self._SO_set_continent(continent.upper())
+
+
###############################################################
# These classes are only used if you're not using the
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index d9674b1..c164091 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -19,6 +19,7 @@ def change_tables():
HostLocation.createTable(ifNotExists=True)
Country.createTable(ifNotExists=True)
HostCountry.createTable(ifNotExists=True)
+ NetblockCountry.createTable(ifNotExists=True)
if 'emailOnDrop' not in OldSite.sqlmeta.columns and \
'emailOnAdd' not in OldSite.sqlmeta.columns:
12 years, 11 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 17 ++++++++++++++---
server/mirrormanager/model.py | 3 +++
server/mirrormanager/schema_updates/__init__.py | 13 ++++++++++++-
server/mirrormanager/schema_updates/oldtables.py | 5 +++++
4 files changed, 34 insertions(+), 4 deletions(-)
New commits:
commit be1cc6b445c09d6a80c0df38e612e59b90f673c4
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue May 24 23:16:02 2011 -0500
SCHEMA CHANGE: add username/password fields to SiteToSite, to facilitate keeping rsync secrets files in MM
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 47f7d4b..66c6c5b 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -237,7 +237,8 @@ class SiteToSiteFields(widgets.WidgetsList):
sites = widgets.MultipleSelectField(options=get_sites_options, size=15,
validator=validators.NotEmpty())
-
+ username = widgets.TextField(validator=validators.UnicodeString, help_text="per-upstream-mirror username to be given to downstream mirror admin")
+ password = widgets.TextField(validator=validators.UnicodeString, help_text="pser-upstream-mirror username password")
site_to_site_form = widgets.TableForm(fields=SiteToSiteFields(),
submit_text="Add Downstream Site")
@@ -265,7 +266,7 @@ class SiteToSiteController(controllers.Controller, identity.SecureResource, cont
@expose()
@validate(form=site_to_site_form)
@error_handler(new)
- def create(self, **kwargs):
+ def create(self, tg_errors=None, **kwargs):
if not kwargs.has_key('siteid'):
turbogears.flash("Error: form didn't provide siteid")
raise redirect("/")
@@ -277,13 +278,23 @@ class SiteToSiteController(controllers.Controller, identity.SecureResource, cont
turbogears.flash("Error: Site %s does not exist" % siteid)
raise redirect("/")
+ errordict = dict(form=site_to_site_form, values=None, action=submit_action, page_title="Create Site to Site", site=site)
+
+ # handle the validation error
+ if tg_errors:
+ turbogears.flash("Error creating SiteToSite: %s" % (createErrorString(tg_errors)))
+ return errordict
+
+
siteadmin_check(site, identity)
sites = kwargs['sites']
+ username = kwargs.get('username')
+ password = kwargs.get('password')
for dssite in sites:
if dssite == site.id:
continue
try:
- site2site = SiteToSite(upstream_site=site, downstream_site=dssite)
+ site2site = SiteToSite(upstream_site=site, downstream_site=dssite, username=username, password=password)
except:
pass
turbogears.flash("SiteToSite created.")
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index a865ecd..17576c8 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -26,6 +26,9 @@ class SiteToSite(SQLObject):
upstream_site = ForeignKey('Site')
downstream_site = ForeignKey('Site')
idx = DatabaseIndex('upstream_site', 'downstream_site', unique=True)
+ username = UnicodeCol(default=None)
+ password = UnicodeCol(default=None)
+ username_idx = DatabaseIndex('upstream_site', 'username', unique=True)
def my_site(self):
return self.upstream_site
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index bc218c8..d9674b1 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -45,6 +45,14 @@ def change_tables():
OldCategory.sqlmeta.addColumn(UnicodeCol("GeoDNSDomain", default=None), changeSchema=True)
changes['category.geo_dns_domain'] = True
+ if 'username' not in OldSiteToSite.sqlmeta.columns and \
+ 'password' not in OldSiteToSite.sqlmeta.columns:
+ OldSiteToSite.sqlmeta.addColumn(UnicodeCol("username", default=None), changeSchema=True)
+ OldSiteToSite.sqlmeta.addColumn(UnicodeCol("password", default=None), changeSchema=True)
+ OldSiteToSite.sqlmeta.addColumn(DatabaseIndex("username_idx", 'upstream_site', 'username', unique=True), changeSchema=True)
+ changes['sitetosite.username_password'] = True
+
+
def fill_new_columns():
global changes
@@ -66,7 +74,10 @@ def fill_new_columns():
for c in Category.select():
c.GeoDNSDomain = None
-
+ if changes.get('sitetosite.username_password'):
+ for s in SiteToSite.select():
+ s.username = None
+ s.password = None
def update():
diff --git a/server/mirrormanager/schema_updates/oldtables.py b/server/mirrormanager/schema_updates/oldtables.py
index b7779f3..cc102ca 100644
--- a/server/mirrormanager/schema_updates/oldtables.py
+++ b/server/mirrormanager/schema_updates/oldtables.py
@@ -7,6 +7,11 @@ __connection__ = hub
# upgrade methodology borrowed from
# http://www.mail-archive.com/sqlobject-discuss@lists.sourceforge.net/msg04...
+class OldSiteToSite(SQLObject):
+ class sqlmeta:
+ fromDatabase = True
+ table = 'site_to_site'
+
class OldSite(SQLObject):
class sqlmeta:
fromDatabase = True
12 years, 11 months
Branch 'v1.4' - server/update-master-directory-list
by Matt Domsch
server/update-master-directory-list | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
New commits:
commit d1024a4b3323224e8f617651ec6b781839fdea45
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue May 24 22:16:09 2011 -0500
umdl: use set() instead of dict for unreadable_dirs
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index 0c130d2..441b518 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -254,7 +254,7 @@ def nuke_gone_directories(category, category_directories):
if not category_directories.has_key(d.name):
d.destroySelf()
-unreadable_dirs = {}
+unreadable_dirs = set()
def parent_dir(path):
sdir = path.split('/')[:-1]
@@ -279,7 +279,7 @@ def make_one_directory(line, category, path, category_directories):
perms = line.split()[0]
if not re.compile('^d......r.x').match(perms) or parent_dir(dname) in unreadable_dirs:
readable=False
- unreadable_dirs[dname] = True
+ unreadable_dirs.add(dname)
try:
perm, size, date, hms, filepath = line.split()
@@ -455,7 +455,7 @@ def sync_directories_from_directory(directory, category, excludes=[]):
mode = s.st_mode
readable = (mode & stat.S_IRWXO & (stat.S_IROTH|stat.S_IXOTH))
if not readable or parent_dir(dname) in unreadable_dirs:
- unreadable_dirs[dname] = True
+ unreadable_dirs.add(dname)
isRepo = 'repodata' in dirnames
changed = (d_ctime != ctime)
12 years, 11 months
Branch 'v1.4' - server/update-master-directory-list
by Matt Domsch
server/update-master-directory-list | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
New commits:
commit 6ca60f58ac66fc8d79440f5c28d22170b7776a7b
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue May 24 22:10:26 2011 -0500
mark child dirs unreadable if their parent is unreadable
We were doing this on the rsync list parsing path, but not on the
directory walk path.
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index a25a99d..0c130d2 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -454,7 +454,7 @@ def sync_directories_from_directory(directory, category, excludes=[]):
mode = s.st_mode
readable = (mode & stat.S_IRWXO & (stat.S_IROTH|stat.S_IXOTH))
- if not readable:
+ if not readable or parent_dir(dname) in unreadable_dirs:
unreadable_dirs[dname] = True
isRepo = 'repodata' in dirnames
12 years, 11 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/mirrorlist.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
New commits:
commit cadc807592d1bb2bbb839ab9ef9596e7674a0b93
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri May 6 00:02:02 2011 -0500
use os.path.dirname rather than splits and joins
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index f82ebde..2181188 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -122,8 +122,8 @@ def populate_directory_cache():
# repodata/ directories aren't themselves repositories, their parent dir is
# we're walking the list in order, so the parent will be added to the cache before the child
- if 'repodata' in directoryname:
- parent = '/'.join(directoryname.split('/')[:-1])
+ if directoryname.endswith('/repodata'):
+ parent = os.path.dirname(directoryname) # parent
cache[directoryname]['ordered_mirrorlist'] = cache[parent]['ordered_mirrorlist']
cache[directoryname]['subpath'] = directoryname[len(category.topdir.name)+1:]
12 years, 11 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/model.py | 13 ++++++
server/mirrormanager/schema_updates/__init__.py | 2
server/mirrormanager/zonecreator.py | 51 +++++++++++++-----------
3 files changed, 43 insertions(+), 23 deletions(-)
New commits:
commit bfa67b0b06aa7c7110bda32f263e4d7976aa3be2
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu May 5 23:56:32 2011 -0500
SCHEMA CHANGE: add Country and HostCountry tables, use in zonecreator
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 16eeb78..a865ecd 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -197,6 +197,7 @@ class Host(SQLObject):
categories = MultipleJoin('HostCategory')
exclusive_dirs = MultipleJoin('DirectoryExclusiveHost')
locations = SQLRelatedJoin('Location')
+ countries = SQLRelatedJoin('Country')
def destroySelf(self):
"""Cascade the delete operation"""
@@ -913,6 +914,18 @@ class FileGroup(SQLObject):
name = UnicodeCol(alternateID=True, length=UnicodeColKeyLength)
files = SQLRelatedJoin('FileDetail')
+class Country(SQLObject):
+ code = UnicodeCol(alternateID=True, length=UnicodeColKeyLength)
+ hosts = SQLRelatedJoin('Host')
+
+class HostCountry(SQLObject):
+ class sqlmeta:
+ table = 'host_country'
+ host = ForeignKey('Host')
+ country = ForeignKey('Country')
+ hlidx = DatabaseIndex('host', 'country', unique=True)
+
+
###############################################################
# These classes are only used if you're not using the
# Fedora Account System or some other backend that provides
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index 03b9dcb..bc218c8 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -17,6 +17,8 @@ def change_tables():
Location.createTable(ifNotExists=True)
FileGroup.createTable(ifNotExists=True)
HostLocation.createTable(ifNotExists=True)
+ Country.createTable(ifNotExists=True)
+ HostCountry.createTable(ifNotExists=True)
if 'emailOnDrop' not in OldSite.sqlmeta.columns and \
'emailOnAdd' not in OldSite.sqlmeta.columns:
diff --git a/server/mirrormanager/zonecreator.py b/server/mirrormanager/zonecreator.py
index c011a70..15ce0a8 100644
--- a/server/mirrormanager/zonecreator.py
+++ b/server/mirrormanager/zonecreator.py
@@ -17,33 +17,38 @@ default_output_dir = '/var/lib/mirrormanager/dns'
def domainTemplateToName(domain, country):
return re.sub(u'CC', country, domain)
+def generateOneCountry(category, zone, hosts):
+ for h in hosts:
+ domainname = domainTemplateToName(category.GeoDNSDomain, h.country)
+ n = dns.name.from_text(domainname)
+
+ try:
+ ips = [IPy.IP(h.name)]
+ except ValueError:
+ ips = name_to_ips(h.name)
+ except:
+ continue
+
+ for ip in ips:
+ if ip.version() == 4:
+ rdatasetA = zone.find_rdataset(n, rdtype=A, create=True)
+ rdata = dns.rdtypes.IN.A.A(IN, A, ip.strNormal())
+ rdatasetA.add(rdata, ttl=default_ttl)
+ elif ip.version() == 6:
+ rdatasetAAAA = zone.find_rdataset(n, rdtype=AAAA, create=True)
+ rdata = dns.rdtypes.IN.AAAA.AAAA(IN, AAAA, ip.strNormal())
+ rdatasetAAAA.add(rdata, ttl=default_ttl)
+ else:
+ continue
+
+
def generateZoneFile(directory):
for c in Category.select():
z = dns.zone.Zone(dns.name.from_text(''))
if c.GeoDNSDomain is None: continue
- hosts = [] # fixme with real lookup
- for h in hosts:
- domainname = domainTemplateToName(c.GeoDNSDomain, h.country)
- n = dns.name.from_text(domainname)
-
- try:
- ips = [IPy.IP(h.name)]
- except ValueError:
- ips = name_to_ips(h.name)
- except:
- continue
-
- for ip in ips:
- if ip.version() == 4:
- rdatasetA = z.find_rdataset(n, rdtype=A, create=True)
- rdata = dns.rdtypes.IN.A.A(IN, A, ip.strNormal())
- rdatasetA.add(rdata, ttl=default_ttl)
- elif ip.version() == 6:
- rdatasetAAAA = z.find_rdataset(n, rdtype=AAAA, create=True)
- rdata = dns.rdtypes.IN.AAAA.AAAA(IN, AAAA, ip.strNormal())
- rdatasetAAAA.add(rdata, ttl=default_ttl)
- else:
- continue
+ for cc in Country.select():
+ hosts = cc.hosts
+ generateOneCountry(c, z, hosts)
z.to_file(os.path.join(directory, u'zone-' + c.GeoDNSDomain))
12 years, 11 months
Branch 'v1.4' - 2 commits - server/mirrormanager
by Matt Domsch
server/mirrormanager/controllers.py | 2 --
server/mirrormanager/lib.py | 2 +-
server/mirrormanager/model.py | 1 -
server/mirrormanager/schema_updates/__init__.py | 10 +++-------
server/mirrormanager/zonecreator.py | 2 +-
5 files changed, 5 insertions(+), 12 deletions(-)
New commits:
commit 7f76e4044eb35b692b1a110f2e7cd99a98b35a54
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu May 5 23:26:52 2011 -0500
excise dnsCountryHost
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index b06d455..47f7d4b 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -313,7 +313,6 @@ class HostFields(widgets.WidgetsList):
asn = widgets.TextField("asn", label="ASN", default='', validator=validators.Int, help_text="Autonomous System Number, used in BGP routing tables.")
asn_clients = widgets.CheckBox('asn_clients', label="ASN Clients?", default=True, help_text="Serve all clients from the same ASN. Used for ISPs, companies, or schools, not personal networks.")
robot_email = widgets.TextField(label="Robot email", validator=validators.All(validators.UnicodeString,validators.Email), help_text="email address, will receive notice of upstream content updates")
- dnsCountryHost = widgets.CheckBox(label="DNS Country Host", help_text="use host in <country-code>.<domain> DNS entries")
comment = widgets.TextField(validator=validators.Any(validators.UnicodeString, validators.Empty), help_text="text, anything else you'd like a public end user to know about your mirror")
host_form = widgets.TableForm(fields=HostFields(),
@@ -326,7 +325,6 @@ class HostController(controllers.Controller, identity.SecureResource, content):
disabled_fields = []
if not identity.in_group(admin_group):
disabled_fields.append('admin_active')
- disabled_fields.append('dnsCountryHost')
if host is not None:
site = host.my_site()
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 29d2949..16eeb78 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -191,7 +191,6 @@ class Host(SQLObject):
internet2_clients = BoolCol(default=False)
asn = IntCol(default=None)
asn_clients = BoolCol(default=True)
- dnsCountryHost = BoolCol(default=False)
countries_allowed = MultipleJoin('HostCountryAllowed')
netblocks = MultipleJoin('HostNetblock', orderBy='netblock')
acl_ips = MultipleJoin('HostAclIp', orderBy='ip')
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index 22dd19d..03b9dcb 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -25,9 +25,9 @@ def change_tables():
changes['site.email_on_drop_add'] = True
# Host
- if 'dnsCountryHost' not in OldHost.sqlmeta.columns:
- OldHost.sqlmeta.addColumn(BoolCol("dnsCountryHost", default=False), changeSchema=True)
- changes['host.dns_country_host'] = True
+ # ensure this isn't present anymore
+ if 'dnsCountryHost' in OldHost.sqlmeta.columns:
+ OldHost.sqlmeta.delColumn("dnsCountryHost"), changeSchema=True)
if 'sortorder' not in OldVersion.sqlmeta.columns and \
'codename' not in OldVersion.sqlmeta.columns:
@@ -51,10 +51,6 @@ def fill_new_columns():
s.emailOnDrop=False
s.emailOnAdd=False
- if changes.get('host.dns_country_host'):
- for h in Host.select():
- h.dnsCountryHost = False
-
if changes.get('version.sortorder_codename'):
for v in Version.select():
v.sortorder = 0
diff --git a/server/mirrormanager/zonecreator.py b/server/mirrormanager/zonecreator.py
index 96c2940..c011a70 100644
--- a/server/mirrormanager/zonecreator.py
+++ b/server/mirrormanager/zonecreator.py
@@ -21,7 +21,7 @@ def generateZoneFile(directory):
for c in Category.select():
z = dns.zone.Zone(dns.name.from_text(''))
if c.GeoDNSDomain is None: continue
- hosts = [ hc.host for hc in c.hostCategories if hc.host.dnsCountryHost ]
+ hosts = [] # fixme with real lookup
for h in hosts:
domainname = domainTemplateToName(c.GeoDNSDomain, h.country)
n = dns.name.from_text(domainname)
commit 9804298c5e80dd3166910af1ea64bfe73a8b817d
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu May 5 23:26:11 2011 -0500
fix pidfile creation
diff --git a/server/mirrormanager/lib.py b/server/mirrormanager/lib.py
index 52892d7..aee89e1 100644
--- a/server/mirrormanager/lib.py
+++ b/server/mirrormanager/lib.py
@@ -38,7 +38,7 @@ def remove_pidfile(pidfile):
os.unlink(pidfile)
def create_pidfile_dir(pidfile):
- piddir = pidfile.split('/')[:-1]
+ piddir = os.path.dirname(pidfile)
try:
os.makedirs(piddir, mode=0755)
except OSError, err:
12 years, 11 months
Branch 'v1.4' - mirrorlist-server/vacuum-mirrormanager server/vacuum-mirrormanager
by Matt Domsch
mirrorlist-server/vacuum-mirrormanager | 2 --
server/vacuum-mirrormanager | 2 ++
2 files changed, 2 insertions(+), 2 deletions(-)
New commits:
commit 87e2349f347b9e2ca76c7ce2d2ef5c8fef669543
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon May 2 08:51:34 2011 -0500
cleanups to mirrorlist-server directory
diff --git a/mirrorlist-server/vacuum-mirrormanager b/mirrorlist-server/vacuum-mirrormanager
deleted file mode 100755
index 44c9809..0000000
--- a/mirrorlist-server/vacuum-mirrormanager
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-vacuumdb -d mirrormanager -h db1 -U mirroradmin -f -v
diff --git a/server/vacuum-mirrormanager b/server/vacuum-mirrormanager
new file mode 100755
index 0000000..44c9809
--- /dev/null
+++ b/server/vacuum-mirrormanager
@@ -0,0 +1,2 @@
+#!/bin/sh
+vacuumdb -d mirrormanager -h db1 -U mirroradmin -f -v
12 years, 11 months
Branch 'v1.4' - README.rpm_setup
by Matt Domsch
README.rpm_setup | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
New commits:
commit 5e49f7d6b6b6aa27e19771f1dc3931639a4205ce
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun May 1 23:13:48 2011 -0500
add comments to README.rpm_setup
diff --git a/README.rpm_setup b/README.rpm_setup
index 1a94647..8ebdbfc 100644
--- a/README.rpm_setup
+++ b/README.rpm_setup
@@ -26,4 +26,19 @@ following actions.
- Visit your Mirror Manager app:
- http://www.example.com/mirrormanager
+ http://localhost/mirrormanager
+
+
+There are three continuously running processes.
+1) mirrormanager.wsgi, started by apache (or your http server). This
+ is the administrative web user interface, written in TurboGears.
+2) mirrorlist-client.wsgi, started by apache (or your http server).
+ This is the web server -> mirrorlist_server.py connector, which takes
+ client (typically yum or another http client) requests, validates
+ them, forwards the request to mirrorlist_server.py, receives the
+ answer back, and replies to the client.
+3) mirrorlist_server.py, started by supervisor. This is the heart of
+ the client responder, where the answer to any client query is
+ generated.
+
+You must ensure that apache and supervisor are configured to start.
12 years, 12 months