[mirrormanager] v1.4: fix permissions on import_centos (03354d7)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 03354d7559b694ae47bddbc5cc40cf055a4c2632
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Feb 24 23:45:18 2013 -0600
fix permissions on import_centos
>---------------------------------------------------------------
0 files changed, 0 insertions(+), 0 deletions(-)
diff --git a/server/import_centos b/server/import_centos
old mode 100644
new mode 100755
11 years, 2 months
[mirrormanager] v1.4: bump spec (0c4fb13)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 0c4fb1340d92b5e409603a496dad932ceb13b1d8
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Feb 24 23:41:30 2013 -0600
bump spec
>---------------------------------------------------------------
mirrormanager.spec.in | 7 +++++--
1 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/mirrormanager.spec.in b/mirrormanager.spec.in
index 04f64d5..01e6582 100644
--- a/mirrormanager.spec.in
+++ b/mirrormanager.spec.in
@@ -1,6 +1,6 @@
Name: mirrormanager
Version: ##VERSION##
-Release: 0.1%{?dist}
+Release: 0.2%{?dist}
Summary: Fedora mirror management system
BuildArch: noarch
@@ -102,7 +102,10 @@ exit 0
%doc LICENSES LICENSE-GPLv2
%changelog
-* Fri Jan 25 2013 Matt Domsch <mdomsch(a)localhost.localdomain> - 1.4.0-0.1
+* Mon Feb 25 2013 Matt Domsch <mdomsch(a)fedoraproject.org> - 1.4.0-0.2
+- bug fixes on the 1.4 branch
+
+* Fri Jan 25 2013 Matt Domsch <mdomsch(a)fedoraproject.org> - 1.4.0-0.1
- initial 1.4 branch for testing
* Sat Jun 9 2012 Matt Domsch <mdomsch(a)fedoraproject.org>
11 years, 2 months
[mirrormanager] v1.4: add distros/ as placeholder for distro-specific code (13e07db)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 13e07dbab28c47e4888802d04d6e180580f58bb6
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Feb 24 23:39:55 2013 -0600
add distros/ as placeholder for distro-specific code
>---------------------------------------------------------------
server/mirrormanager/{ => distros}/__init__.py | 0
server/mirrormanager/distros/base.py | 36 ++++++++++++++++++++++++
server/mirrormanager/distros/centos.py | 5 +++
server/mirrormanager/distros/debian.py | 11 +++++++
4 files changed, 52 insertions(+), 0 deletions(-)
diff --git a/server/mirrormanager/__init__.py b/server/mirrormanager/distros/__init__.py
similarity index 100%
copy from server/mirrormanager/__init__.py
copy to server/mirrormanager/distros/__init__.py
diff --git a/server/mirrormanager/distros/base.py b/server/mirrormanager/distros/base.py
new file mode 100644
index 0000000..8dea452
--- /dev/null
+++ b/server/mirrormanager/distros/base.py
@@ -0,0 +1,36 @@
+from turbogears.util import load_class
+import re
+
+class Distro(object):
+ legacy_site_name = 'Fedora Mirror'
+
+ def __init__(self):
+ super(Distro, self).__init__()
+ glob_ns = globals()
+
+
+ classname = 'Base'
+ class_path = config.get('mirrormanager.distro', classname)
+ class_ = load_class(class_path)
+ if class_:
+ log.info('Successfully loaded "%s".', class_path)
+ glob_ns['%s_class' % classname] = class_
+ else:
+ log.error('Could not load class "%s".'
+ ' Check mirrormanager.distro setting')
+
+
+ def trim_os_from_dirname(dirname):
+ # trim the /os off the name
+ index = dirname.rfind('/os')
+ if index > 0:
+ dirname = dirname[:index]
+ return dirname
+
+ def get_version_from_path(path):
+ # Fedora versioning
+ s = r'/?(([\.\d]+)(\-\w+)?)/'
+ m = re.search(re.compile(s), path)
+ if m is not None:
+ return m.group(1)
+ return None
diff --git a/server/mirrormanager/distros/centos.py b/server/mirrormanager/distros/centos.py
new file mode 100644
index 0000000..cec48f1
--- /dev/null
+++ b/server/mirrormanager/distros/centos.py
@@ -0,0 +1,5 @@
+from base import Distro
+
+class CentosDistro(Distro):
+ def trim_os_from_dirname(dirname):
+ return dirname
diff --git a/server/mirrormanager/distros/debian.py b/server/mirrormanager/distros/debian.py
new file mode 100644
index 0000000..3c08415
--- /dev/null
+++ b/server/mirrormanager/distros/debian.py
@@ -0,0 +1,11 @@
+from base import Distro
+import re
+
+class DebianDistro(Distro):
+ def get_version_from_path(path):
+ # Debian/Ubuntu versioning
+ s = r'dists/(\w+)/' # this ignores 10.10 and maverick-{anything}, but picks up 'maverick'
+ m = re.search(re.compile(s), path)
+ if m is not None:
+ return m.group(1)
+ return None
11 years, 2 months
[mirrormanager] v1.4: add import_centos; initial draft (3b87a3f)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 3b87a3f781ee9bdbae4697a89e171283e66e1094
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Feb 24 23:37:13 2013 -0600
add import_centos; initial draft
>---------------------------------------------------------------
server/import_centos | 376 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 376 insertions(+), 0 deletions(-)
diff --git a/server/import_centos b/server/import_centos
new file mode 100644
index 0000000..eb29612
--- /dev/null
+++ b/server/import_centos
@@ -0,0 +1,376 @@
+#!/usr/bin/python
+#
+# Copyright 2011 Dell, Inc.
+# by Matt Domsch <Matt_Domsch(a)dell.com>
+# MIT/X11 license
+
+# FIXME: add a wiki page to the CentOS wiki describing how to configure report_mirror.conf, and fix the email link
+# FIXME: add header to CSV file noted below
+# FIXME: filter out duplicates from the CSV file (if any)
+
+
+__requires__ = 'TurboGears[future]'
+import pkg_resources
+pkg_resources.require("TurboGears")
+
+from sqlobject import *
+import sys
+import turbogears
+from mirrormanager.model import *
+from mirrormanager.mirrorlist import name_to_ips
+from optparse import OptionParser
+import csv
+import random, string
+import urllib
+import radix
+from IPy import IP
+import GeoIP
+import smtplib
+from email.mime.text import MIMEText
+
+from turbogears.database import PackageHub
+from turbogears import config
+hub = __connection__ = None
+
+centos_upstream = u'rsync://msync.centos.org/CentOS'
+internet2_netblocks_file = '/var/lib/mirrormanager/i2_netblocks.txt'
+global_netblocks_file = '/var/lib/mirrormanager/global_netblocks.txt'
+internet2_tree = radix.Radix()
+global_tree = radix.Radix()
+gipv4=None
+gipv6=None
+
+
+def generate_password():
+ # from http://en.wikipedia.org/wiki/Random_password_generator
+ myrg = random.SystemRandom()
+ length = 16
+ # If you want non-English characters, remove the [0:52]
+ alphabet = string.letters[0:52] + string.digits
+ pw = str().join(myrg.choice(alphabet) for _ in range(length))
+ return pw
+
+def setup_netblocks(netblocks_file):
+
+ tree = radix.Radix()
+ if netblocks_file is not None:
+ try:
+ f = open(netblocks_file, 'r')
+ except:
+ return tree
+ for l in f:
+ try:
+ s = l.split()
+ start, mask = s[0].split('/')
+ mask = int(mask)
+ if mask == 0: continue
+ asn = int(s[1])
+ node = tree.add(s[0])
+ node.data['asn'] = asn
+ except:
+ pass
+ f.close()
+
+ return tree
+
+internet2_tree = setup_netblocks(internet2_netblocks_file)
+global_tree = setup_netblocks(global_netblocks_file)
+
+def lookup_ip_asn(tree, ip):
+ """ @t is a radix tree
+ @ip is an IPy.IP object which may be contained in an entry in l
+ """
+ node = tree.search_best(str(ip))
+ if node is None:
+ return None
+ return node.data['asn']
+
+def internet2(ip):
+ asn = lookup_ip_asn(internet2_tree, ip)
+ if asn is not None:
+ return True
+ return False
+
+def asn(ip):
+ asn = lookup_ip_asn(global_tree, ip)
+ return asn
+
+def hostnames_from_urls(*urls):
+ hostnames = set()
+ for url in urls:
+ hostport,path = urllib.splithost(url)
+ host, port = urllib.splitnport(hostport)
+ hostnames.add(host)
+ return hostnames
+
+def ips_from_hostnames(hostnames):
+ result = set()
+ for hostname in hostnames:
+ ips = name_to_ips(hostname)
+ for ip in ips:
+ result.add(ip)
+ return result
+
+def urls_in_internet2(*urls):
+ rc = False
+ hostnames = hostnames_from_urls(urls)
+ ips = ips_from_hostnames(hostnames)
+ for ip in ips:
+ if internet2(ip):
+ rc = True
+ return rc
+
+def asn_from_urls(*urls):
+ result = set()
+ hostnames = hostnames_from_urls(urls)
+ ips = ips_from_hostnames(hostnames)
+ for ip in ips:
+ asn = asn(ip)
+ if asn is not None:
+ result.add(asn)
+
+ result = list(result)
+ if len(result) > 1:
+ print "hostnames %s has several ASNs" % hostnames
+ result = result[0]
+ elif len(result) == 1:
+ result = result[0]
+ else:
+ result = []
+ return result
+
+def open_geoip_databases():
+ global gipv4
+ global gipv6
+ try:
+ gipv4 = GeoIP.open("/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_STANDARD)
+ except:
+ gipv4=None
+ try:
+ gipv6 = GeoIP.open("/usr/share/GeoIP/GeoIPv6.dat", GeoIP.GEOIP_STANDARD)
+ except:
+ gipv6=None
+
+def convert_6to4_v4(ip):
+ all_6to4 = IP('2002::/16')
+ if ip.version() != 6 or ip not in all_6to4:
+ return None
+ parts=ip.strNormal().split(':')
+
+ ab = int(parts[1],16)
+ a = (ab >> 8) & 0xFF
+ b = ab & 0xFF
+ cd = int(parts[2],16)
+ c = (cd >> 8) & 0xFF
+ d = cd & 0xFF
+
+ v4addr = '%d.%d.%d.%d' % (a,b,c,d)
+ return IP(v4addr)
+
+def convert_teredo_v4(ip):
+ teredo_std = IP('2001::/32')
+ teredo_xp = IP('3FFE:831F::/32')
+ if ip.version() != 6 or (ip not in teredo_std and ip not in teredo_xp):
+ return None
+ parts=ip.strNormal().split(':')
+
+ ab = int(parts[6],16)
+ a = ((ab >> 8) & 0xFF) ^ 0xFF
+ b = (ab & 0xFF) ^ 0xFF
+ cd = int(parts[7],16)
+ c = ((cd >> 8) & 0xFF) ^ 0xFF
+ d = (cd & 0xFF) ^ 0xFF
+
+ v4addr = '%d.%d.%d.%d' % (a,b,c,d)
+ return IP(v4addr)
+
+def lookup_geoip(ip):
+ clientCounry = None
+ # attempt IPv6, then IPv6 6to4 as IPv4, then Teredo, then IPv4
+ try:
+ if ip.version() == 6:
+ if gipv6 is not None:
+ clientCountry = gipv6.country_code_by_addr_v6(ip.strNormal())
+ if clientCountry is None:
+ # Try the IPv6-to-IPv4 translation schemes
+ for scheme in (convert_6to4_v4, convert_teredo_v4):
+ result = scheme(ip)
+ if result is not None:
+ ip = result
+ break
+ if ip.version() == 4 and gipv4 is not None:
+ clientCountry = gipv4.country_code_by_addr(ip.strNormal())
+ except:
+ pass
+ return clientCountry
+
+def country_from_urls(*urls):
+ hostnames = hostnames_from_urls(urls)
+ ips = ips_from_hostnames(hostnames)
+ for ip in ips:
+ country = lookup_geoip(ip)
+ if country:
+ return country
+ return None
+
+# fixme make sure the CSV file has this header:
+# "site_name","http_url","ftp_url","rsync_url","bandwidth","display_name","user_name","site_url","country"
+# "LMU Muenchen, Dpt. Biologie 2, IT-Gruppe","http://centos.bio.lmu.de/","ftp://centos.bio.lmu.de/centos/","rsync://centos.bio.lmu.de/CentOS/","100Mbit","Ralph Angenendt","centos-mirror-lmu(a)strg-alt-entf.org","http://zi.bio.lmu.de/","de"
+#
+#These are the fields: Name of the mirror's sponsor, then the mirror URLs
+#(http,ftp and rsync). Next field is bandwidth (where we have *tons* of
+#mirrors without any entry - mostly older entries). Then "Contact Name",
+#"Contact Mail", "URL to sponsor" and the countrycode (here de for Germany).
+
+
+def process(row):
+ # Create User
+ # avoid duplicate users
+ try:
+ new_user = True
+ user = User(user_name=row['user_name'],
+ email_address = row['user_name'],
+ display_name = row['display_name'],
+ password = generate_password())
+ except:
+ user = User.byName(row['user_name'])
+ new_user = False
+ # Create Site
+ # avoid duplicate sites
+ try:
+ site = Site(name=row['site_name'],
+ password=generate_password(),
+ orgUrl=row['site_url'])
+ except:
+ site = Site.byName(row['site_name'])
+ # Create SiteAdmin
+ # if user and site were both duplicates, this could fail
+ try:
+ siteAdmin = SiteAdmin(user, site)
+ except:
+ pass
+ # Create Host
+ # fixme: be sure to filter out duplicate hosts from the CSV file before calling this program
+ try:
+ host = Host(name=hostname,
+ site=site,
+ country=row['country'].upper() or country_from_urls(row['http_url'], row['ftp_url'], row['rsync_url']),
+ bandwidth_int = row['bandwidth'],
+ internet2 = urls_in_internet2(row['http_url'], row['ftp_url'], row['rsync_url']),
+ asn = asn_from_urls(row['http_url'], row['ftp_url'], row['rsync_url']),
+ )
+ except:
+ # duplicate hosts means we've gone this far already somehow before.
+ # don't bother doing any more
+ return user, False
+ # Create HostCategory
+ category = Category.byName("CentOS")
+ host_category = host_category(host=host,
+ category=category,
+ upstream=centos_upstream)
+ # Create HostCategoryURLs
+ if row['http_url']:
+ http_url = HostCategoryUrl(host_category=host_category,
+ url = row['http_url'])
+ if row['ftp_url']:
+ ftp_url = HostCategoryUrl(host_category=host_category,
+ url = row['ftp_url'])
+ if row['rsync_url']:
+ rsync_url = HostCategoryUrl(host_category=host_category,
+ url = row['rsync_url'])
+ return user, new_user
+
+def email_user(user):
+ subject = "Welcome to the CentOS MirrorManager"
+ msg = """Thank you for your continued support of CentOS [1] by
+providing a public mirror for users worldwide.
+
+CentOS has chosen to use MirrorManager [2] to keep track of the list of
+mirrors. MirrorManager allows you as a mirror administrator to maintain
+information about your own mirror (your URLs, etc.) yourself. In this email
+you will find your personal username and password with which you may log into the
+CentOS MirrorManager instance and see and edit information about your mirror.
+
+MirrorManager web interface: http://mirrormanager.centos.org
+Username: %(user_name)s
+Password: %(password)s
+
+Mirrors should run the 'report_mirror' script, part of the mirrormanger-client
+RPM available from the EPEL (Extra Package for Enterprise Linux) repository,
+after each successful rsync run. Consult the CentOS wiki [3] for configuration
+information.
+
+You will start to see the MirrorManager crawler in your HTTP and FTP logs,
+checking on the status of your mirror. The crawler uses lightweight
+HTTP HEAD and FTP DIR calls to verify your mirror has content at the expected
+directory locations.
+
+If you have any questions, please direct them to the mailing list:
+CentOS-mirror(a)centos.org [4].
+
+Thanks,
+CentOS
+
+[1] http://centos.org
+[2] http://mirrormanager.org
+[3] http://wiki.centos.org/
+[4] http://lists.centos.org/mailman/listinfo/centos-mirror
+"""
+
+# Create a text/plain message
+ msg = MIMEText(msg % dict(user_name = user.user_name,
+ password = user.password))
+
+# me == the sender's email address
+# you == the recipient's email address
+ msg['Subject'] = subject
+ msg['From'] = 'centos-mirror-admin(a)centos.org'
+ msg['To'] = user.email_address
+
+# Send the message via our own SMTP server, but don't include the
+# envelope header.
+ s = smtplib.SMTP('localhost')
+ s.sendmail(me, [you], msg.as_string())
+ s.quit()
+
+
+def doit(options, args):
+ if options.filename == '-':
+ f = sys.stdin
+ else:
+ f = open(options.filename, 'r')
+ csv_reader = reader(f, dialect='excel')
+ for row in csv_reader:
+ user, new_user = process(row)
+ if new_user:
+ email_user(user)
+
+
+def main():
+ parser = OptionParser(usage=sys.argv[0] + " [options]")
+ parser.add_option("-c", "--config",
+ dest="config", default='/etc/mirrormanager/prod.cfg',
+ help="TurboGears config file to use")
+ parser.add_option("-f", "--file",
+ dest="filename", default=None,
+ help="file name to import")
+
+ (options, args) = parser.parse_args()
+ if not options.filename:
+ parser.print_help()
+ sys.exit(1)
+
+ turbogears.update_config(configfile=options.config,
+ modulename="mirrormanager.config")
+ global hub
+ global __connection__
+ global repos
+ hub = PackageHub("mirrormanager")
+ __connection__ = hub
+
+ doit(options, args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
+
11 years, 2 months
[mirrormanager] v1.4: generate-publiclist: bandwidth_int must be >=1, if None set to 1 (e215af0)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit e215af08d89780abbe6ca70ed261019e6e37cc81
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Feb 24 23:27:34 2013 -0600
generate-publiclist: bandwidth_int must be >=1, if None set to 1
>---------------------------------------------------------------
server/generate-publiclist | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/generate-publiclist b/server/generate-publiclist
index 73f2f75..ed9469f 100755
--- a/server/generate-publiclist
+++ b/server/generate-publiclist
@@ -162,7 +162,7 @@ def generate_one_html(engine, pvaMatrix, productname=None, vername=None, archnam
for h in hosts:
if not isinstance(h.bandwidth_int, int):
print '%s bandwidth_int was not an int: %s' % (h, h.bandwidth_int)
- h.bandwidth_int = 0
+ h.bandwidth_int = 1
bandwidth = bandwidth + h.bandwidth_int
bandwidth = bandwidth / 1024 # convert to Gb
11 years, 2 months
[mirrormanager] v1.4: mirrorlist.py: fix broken logic that lead to tracebacks (9c312c7)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 9c312c73358fe0303edade52db371282f5c72139
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Feb 24 22:14:23 2013 -0600
mirrorlist.py: fix broken logic that lead to tracebacks
>---------------------------------------------------------------
server/mirrormanager/mirrorlist.py | 31 +++++++++++++++----------------
1 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 1d3651a..09c391e 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -131,22 +131,21 @@ def populate_directory_cache():
if repo is not None and repo.arch is not None:
global_caches['repo_arch_to_directoryname'][(repo.prefix, repo.arch.name)] = directoryname
cache[directoryname]['ordered_mirrorlist'] = repo.version.ordered_mirrorlist # WARNING - this is a query # fixme use cache
- category_id = directory_category_cache[directory_id]
- else:
- numcats = len(directory_category_cache.get(directory_id, []))
- if numcats == 0:
- # no category, so we can't know a mirror host's URLs.
- # nothing to add.
- continue
- elif numcats >= 1:
- # any of them will do, so just look at the first one
- category_id = directory_category_cache[directory_id][0]
-
- # repodata/ directories aren't themselves repositories, their parent dir is
- # we're walking the list in order, so the parent will be added to the cache before the child
- if directoryname.endswith('/repodata'):
- parent = os.path.dirname(directoryname) # parent
- cache[directoryname]['ordered_mirrorlist'] = cache[parent]['ordered_mirrorlist']
+
+ numcats = len(directory_category_cache.get(directory_id, []))
+ if numcats == 0:
+ # no category, so we can't know a mirror host's URLs.
+ # nothing to add.
+ continue
+ elif numcats >= 1:
+ # any of them will do, so just look at the first one
+ category_id = directory_category_cache[directory_id][0]
+
+ # repodata/ directories aren't themselves repositories, their parent dir is
+ # we're walking the list in order, so the parent will be added to the cache before the child
+ if directoryname.endswith('/repodata'):
+ parent = os.path.dirname(directoryname) # parent
+ cache[directoryname]['ordered_mirrorlist'] = cache[parent]['ordered_mirrorlist']
cache[directoryname]['subpath'] = directoryname[category_topdir_cache[category_id]:]
del repo
11 years, 2 months
[mirrormanager] v1.4: mirrorlist.py: list()ify selects (ca363f9)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit ca363f98f06bc49b7587906d4593110c5a0f4392
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Feb 22 15:39:53 2013 -0600
mirrorlist.py: list()ify selects
This may speed up processing a little, as it materializes a few more
selects and joins which we know we are going to materialize shortly
thereafter anyhow. This may increase memory usage somewhat in the
generate_mirrorlist_cache process.
>---------------------------------------------------------------
server/mirrormanager/mirrorlist.py | 26 +++++++++++++-------------
1 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 1985b1a..1d3651a 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -97,7 +97,7 @@ def populate_directory_cache():
def setup_directory_repo_cache():
cache = {}
- for r in Repository.select():
+ for r in list(Repository.select()):
if r.directory:
cache[r.directory.id] = r
return cache
@@ -189,7 +189,7 @@ def name_to_ips(name):
def populate_netblock_cache(cache, host):
if host.is_active() and len(host.netblocks) > 0:
- for n in host.netblocks:
+ for n in list(host.netblocks):
try:
ip = IP(n.netblock)
ips = [ip]
@@ -204,7 +204,7 @@ def populate_netblock_cache(cache, host):
def populate_host_country_allowed_cache(cache, host):
if host.is_active() and len(host.countries_allowed) > 0:
- cache[host.id] = [c.country.upper() for c in host.countries_allowed]
+ cache[host.id] = [c.country.upper() for c in list(host.countries_allowed)]
return cache
@@ -232,25 +232,25 @@ def populate_host_asn_cache(cache, host):
if host.asn is not None:
append_value_to_cache(cache, host.asn, host.id)
- for peer_asn in host.peer_asns:
+ for peer_asn in list(host.peer_asns):
append_value_to_cache(cache, peer_asn.asn, host.id)
return cache
def repository_redirect_cache():
cache = {}
- for r in RepositoryRedirect.select():
+ for r in list(RepositoryRedirect.select()):
cache[r.fromRepo] = r.toRepo
return cache
def country_continent_redirect_cache():
cache = {}
- for c in CountryContinentRedirect.select():
+ for c in list(CountryContinentRedirect.select()):
cache[c.country] = c.continent
return cache
def disabled_repository_cache():
cache = {}
- for r in Repository.select():
+ for r in list(Repository.select()):
if r.disabled:
cache[r.prefix] = True
return cache
@@ -262,26 +262,26 @@ def file_details_cache():
for d in list(Directory.select()):
if len(d.fileDetails) > 0:
cache[d.name] = {}
- for fd in d.fileDetails:
+ for fd in list(d.fileDetails):
details = dict(timestamp=fd.timestamp, sha1=fd.sha1, md5=fd.md5, sha256=fd.sha256, sha512=fd.sha512, size=fd.size)
append_value_to_cache(cache[d.name], fd.filename, details)
return cache
def hcurl_cache():
cache = {}
- for hcurl in HostCategoryUrl.select():
+ for hcurl in list(HostCategoryUrl.select()):
cache[hcurl.id] = hcurl.url
return cache
def location_cache():
cache = {}
- for location in Location.select():
- cache[location.name] = [host.id for host in location.hosts]
+ for location in list(Location.select()):
+ cache[location.name] = [host.id for host in list(location.hosts)]
return cache
def netblock_country_cache():
cache = {}
- for n in NetblockCountry.select():
+ for n in list(NetblockCountry.select()):
ip = None
try:
ip = IP(n.netblock)
@@ -300,7 +300,7 @@ def populate_host_caches():
a = dict()
mc = dict()
- for host in Host.select():
+ for host in list(Host.select()):
n = populate_netblock_cache(n, host)
ca = populate_host_country_allowed_cache(ca, host)
b = populate_host_bandwidth_cache(b, host)
11 years, 2 months
[mirrormanager] v1.4: mirrorlist_server: catch SIGTERM and cleanup (bae5a85)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit bae5a853486162292bbefa81c8a188fb21ec7b0a
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Feb 12 22:45:56 2013 -0600
mirrorlist_server: catch SIGTERM and cleanup
Presently, any signal sent to the process will most likely interrupt
ss.serve_forever() in its select() routine, which raises an error.
This then kills the whole process, which is not what we want.
What we want is for serve_forever() to get restarted if SIGHUP comes
in, and for the program to cleanly exit if SIGTERM comes in. That's
what this patch does.
>---------------------------------------------------------------
mirrorlist-server/mirrorlist_server.py | 16 +++++++++++++++-
1 files changed, 15 insertions(+), 1 deletions(-)
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 1ce10c6..76e427b 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -12,6 +12,7 @@ import logging.handlers
import os
import random
import cPickle as pickle
+import select
import signal
import socket
from SocketServer import StreamRequestHandler, ForkingMixIn, UnixStreamServer, BaseServer
@@ -37,6 +38,7 @@ internet2_netblocks_file = '/var/lib/mirrormanager/i2_netblocks.txt'
global_netblocks_file = '/var/lib/mirrormanager/global_netblocks.txt'
logfile = None
debug = False
+must_die = False
# at a point in time when we're no longer serving content for versions
# that don't use yum prioritymethod=fallback
# (e.g. after Fedora 7 is past end-of-life)
@@ -849,6 +851,13 @@ def sighup_handler(signum, frame):
signal.signal(signal.SIGHUP, sighup_handler)
+def sigterm_handler(signum, frame):
+ global must_die
+ signal.signal(signal.SIGHUP, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ if signum == signal.SIGTERM:
+ must_die = True
+
class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer):
request_queue_size = 300
def finish_request(self, request, client_address):
@@ -943,7 +952,12 @@ def main():
load_databases_and_caches()
signal.signal(signal.SIGHUP, sighup_handler)
ss = ForkingUnixStreamServer(socketfile, MirrorlistHandler)
- ss.serve_forever()
+
+ while not must_die:
+ try:
+ ss.serve_forever()
+ except select.error:
+ pass
try:
os.unlink(socketfile)
11 years, 2 months
[mirrormanager] v1.4: mirrorlist_server: reload caches in a separate thread (8d8d59a)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 8d8d59ab1f6c2ac83d21d5f9bcb1da45693b4064
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Feb 12 21:46:46 2013 -0600
mirrorlist_server: reload caches in a separate thread
Right now mirrorlist_server reloads its caches when it gets a SIGHUP.
This can take over a minute, during which time no more connections
from mirrorlist_client.wsgi are accepted and processed. Clients could
even be returned HTTP 503 errors during this time.
This patch spawns a thread in the sighup_handler() to reload the
caches asynchronously. While it's running, clients can still connect
and be served from the current data. Only at the very end of the
reload process are the global cache references updated, and
thereafter, new client connections will fork() and get the new data.
>---------------------------------------------------------------
mirrorlist-server/mirrorlist_server.py | 40 ++++++++++++++++++++++++--------
1 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 1ff9c84..1ce10c6 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -4,18 +4,30 @@
# by Matt Domsch <Matt_Domsch(a)dell.com>
# Licensed under the MIT/X11 license
-from SocketServer import StreamRequestHandler, ForkingMixIn, UnixStreamServer, BaseServer
-import cPickle as pickle
-import os, sys, signal, random, socket, getopt
-from string import zfill, atoi
+# standard library modules in alphabetical order
import datetime
-import time
-import radix
+import getopt
import logging
import logging.handlers
+import os
+import random
+import cPickle as pickle
+import signal
+import socket
+from SocketServer import StreamRequestHandler, ForkingMixIn, UnixStreamServer, BaseServer
+import sys
+from string import zfill, atoi
+import time
+try:
+ import threading
+except ImportError:
+ import dummy_threading as threading
+
+# not-so-standard library modules that this program needs
from IPy import IP
import GeoIP
+import radix
from weighted_shuffle import weighted_shuffle
# can be overridden on the command line
@@ -824,12 +836,17 @@ def sighup_handler(signum, frame):
if signum == signal.SIGHUP:
if debug:
print "Got SIGHUP; reloading data"
- open_geoip_databases()
- read_caches()
if logfile is not None:
name = logfile.name
logfile.close()
logfile = open(name, 'a')
+
+ # put this in a separate thread so it doesn't block clients
+ if threading.active_count() < 2:
+ thread = threading.Thread(target=load_databases_and_caches)
+ thread.daemon = False
+ thread.start()
+
signal.signal(signal.SIGHUP, sighup_handler)
class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer):
@@ -910,6 +927,10 @@ def convert_teredo_v4(ip):
v4addr = '%d.%d.%d.%d' % (a,b,c,d)
return IP(v4addr)
+def load_databases_and_caches(*args, **kwargs):
+ open_geoip_databases()
+ read_caches()
+
def main():
global logfile
parse_args()
@@ -919,8 +940,7 @@ def main():
except:
pass
- open_geoip_databases()
- read_caches()
+ load_databases_and_caches()
signal.signal(signal.SIGHUP, sighup_handler)
ss = ForkingUnixStreamServer(socketfile, MirrorlistHandler)
ss.serve_forever()
11 years, 2 months
[mirrormanager] v1.4: model.py: fix RLIKE usage (e995677)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit e99567756ff1573c2f8413cd41d69ea7ad29d835
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Feb 4 22:33:18 2013 -0600
model.py: fix RLIKE usage
The previous attempt to make RLIKE vs ~ database agnostic made the
mistake of using the stock RLIKE sqlrepr implementation. This was
causing the database table.column referred to in the RLIKE statement
to instead be treated as the explicit constant string 'table.column'
which is not what the pattern should be testing against - it will
always fail.
This patch extends the stock RLIKE implementation, to not turn the
table.column reference into a constant string.
This routine is still really slow, on the order of 15 seconds per
product + regexp (version + architecture) pair. For 18+ versions * 11
architectures, this is _really_ slow. This routine alone could take
30-45 minutes against the Fedora tree.
>---------------------------------------------------------------
server/mirrormanager/model.py | 12 +++++++++---
1 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 32c3502..ec413d6 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -1,4 +1,5 @@
from sqlobject import *
+from sqlobject.converters import sqlrepr
from sqlobject.sqlbuilder import RLIKE
from turbogears import identity, config
from datetime import datetime
@@ -537,15 +538,20 @@ def _publiclist_hosts(product=None, re=None):
sql2_join = 'AND category_directory.directory_id = directory.id '
sql2_join += 'AND category_directory.category_id = category.id '
+ class MY_RLIKE(RLIKE):
+ def __sqlrepr__(self, db):
+ return "(%s %s (%s))" % (
+ sqlrepr(self.expr, db), self._get_op(db), self.string)
+
def _rlike(pattern, string):
# there's probably a beter way to get this, but this works...
_dburi = config.get('sqlobject.dburi', 'sqlite://')
dbtype = _dburi.strip('notrans_').split(':')[0]
- return RLIKE('pattern', 'bar').__sqlrepr__(dbtype) + " "
+ return MY_RLIKE(pattern, string).__sqlrepr__(dbtype) + " "
if re is not None:
- sql1_filter = "AND " + _rlike('host_category_dir.path', re)
- sql2_filter = "AND " + _rlike('directory.name', re)
+ sql1_filter = "AND " + _rlike(re, 'host_category_dir.path')
+ sql2_filter = "AND " + _rlike(re, 'directory.name')
sql1_up2date = 'AND host_category_dir.up2date '
sql2_up2date = 'AND host_category.always_up2date '
11 years, 2 months