[mirrormanager] v1.4: write tracebacks to stderr (93a2f9a)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 93a2f9af1a0f485ae0732d3a2b348526db293fdb
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Jun 8 23:05:22 2013 -0500
write tracebacks to stderr
If do_mirrorlist() tracebacks, we have not had a log of such to help
debug the failure. This patch makes us write that to stderr, which
something (e.g. supervisord) will write into a logfile we can use to
debug further.
>---------------------------------------------------------------
mirrorlist-server/mirrorlist_server.py | 5 +++++
1 files changed, 5 insertions(+), 0 deletions(-)
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 12a7788..4fbcc1c 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -19,6 +19,7 @@ from SocketServer import StreamRequestHandler, ForkingMixIn, UnixStreamServer, B
import sys
from string import zfill, atoi
import time
+import traceback
try:
import threading
@@ -824,6 +825,10 @@ class MirrorlistHandler(StreamRequestHandler):
returncode = r['returncode']
except Exception, e:
message=u'# Bad Request %s\n# %s' % (e, d)
+ exception_msg = traceback.format_exc(e)
+ sys.stderr.write(message+'\n')
+ sys.stderr.write(exception_msg)
+ sys.stderr.flush()
returncode = 400
results = []
resulttype = 'mirrorlist'
11 years
[mirrormanager] v1.4: Fix sorting results by preferred netblock size (e736171)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit e7361710d9785e4396c252af7c4fbcd63ea0b974
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Jun 8 22:58:10 2013 -0500
Fix sorting results by preferred netblock size
In a previous commit, we stopped trying to sort the netblock_results
list, and left it up to a random shuffle. We instead really want to
sort this list from smallest (most precise) netblock size to largest,
so that an individual company's mirror may be preferred over that of
the company's upstream ISP. This patch fixes the return of the data
from the tree_lookup() function to recognize the node data may be a
list or a singleton and handle accordingly.
>---------------------------------------------------------------
mirrorlist-server/mirrorlist_server.py | 50 +++++++++++++++++++-------------
1 files changed, 30 insertions(+), 20 deletions(-)
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 2519f0d..12a7788 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -212,14 +212,21 @@ def tree_lookup(tree, ip, field, maxResults=None):
# fast lookup in the tree; if present, find all the matching values by deleting the found one and searching again
# this is safe w/o copying the tree again only because this is the only place the tree is used, and
# we'll get a new copy of the tree from our parent the next time it fork()s.
+ # returns a list of tuples (prefix, data)
result = []
+ len_data = 0
if ip is None:
return result
node = tree.search_best(ip.strNormal())
while node is not None:
- result.extend(node.data[field])
prefix = node.prefix
- if maxResults is None or len(result) < maxResults:
+ if type(node.data[field]) == list:
+ len_data += len(node.data[field])
+ else:
+ len_data += 1
+ t = (prefix, node.data[field],)
+ result.append(t)
+ if maxResults is None or len_data < maxResults:
tree.delete(prefix)
node = tree.search_best(ip.strNormal())
else:
@@ -316,11 +323,12 @@ def do_country(kwargs, cache, clientCountry, requested_countries, header):
def do_netblocks(kwargs, cache, header):
hostresults = set()
if not kwargs.has_key('netblock') or kwargs['netblock'] == "1":
- hosts = tree_lookup(host_netblocks_tree, kwargs['IP'], 'hosts')
- for hostid in hosts:
- if hostid in cache['byHostId']:
- hostresults.add(hostid)
- header += 'Using preferred netblock '
+ tree_results = tree_lookup(host_netblocks_tree, kwargs['IP'], 'hosts')
+ for (prefix, hostids) in tree_results:
+ for hostid in hostids:
+ if hostid in cache['byHostId']:
+ hostresults.add((prefix, hostid,))
+ header += 'Using preferred netblock '
return (header, hostresults)
def do_internet2(kwargs, cache, clientCountry, header):
@@ -424,9 +432,9 @@ def client_ip_to_country(ip):
return None
# lookup in the cache first
- result = tree_lookup(netblock_country_tree, ip, 'country', maxResults=1)
- if len(result) > 0:
- clientCountry = result[0]
+ tree_results = tree_lookup(netblock_country_tree, ip, 'country', maxResults=1)
+ if len(tree_results) > 0:
+ (prefix, clientCountry) = tree_results[0]
return clientCountry
# attempt IPv6, then IPv6 6to4 as IPv4, then Teredo, then IPv4
@@ -604,25 +612,25 @@ def do_mirrorlist(kwargs):
random.shuffle(l)
return l
- # FIXME this is unused. Need to figure out how to retain the object that was the netblock that we found this host in
- # during the tree_lookup() above, so we know how big each netblock containing this item was, so we can later
- # sort on the size of the netblock. But that's a longer-term goal. For now, ignore all this and just randomize the list.
def _ordered_netblocks(s):
- def ipy_len(ip):
- return ip.len()
+ def ipy_len(t):
+ (prefix, hostid) = t
+ return IP(prefix).len()
v4_netblocks = []
v6_netblocks = []
- for hostid in s:
- ip = IP(n) # FIXME n is undefined to make this fail
+ for (prefix, hostid) in s:
+ ip = IP(prefix)
if ip.version() == 4:
- v4_netblocks.append(ip)
+ v4_netblocks.append((prefix, hostid))
elif ip.version() == 6:
- v6_netblocks.append(ip)
+ v6_netblocks.append((prefix, hostid))
# mix up the order, as sort will preserve same-key ordering
random.shuffle(v4_netblocks)
v4_netblocks.sort(key=ipy_len)
random.shuffle(v6_netblocks)
v6_netblocks.sort(key=ipy_len)
+ v4_netblocks = [t[1] for t in v4_netblocks]
+ v6_netblocks = [t[1] for t in v6_netblocks]
return v6_netblocks + v4_netblocks
def whereismymirror(result_sets):
@@ -641,7 +649,7 @@ def do_mirrorlist(kwargs):
result_sets = [
(location_results, "location", _random_shuffle),
- (netblock_results, "netblocks", _random_shuffle),
+ (netblock_results, "netblocks", _ordered_netblocks),
(asn_results, "asn", _random_shuffle),
(internet2_results, "I2", _random_shuffle),
(country_results, "country", shuffle),
@@ -769,7 +777,9 @@ def read_caches():
internet2_tree = setup_netblocks(internet2_netblocks_file)
global_tree = setup_netblocks(global_netblocks_file)
+ # host_netblocks_tree key is a netblock, value is a list of host IDs
host_netblocks_tree = setup_cache_tree(host_netblock_cache, 'hosts')
+ # netblock_country_tree key is a netblock, value is a single country string
netblock_country_tree = setup_cache_tree(netblock_country_cache, 'country')
def errordoc(metalink, message):
11 years
[mirrormanager] v1.4: bump release (d671bc2)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit d671bc2989b26243e3f0e96216fbaceced8aa61e
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Jun 7 14:04:52 2013 -0500
bump release
>---------------------------------------------------------------
mirrormanager.spec.in | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/mirrormanager.spec.in b/mirrormanager.spec.in
index 01e6582..15adda6 100644
--- a/mirrormanager.spec.in
+++ b/mirrormanager.spec.in
@@ -1,6 +1,6 @@
Name: mirrormanager
Version: ##VERSION##
-Release: 0.2%{?dist}
+Release: 0.4%{?dist}
Summary: Fedora mirror management system
BuildArch: noarch
@@ -102,8 +102,8 @@ exit 0
%doc LICENSES LICENSE-GPLv2
%changelog
-* Mon Feb 25 2013 Matt Domsch <mdomsch(a)fedoraproject.org> - 1.4.0-0.2
-- bug fixes on the 1.4 branch
+* Fri Jun 7 2013 Matt Domsch <mdomsch(a)fedoraproject.org> - 1.4.0-0.4
+- more bug fixes
* Fri Jan 25 2013 Matt Domsch <mdomsch(a)fedoraproject.org> - 1.4.0-0.1
- initial 1.4 branch for testing
11 years
[mirrormanager] v1.4: publiclist: remove comment about hourly generation (2d711e3)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 2d711e32429d7bf7365d9a9d4a2931f2da071e08
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Jun 7 13:28:09 2013 -0500
publiclist: remove comment about hourly generation
>---------------------------------------------------------------
server/mirrormanager/templates/publiclist.kid | 1 -
1 files changed, 0 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/templates/publiclist.kid b/server/mirrormanager/templates/publiclist.kid
index 91112d1..35028ef 100644
--- a/server/mirrormanager/templates/publiclist.kid
+++ b/server/mirrormanager/templates/publiclist.kid
@@ -42,7 +42,6 @@ be possible without the donations of time, disk space, and bandwidth
by hundreds of volunteer system administrators and their companies or
institutions. Your fast download experience is made possible by these
donations.
-This list is dynamically generated every hour, listing only up-to-date mirrors.
</p>
<p>
To become a public ${tg.config('mirrormanager.projectname','Fedora')} mirror, please see <a
11 years
[mirrormanager] v1.4: cron: add update-static-content daily job (573018b)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 573018b885fed18cfa849e13e12b5396a290e11b
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Jun 7 13:24:34 2013 -0500
cron: add update-static-content daily job
>---------------------------------------------------------------
cron/mirrormanager.cron | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)
diff --git a/cron/mirrormanager.cron b/cron/mirrormanager.cron
index 6df7945..8779795 100644
--- a/cron/mirrormanager.cron
+++ b/cron/mirrormanager.cron
@@ -4,6 +4,9 @@
# refresh the mirrorlist cache at the top of the hour
55 * * * * mirrormanager /usr/share/mirrormanager/server/update-mirrorlist-server >> /var/log/mirrormanager/mirrorlist.log 2>&1
+# refresh the publiclist static pages daily
+55 0 * * * mirrormanager /usr/share/mirrormanager/server/update-static-content >> /var/log/mirrormanager/static-content.log 2>&1
+
# update master directory list
0 */2 * * * mirrormanager cd /usr/share/mirrormanager/server && ./update-master-directory-list -c /etc/mirrormanager/prod.cfg >> /var/log/mirrormanager/umdl.log 2>&1
11 years
[mirrormanager] v1.4: move update-static-content into its own cronjob (e5baa9f)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit e5baa9f6720766b4bfd4c9ff08b46f4e6f249673
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Jun 7 13:21:34 2013 -0500
move update-static-content into its own cronjob
Instead of running at the same time as update-mirrorlist-server,
update-static-content should run on its own schedule, e.g. daily
(instead of hourly), as it takes quite a while to run, and the content
doesn't change that often, daily should be just fine.
>---------------------------------------------------------------
server/update-mirrorlist-server | 2 --
server/update-static-content | 18 ++++++++++++++----
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/server/update-mirrorlist-server b/server/update-mirrorlist-server
index b95eba5..73bc7db 100755
--- a/server/update-mirrorlist-server
+++ b/server/update-mirrorlist-server
@@ -1,7 +1,6 @@
#!/bin/bash
MM_DIR=/usr/share/mirrormanager/server
-MIRRORLIST_DIR=/var/lib/mirrormanager/mirrorlists
LOCKFILE=/var/lock/mirrormanager/update-mirrorlist-server.lock
CACHEDIR=/var/lib/mirrormanager
@@ -15,7 +14,6 @@ rm -rf ${CACHEDIR}/old
mkdir -p ${CACHEDIR}/old
cp -ar ${CACHEDIR}/* ${CACHEDIR}/old/
-${MM_DIR}/update-static-content ${MIRRORLIST_DIR}
${MM_DIR}/refresh_mirrorlist_cache -c /etc/mirrormanager/prod.cfg
${MM_DIR}/get_internet2_netblocks ${CACHEDIR}/i2_netblocks.txt
killall -HUP mirrorlist_server.py
diff --git a/server/update-static-content b/server/update-static-content
index 7d137ba..e8a5be3 100755
--- a/server/update-static-content
+++ b/server/update-static-content
@@ -1,11 +1,21 @@
#!/bin/bash
-PUBLICLIST_DIR="$1"
-[ -z "${PUBLICLIST_DIR}" ] && exit 1
-mkdir -p "${PUBLICLIST_DIR}"
+LOCKFILE=/var/lock/mirrormanager/update-static-content.lock
+PUBLICLIST_DIR=/var/lib/mirrormanager/mirrorlists
+
+[ -e ${LOCKFILE} ] && kill -0 $(cat ${LOCKFILE}) && exit 2
+mkdir -p $(dirname ${LOCKFILE})
+echo $$ > ${LOCKFILE}
+trap "cleanup" QUIT TERM INT HUP EXIT
+function cleanup()
+{
+ [ -n "$temp_dir_name" ] && rm -rf $temp_dir_name
+ rm -f ${LOCKFILE}
+}
+
+mkdir -p "${PUBLICLIST_DIR}"
temp_dir_name=`mktemp -d /tmp/mirrorlists.XXXXXX`
-trap "rm -rf $temp_dir_name" QUIT TERM INT HUP EXIT
pushd $temp_dir_name > /dev/null 2>&1
/usr/share/mirrormanager/server/generate-publiclist -c /etc/mirrormanager/prod.cfg -o ${temp_dir_name}/publiclist
11 years
[mirrormanager] v1.4: fix rlike syntax - it was backwards (69b7e03)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 69b7e03ec023466a68d4aaa9ab3929321eae8905
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Jun 7 13:12:32 2013 -0500
fix rlike syntax - it was backwards
>---------------------------------------------------------------
server/mirrormanager/model.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 37bebff..f098b45 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -541,7 +541,7 @@ def _publiclist_hosts(product=None, re=None):
class MY_RLIKE(RLIKE):
def __sqlrepr__(self, db):
return "(%s %s (%s))" % (
- sqlrepr(self.expr, db), self._get_op(db), self.string)
+ self.string, self._get_op(db), sqlrepr(self.expr, db))
def _rlike(pattern, string):
# there's probably a beter way to get this, but this works...
11 years
[mirrormanager] v1.4: mirrorlist_server: remove _ordered_netblocks() usage (fb02391)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit fb02391358377dbc692cf0b0011dd9b444589a89
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Jun 7 11:02:06 2013 -0500
mirrorlist_server: remove _ordered_netblocks() usage
_ordered_netblocks() usage is broken. It is taking a hostid, and
calling IP() on it, which is plain wrong.
The goal of _ordered_netblocks() is to, given a list of (hostid,
netblock), where netblock is the block that contains the client's IP
address that we're looking up, we want to later be able to sort that
list by the size of the netblock. However, do_netblocks() isn't
returning us such a list, it's only returning the list of hostids.
So, we need to fix do_netblocks(), then we can fix
_ordered_netblocks(). We will do neither here.
This patch drops the use of _ordered_netblocks(), reverting to the
v1.3 behavior of simply randomizing the list of hosts that claim a
given client IP address in its netblocks. We can revisit the better
fix later.
>---------------------------------------------------------------
mirrorlist-server/mirrorlist_server.py | 9 ++++++---
1 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 76e427b..2519f0d 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -604,13 +604,16 @@ def do_mirrorlist(kwargs):
random.shuffle(l)
return l
+ # FIXME this is unused. Need to figure out how to retain the object that was the netblock that we found this host in
+ # during the tree_lookup() above, so we know how big each netblock containing this item was, so we can later
+ # sort on the size of the netblock. But that's a longer-term goal. For now, ignore all this and just randomize the list.
def _ordered_netblocks(s):
def ipy_len(ip):
return ip.len()
v4_netblocks = []
v6_netblocks = []
- for n in s:
- ip = IP(n)
+ for hostid in s:
+ ip = IP(n) # FIXME n is undefined to make this fail
if ip.version() == 4:
v4_netblocks.append(ip)
elif ip.version() == 6:
@@ -638,7 +641,7 @@ def do_mirrorlist(kwargs):
result_sets = [
(location_results, "location", _random_shuffle),
- (netblock_results, "netblocks", _ordered_netblocks),
+ (netblock_results, "netblocks", _random_shuffle),
(asn_results, "asn", _random_shuffle),
(internet2_results, "I2", _random_shuffle),
(country_results, "country", shuffle),
11 years
[mirrormanager] v1.4: model: fix GROUP BY clause for postgres use (f7b6886)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit f7b6886c3a9932c5ba95e7c434bf40c3cfdb82b8
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jun 6 22:57:06 2013 -0500
model: fix GROUP BY clause for postgres use
>---------------------------------------------------------------
server/mirrormanager/model.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index ec413d6..37bebff 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -163,7 +163,7 @@ def all_sites_and_hosts():
sql += 'FROM site, host '
sql += 'WHERE '
sql += 'host.site_id = site.id '
- sql += 'GROUP BY site.id '
+ sql += 'GROUP BY site.id, site.name, host.id, host.name '
sql += 'ORDER BY site.name, host.name '
result = Site._connection.queryAll(sql)
return result
11 years
[mirrormanager] v1.4: fix setting host.max_connections not null (d956dd8)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit d956dd85c31f8af64553e7e7c5ca2db5277f1262
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jun 6 22:36:00 2013 -0500
fix setting host.max_connections not null
>---------------------------------------------------------------
server/mirrormanager/schema_updates/__init__.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index eb766c7..551589b 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -175,7 +175,7 @@ def fill_new_columns():
if changes.get('host.max_connections'):
for h in Host.select():
h.max_connections = 1
- _set_not_null(OldHost, 'max_connections')
+ _set_not_null(OldHost, 'max_connections')
if changes.get('version.sortorder_codename'):
11 years