[mirrormanager] v1.4: log mirrorlist_server stderr by supervisord (aa7caa0)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit aa7caa0513641e607bd5d45634cf8fcb62bed10d
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jan 31 16:43:42 2013 -0600
log mirrorlist_server stderr by supervisord
Right now when mirrorlist_server crashes or tracebacks, we don't get
anything in any logs. This patch causes supervisord to log stderr to
the logfile, which will help debug such crashes.
>---------------------------------------------------------------
mirrorlist-server/supervisor/mirrorlist-server.ini | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/mirrorlist-server/supervisor/mirrorlist-server.ini b/mirrorlist-server/supervisor/mirrorlist-server.ini
index c5aad22..7aeca58 100644
--- a/mirrorlist-server/supervisor/mirrorlist-server.ini
+++ b/mirrorlist-server/supervisor/mirrorlist-server.ini
@@ -8,3 +8,4 @@ startretries=5
stopsignal=TERM
stopwaitsecs=10
user=mirrormanager
+log_stderr=true
11 years, 2 months
[mirrormanager] v1.4: keep 1 old copy of /var/lib/mirrormanager in /var/lib/mirrormanager/old/ (67b5bd5)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 67b5bd584814d6f09493659ba98b569ec6fca6e7
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jan 31 13:43:20 2013 -0600
keep 1 old copy of /var/lib/mirrormanager in /var/lib/mirrormanager/old/
We have occasionally needed the previous copy of these contents, which
are otherwise overwritten. So, keep a copy.
>---------------------------------------------------------------
server/mirrormanager/mirrorlist.py | 4 ++--
server/refresh_mirrorlist_cache | 9 ++++++++-
server/update-mirrorlist-server | 4 ++++
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index ab91081..96bb5bc 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -323,7 +323,7 @@ def populate_all_caches():
populate_directory_cache()
import cPickle as pickle
-def dump_caches():
+def dump_caches(filename):
data = {'mirrorlist_cache':global_caches['mirrorlist_cache'],
'host_netblock_cache':global_caches['host_netblock_cache'],
'host_country_allowed_cache':global_caches['host_country_allowed_cache'],
@@ -341,7 +341,7 @@ def dump_caches():
'netblock_country_cache':netblock_country_cache()}
try:
- f = open('/var/lib/mirrormanager/mirrorlist_cache.pkl', 'w')
+ f = open(filename, 'w')
pickle.dump(data, f)
f.close()
except:
diff --git a/server/refresh_mirrorlist_cache b/server/refresh_mirrorlist_cache
index 06cb5b2..5139b28 100755
--- a/server/refresh_mirrorlist_cache
+++ b/server/refresh_mirrorlist_cache
@@ -14,16 +14,23 @@ import mirrormanager.zonecreator
from turbogears.database import PackageHub
hub = __connection__ = None
+options = None
+
def doit():
+ global options
populate_all_caches()
- dump_caches()
+ dump_caches(options.output)
mirrormanager.zonecreator.writeZoneFiles()
def main():
+ global options
parser = OptionParser(usage=sys.argv[0] + " [options]")
parser.add_option("-c", "--config",
dest="config", default='dev.cfg',
help="TurboGears config file to use")
+ parser.add_option("-o", "--output",
+ dest="output", default='/var/lib/mirrormanager/mirrorlist_cache.pkl',
+ help="output file")
(options, args) = parser.parse_args()
diff --git a/server/update-mirrorlist-server b/server/update-mirrorlist-server
index b49781a..b95eba5 100755
--- a/server/update-mirrorlist-server
+++ b/server/update-mirrorlist-server
@@ -11,6 +11,10 @@ echo $$ > ${LOCKFILE}
trap "rm -f ${LOCKFILE}" QUIT TERM INT HUP EXIT
cd ${MM_DIR}
+rm -rf ${CACHEDIR}/old
+mkdir -p ${CACHEDIR}/old
+cp -ar ${CACHEDIR}/* ${CACHEDIR}/old/
+
${MM_DIR}/update-static-content ${MIRRORLIST_DIR}
${MM_DIR}/refresh_mirrorlist_cache -c /etc/mirrormanager/prod.cfg
${MM_DIR}/get_internet2_netblocks ${CACHEDIR}/i2_netblocks.txt
11 years, 2 months
[mirrormanager] v1.4: Merge branch 'v1.4' of ssh+git://fedorahosted.org/git/mirrormanager into v1.4 (10b5add)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 10b5add9c53fa8f9db2b8dc98822732f0f7b555d
Merge: 5a83e95 e8527c0
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jan 31 12:55:46 2013 -0600
Merge branch 'v1.4' of ssh+git://fedorahosted.org/git/mirrormanager into v1.4
>---------------------------------------------------------------
mirrorlist-server/weighted_shuffle.py | 11 ++++++-----
server/mirrormanager/mirrorlist.py | 9 ++++++++-
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/mirrorlist-server/weighted_shuffle.py b/mirrorlist-server/weighted_shuffle.py
index 50dead3..0c26b07 100644
--- a/mirrorlist-server/weighted_shuffle.py
+++ b/mirrorlist-server/weighted_shuffle.py
@@ -6,14 +6,15 @@ import random
import bisect
class WeightedListItem:
- start = 0
- weight = 0
- data = None
-
def __init__(self, weight, data, start=0):
self.data = data
- self.weight = weight
self.start = start
+ self.weight = weight
+
+ if type(weight) != int:
+ self.weight = 1
+ elif weight < 1:
+ self.weight = 1
def __contains__(self, val):
return (val >= self.start and val < (self.start + self.weight))
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index bf1b346..ab91081 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -213,7 +213,14 @@ def populate_host_max_connections_cache(cache, host):
return cache
def populate_host_bandwidth_cache(cache, host):
- cache[host.id] = host.bandwidth_int
+ try:
+ i = int(host.bandwidth_int)
+ if i < 1: i = 1
+ elif i > 100000: i = 100000 # max bandwidth 100Gb
+ cache[host.id] = i
+ except:
+ cache[host.id] = 1
+
return cache
def populate_host_country_cache(cache, host):
11 years, 2 months
[mirrormanager] v1.4: use hashlib instead of deprecated sha (5a83e95)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 5a83e95074d06202b9d265cf3e66aa71b3647b2f
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Jan 29 12:35:25 2013 -0600
use hashlib instead of deprecated sha
>---------------------------------------------------------------
server/mirrormanager/mirrorlist.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 95f9360..bf1b346 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -1,7 +1,7 @@
from mirrormanager.model import Directory, Host, RepositoryRedirect, CountryContinentRedirect, Repository, HostCategoryUrl, Location, Version, Category, NetblockCountry, setup_directory_category_cache
import os
from IPy import IP
-import sha
+import hashlib
import pprint
import dns.resolver
from mirrormanager.lib import append_value_to_cache
@@ -61,7 +61,7 @@ def shrink(mc):
for d in mc:
for subcache in subcaches:
c = mc[d][subcache]
- s = sha.sha(pp.pformat(c))
+ s = hashlib.sha1(pp.pformat(c)).hexdigest()
if s in matches:
d[c] = matches[s]
else:
11 years, 2 months
[mirrormanager] v1.4: force 1 <= bandwidth_int <= 100000 in cache creation (e8527c0)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit e8527c01fe390484e5c272db4200052491979ee4
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 30 22:41:01 2013 -0600
force 1 <= bandwidth_int <= 100000 in cache creation
This handles invalid types as well as out-of-range values before they
can break mirrorlist_server.py. And really, who has servers on the
net with 100Gb pipes yet?
>---------------------------------------------------------------
server/mirrormanager/mirrorlist.py | 9 ++++++++-
1 files changed, 8 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 95f9360..ebfd300 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -213,7 +213,14 @@ def populate_host_max_connections_cache(cache, host):
return cache
def populate_host_bandwidth_cache(cache, host):
- cache[host.id] = host.bandwidth_int
+ try:
+ i = int(host.bandwidth_int)
+ if i < 1: i = 1
+ elif i > 100000: i = 100000 # max bandwidth 100Gb
+ cache[host.id] = i
+ except:
+ cache[host.id] = 1
+
return cache
def populate_host_country_cache(cache, host):
11 years, 2 months
[mirrormanager] v1.4: Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4 (7f0cf8f)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 7f0cf8fce588cf0ded945cb0bf9128dfbbf0944a
Merge: 1e74c9c d781cc4
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 30 22:16:47 2013 -0600
Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4
>---------------------------------------------------------------
server/mirrormanager/schema_updates/__init__.py | 27 +++++++++++++++-------
1 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index b6c070e..676fe82 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -14,19 +14,25 @@ __connection__ = hub
changes = {}
-def _bandwidth_int_not_null():
+# On Postgres at least, one cannot create a new column with notNull=True, even with default=<something>
+# because it will fail to create the column, claiming the new column is full of null values,
+# which violates the notNull constraint.
+# So, you have to create the column, fill it with the default value, and then
+# add the notNull=True constraint thereafter.
+
+def _set_not_null(table, col):
_dburi = config.get('sqlobject.dburi', '')
if 'mysql://' in _dburi:
- sql = 'ALTER TABLE host CHANGE COLUMN bandwidth_int bandwidth_int BIGINT NOT NULL'
+ sql = 'ALTER TABLE %s CHANGE COLUMN %s BIGINT NOT NULL' % (table.sqlmeta.table, col)
elif 'postgres://' in _dburi:
- sql = 'ALTER TABLE host ALTER COLUMN bandwidth_int SET NOT NULL'
- return OldHost._connection.query(sql)
+ sql = 'ALTER TABLE %s ALTER COLUMN %s SET NOT NULL' % (table.sqlmeta.table, col)
+ return table._connection.query(sql)
def bandwidth_int_not_null():
for h in OldHost.select():
if h.bandwidthInt is None: # note column name differs due to underscore->CamelCase conversion
h.bandwidthInt = 100
- _bandwidth_int_not_null()
+ _set_not_null(OldHost, 'bandwidth_int')
def _SODatabaseIndex_needs_creationOrder():
argspec = inspect.getargspec(SODatabaseIndex.__init__)
@@ -56,12 +62,12 @@ def change_tables():
OldHost.sqlmeta.delColumn("dnsCountryHost", changeSchema=True)
if 'maxConnections' not in OldHost.sqlmeta.columns:
- OldHost.sqlmeta.addColumn(IntCol("max_connections", default=1, notNone=True), changeSchema=True)
+ OldHost.sqlmeta.addColumn(IntCol("max_connections", default=1), changeSchema=True)
changes['host.max_connections'] = True
if 'sortorder' not in OldVersion.sqlmeta.columns and \
'codename' not in OldVersion.sqlmeta.columns:
- OldVersion.sqlmeta.addColumn(IntCol("sortorder", default=0, notNone=True), changeSchema=True)
+ OldVersion.sqlmeta.addColumn(IntCol("sortorder", default=0), changeSchema=True)
OldVersion.sqlmeta.addColumn(UnicodeCol("codename", default=None), changeSchema=True)
changes['version.sortorder_codename'] = True
@@ -77,12 +83,12 @@ def change_tables():
def _add_site_to_site_index():
if _SODatabaseIndex_needs_creationOrder():
idx = SODatabaseIndex(OldSiteToSite, 'username_idx',
- [dict(column='upstream_site'),
+ [dict(column='upstream_site_id'),
dict(column='username', length=UnicodeColKeyLength)],
0, unique=True)
else:
idx = SODatabaseIndex(OldSiteToSite, 'username_idx',
- [dict(column='upstream_site'),
+ [dict(column='upstream_site_id'),
dict(column='username', length=UnicodeColKeyLength)],
unique=True)
@@ -169,11 +175,14 @@ def fill_new_columns():
if changes.get('host.max_connections'):
for h in Host.select():
h.max_connections = 1
+ _set_not_null(OldHost, 'max_connections')
+
if changes.get('version.sortorder_codename'):
for v in Version.select():
v.sortorder = 0
v.codename = None
+ _set_not_null(OldVersion, 'sortorder')
if changes.get('product.publiclist'):
for p in Product.select():
11 years, 2 months
[mirrormanager] v1.4: handle bad weight values in weighted_shuffle (1e74c9c)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 1e74c9cbf2eed5ecda1918e1c153ebc64be676e8
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jan 30 22:14:22 2013 -0600
handle bad weight values in weighted_shuffle
We were getting invalid (None) weight values being passed to shuffle.
This leads to a TypeError trying to add int and None type objects.
This patch forces the weight values to be a positive integer, and
invalid values are set to 1.
>---------------------------------------------------------------
mirrorlist-server/weighted_shuffle.py | 11 ++++++-----
1 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/mirrorlist-server/weighted_shuffle.py b/mirrorlist-server/weighted_shuffle.py
index 50dead3..0c26b07 100644
--- a/mirrorlist-server/weighted_shuffle.py
+++ b/mirrorlist-server/weighted_shuffle.py
@@ -6,14 +6,15 @@ import random
import bisect
class WeightedListItem:
- start = 0
- weight = 0
- data = None
-
def __init__(self, weight, data, start=0):
self.data = data
- self.weight = weight
self.start = start
+ self.weight = weight
+
+ if type(weight) != int:
+ self.weight = 1
+ elif weight < 1:
+ self.weight = 1
def __contains__(self, val):
return (val >= self.start and val < (self.start + self.weight))
11 years, 2 months
[mirrormanager] v1.4: 2 fixes to schema updates (d781cc4)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit d781cc4d85095b8d8ce95bfdafc455616e0a0226
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Jan 29 12:25:45 2013 -0600
2 fixes to schema updates
1) refactor notNull=True constraints, as Postgres can't create a new notNull column
but can upgrade a column to notNull if it's been filled with non-null values already.
2) fix column names hardcoded in one place
>---------------------------------------------------------------
server/mirrormanager/schema_updates/__init__.py | 27 +++++++++++++++-------
1 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/server/mirrormanager/schema_updates/__init__.py b/server/mirrormanager/schema_updates/__init__.py
index b6c070e..676fe82 100644
--- a/server/mirrormanager/schema_updates/__init__.py
+++ b/server/mirrormanager/schema_updates/__init__.py
@@ -14,19 +14,25 @@ __connection__ = hub
changes = {}
-def _bandwidth_int_not_null():
+# On Postgres at least, one cannot create a new column with notNull=True, even with default=<something>
+# because it will fail to create the column, claiming the new column is full of null values,
+# which violates the notNull constraint.
+# So, you have to create the column, fill it with the default value, and then
+# add the notNull=True constraint thereafter.
+
+def _set_not_null(table, col):
_dburi = config.get('sqlobject.dburi', '')
if 'mysql://' in _dburi:
- sql = 'ALTER TABLE host CHANGE COLUMN bandwidth_int bandwidth_int BIGINT NOT NULL'
+ sql = 'ALTER TABLE %s CHANGE COLUMN %s BIGINT NOT NULL' % (table.sqlmeta.table, col)
elif 'postgres://' in _dburi:
- sql = 'ALTER TABLE host ALTER COLUMN bandwidth_int SET NOT NULL'
- return OldHost._connection.query(sql)
+ sql = 'ALTER TABLE %s ALTER COLUMN %s SET NOT NULL' % (table.sqlmeta.table, col)
+ return table._connection.query(sql)
def bandwidth_int_not_null():
for h in OldHost.select():
if h.bandwidthInt is None: # note column name differs due to underscore->CamelCase conversion
h.bandwidthInt = 100
- _bandwidth_int_not_null()
+ _set_not_null(OldHost, 'bandwidth_int')
def _SODatabaseIndex_needs_creationOrder():
argspec = inspect.getargspec(SODatabaseIndex.__init__)
@@ -56,12 +62,12 @@ def change_tables():
OldHost.sqlmeta.delColumn("dnsCountryHost", changeSchema=True)
if 'maxConnections' not in OldHost.sqlmeta.columns:
- OldHost.sqlmeta.addColumn(IntCol("max_connections", default=1, notNone=True), changeSchema=True)
+ OldHost.sqlmeta.addColumn(IntCol("max_connections", default=1), changeSchema=True)
changes['host.max_connections'] = True
if 'sortorder' not in OldVersion.sqlmeta.columns and \
'codename' not in OldVersion.sqlmeta.columns:
- OldVersion.sqlmeta.addColumn(IntCol("sortorder", default=0, notNone=True), changeSchema=True)
+ OldVersion.sqlmeta.addColumn(IntCol("sortorder", default=0), changeSchema=True)
OldVersion.sqlmeta.addColumn(UnicodeCol("codename", default=None), changeSchema=True)
changes['version.sortorder_codename'] = True
@@ -77,12 +83,12 @@ def change_tables():
def _add_site_to_site_index():
if _SODatabaseIndex_needs_creationOrder():
idx = SODatabaseIndex(OldSiteToSite, 'username_idx',
- [dict(column='upstream_site'),
+ [dict(column='upstream_site_id'),
dict(column='username', length=UnicodeColKeyLength)],
0, unique=True)
else:
idx = SODatabaseIndex(OldSiteToSite, 'username_idx',
- [dict(column='upstream_site'),
+ [dict(column='upstream_site_id'),
dict(column='username', length=UnicodeColKeyLength)],
unique=True)
@@ -169,11 +175,14 @@ def fill_new_columns():
if changes.get('host.max_connections'):
for h in Host.select():
h.max_connections = 1
+ _set_not_null(OldHost, 'max_connections')
+
if changes.get('version.sortorder_codename'):
for v in Version.select():
v.sortorder = 0
v.codename = None
+ _set_not_null(OldVersion, 'sortorder')
if changes.get('product.publiclist'):
for p in Product.select():
11 years, 2 months
[mirrormanager] v1.4: handle CentOS mirrorlist parameters (d017f91)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit d017f9119f58affd91b6754199b8d408db9f466b
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jan 28 11:33:32 2013 -0600
handle CentOS mirrorlist parameters
>---------------------------------------------------------------
mirrorlist-server/mirrorlist_client.wsgi | 12 +++++++++++-
1 files changed, 11 insertions(+), 1 deletions(-)
diff --git a/mirrorlist-server/mirrorlist_client.wsgi b/mirrorlist-server/mirrorlist_client.wsgi
index 148b975..db492ca 100755
--- a/mirrorlist-server/mirrorlist_client.wsgi
+++ b/mirrorlist-server/mirrorlist_client.wsgi
@@ -70,7 +70,7 @@ def real_client_ip(xforwardedfor):
return xforwardedfor.split(',')[-1].strip()
def request_setup(environ, request):
- fields = ['repo', 'arch', 'country', 'path', 'netblock', 'location']
+ fields = ['repo', 'arch', 'country', 'path', 'netblock', 'location', 'version', 'cc']
d = {}
request_data = request.GET
for f in fields:
@@ -88,6 +88,16 @@ def request_setup(environ, request):
client_ip = request.environ['REMOTE_ADDR']
d['client_ip'] = client_ip
+ # convert cc to country (for CentOS)
+ if 'cc' in d and 'country' not in d:
+ d['country'] = d['cc']
+ del d['cc']
+
+ # convert version=&repo=& to repo=<repo>-<version> (for CentOS)
+ if 'version' in d and 'repo' in d:
+ d['repo'] = "%s-%s" % (d['repo'], d['version']))
+ del d['version']
+
d['metalink'] = False
scriptname = ''
pathinfo = ''
11 years, 2 months