Branch 'v1.4' - 2 commits - server/dev.cfg server/mirrormanager
by Matt Domsch
server/dev.cfg | 10 +++++++++-
server/mirrormanager/ubuntu.py | 22 ++++++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
New commits:
commit 3a15ddc0aac84bf1a30c1ed910be7c1f13d09f8e
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Oct 30 17:02:34 2010 -0500
add ubuntu initialization code
diff --git a/server/mirrormanager/ubuntu.py b/server/mirrormanager/ubuntu.py
new file mode 100644
index 0000000..79f8a9a
--- /dev/null
+++ b/server/mirrormanager/ubuntu.py
@@ -0,0 +1,22 @@
+from mirrormanager.model import Product, Category, Arch, Directory
+
+productname = 'Ubuntu'
+categories = {'Ubuntu Archive':('ubuntu','http://archive.ubuntu.com'),
+ 'Ubuntu CD Images':('releases', 'http://releases.ubuntu.com'),
+ 'Ubuntu Ports Archive':('ports', 'http://ports.ubuntu.com'),
+ 'Ubuntu Security Archive':('security', 'http://security.ubuntu.com'),
+ }
+primary_arches = ('i386', 'amd64')
+secondary_arches = ('armel', 'powerpc', 'ia64', 'sparc', 'hppa', 'lpia')
+
+def setup_ubuntu():
+ product = Product(name=productname)
+
+ for name, (dirname, canonicalhost) in categories.iteritems():
+ d = Directory(name=dirname)
+ Category(product=product, name=name, canonicalhost=canonicalhost, topdir=d)
+
+ for a in primary_arches:
+ Arch(name=a, primaryArch=True)
+ for a in secondary_arches:
+ Arch(name=a, primaryArch=False)
commit c3304d9ca68aa004c50b6ff1076e113553b33aae
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sat Oct 30 16:59:46 2010 -0500
dev.cfg: add ubuntu to umdl
diff --git a/server/dev.cfg b/server/dev.cfg
index 962453d..0c92d13 100644
--- a/server/dev.cfg
+++ b/server/dev.cfg
@@ -44,7 +44,15 @@ umdl.master_directories = ''' [
{ 'type':'rsync', 'url':'rsync://secondary1.fedora.phx.redhat.com/alt/',
'category':'Fedora Other' },
{ 'type':'directory', 'path':'/pub/fedora/linux/',
- 'category':'Fedora Linux', 'excludes':['.*/core/?.*', '.*/extras/?.*', '.*/[7-8]/?.*' ] }
+ 'category':'Fedora Linux', 'excludes':['.*/core/?.*', '.*/extras/?.*', '.*/[7-8]/?.*' ] },
+# { 'type':'rsync', 'url':'rsync://archive.ubuntu.com/ubuntu/',
+# 'category':'Ubuntu Archive' },
+# { 'type':'rsync', 'url':'rsync://releases.ubuntu.com/releases/',
+# 'category':'Ubuntu CD Images' },
+# { 'type':'rsync', 'url':'rsync://ports.ubuntu.com/ubuntu-ports/',
+# 'category':'Ubuntu Ports Archive' },
+# { 'type':'rsync', 'url':'rsync://security.ubuntu.com/ubuntu/',
+# 'category':'Ubuntu Security Archive' },
] '''
# manage-repo-redirects (mrr) repository definition
13 years, 6 months
Branch 'v1.4' - Makefile
by Matt Domsch
Makefile | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
New commits:
commit f72f20e244c1d4c909bb752d20208016e2e27549
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Fri Oct 29 13:52:32 2010 -0500
remove pushd bashism from Makefile
diff --git a/Makefile b/Makefile
index 694d519..78aa713 100644
--- a/Makefile
+++ b/Makefile
@@ -28,8 +28,7 @@ $(RELEASE_PY):
sed -e 's/##VERSION##/$(RELEASE_VERSION)/' $(RELEASE_PY).in > $(RELEASE_PY)
prep: $(SPEC) $(RELEASE_PY)
- pushd server ;\
- python setup.py egg_info
+ chdir server ; python setup.py egg_info
echo 'db_module=mirrormanager.model' > server/mirrormanager.egg-info/sqlobject.txt
echo 'history_dir=$$base/mirrormanager/sqlobject-history' >> server/mirrormanager.egg-info/sqlobject.txt
sync ; sync ; sync
13 years, 6 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/lib.py | 2 ++
1 file changed, 2 insertions(+)
New commits:
commit 4ff7a71611e4a0b4562c140d32838904febadd81
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Oct 18 10:33:50 2010 -0500
handle malformed pidfiles
diff --git a/server/mirrormanager/lib.py b/server/mirrormanager/lib.py
index 8c75cbd..1b0f325 100644
--- a/server/mirrormanager/lib.py
+++ b/server/mirrormanager/lib.py
@@ -63,6 +63,8 @@ def manage_pidfile(pidfile):
# is the oldpid process still running?
try:
os.kill(int(oldpid), 0)
+ except ValueError: # malformed oldpid
+ return write_pidfile(pidfile, pid)
except OSError, err:
if err.errno == 3: # No such process
return write_pidfile(pidfile, pid)
13 years, 6 months
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 10 ----------
1 file changed, 10 deletions(-)
New commits:
commit aae2ac8bd620a257e0316ce2ff3943e0ef415667
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Oct 18 10:16:02 2010 -0500
remove dead code
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index ac35cb1..4340d7e 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -214,16 +214,6 @@ def shuffle(s):
results.append(hostid)
return results
-def append_filename_to_results(file, hosts_and_urls):
- if file is None:
- return hosts_and_urls
- results = {}
- for hostid in host_and_urls.keys():
-
- hcurl = hcurl + '/%s' % (file)
- newresults.append((hostid, hcurl))
- return newresults
-
continents = {}
def handle_country_continent_redirect():
13 years, 6 months
Branch 'v1.4' - mirrorlist-server/mirrorlist_server.py
by Matt Domsch
mirrorlist-server/mirrorlist_server.py | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
New commits:
commit 447d0a6e286df6672f22a31d507c97ae79411d29
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Oct 18 09:50:43 2010 -0500
mirrorlist_server: return hosts in netblocks in ascending netblock size order
Fixes https://fedorahosted.org/mirrormanager/ticket/29
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 5ebe198..ac35cb1 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -572,9 +572,26 @@ def do_mirrorlist(kwargs):
l = list(s)
random.shuffle(l)
return l
+
+ def _ordered_netblocks(s):
+ def ipy_len(ip):
+ return ip.len()
+ v4_netblocks = []
+ v6_netblocks = []
+ for n in s:
+ if n.version() == 4:
+ v4_netblocks.append(n)
+ elif n.version() == 6:
+ v6_netblocks.append(n)
+ # mix up the order, as sort will preserve same-key ordering
+ random.shuffle(v4_netblocks)
+ v4_netblocks.sort(key=ipy_len)
+ random.shuffle(v6_netblocks)
+ v6_netblocks.sort(key=ipy_len)
+ return v6_netblocks + v4_netblocks
- location_hosts = _random_shuffle(location_results)
- netblock_hosts = _random_shuffle(netblock_results)
+ location_hosts = _random_shuffle(location_results)
+ netblock_hosts = _ordered_netblocks(netblock_results)
asn_hosts = _random_shuffle(asn_results)
internet2_hosts = _random_shuffle(internet2_results)
country_hosts = shuffle(country_results)
13 years, 6 months
Branch 'v1.4' - 2 commits - server/mirrormanager
by Matt Domsch
server/mirrormanager/model.py | 9 +++++--
server/mirrormanager/schema_updates/update_1_4_0.py | 24 ++++++++++++++++----
2 files changed, 26 insertions(+), 7 deletions(-)
New commits:
commit f16fc949d371ee5bdb8044505698c86231ee0b0c
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Oct 10 08:36:21 2010 -0500
SCHEMA CHANGE: drop class Schema, add cols to Site
Added emailOnDrop and emailOnAdd columns to Site. Not used yet.
Added more intelligent upgrade code, to know when to update these
columns in the database. This removes the need for the (unused)
Schema class.
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 0c42781..b1fb792 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -44,6 +44,8 @@ class Site(SQLObject):
# allow all sites to pull from me
allSitesCanPullFromMe = BoolCol(default=False)
downstreamComments = UnicodeCol(default=None)
+ emailOnDrop = BoolCol(default=False)
+ emailOnAdd = BoolCol(default=False)
admins = MultipleJoin('SiteAdmin')
hosts = MultipleJoin('Host')
@@ -884,12 +886,6 @@ class FileGroup(SQLObject):
cacheValues = False
name = UnicodeCol(alternateID=True, length=UnicodeColKeyLength)
files = SQLRelatedJoin('FileDetail')
-
-class Schema(SQLObject):
- class sqlmeta:
- cacheValues = False
- version = UnicodeCol(alternateID=True, length=UnicodeColKeyLength)
-
###############################################################
# These classes are only used if you're not using the
diff --git a/server/mirrormanager/schema_updates/update_1_4_0.py b/server/mirrormanager/schema_updates/update_1_4_0.py
index 65784c8..cc9f074 100644
--- a/server/mirrormanager/schema_updates/update_1_4_0.py
+++ b/server/mirrormanager/schema_updates/update_1_4_0.py
@@ -1,8 +1,23 @@
-from mirrormanager.model import Location, FileGroup, Schema
+from mirrormanager.model import Location, FileGroup, Site
+from sqlobject import BoolCol
+
+# upgrade methodology borrowed from
+# http://www.mail-archive.com/sqlobject-discuss@lists.sourceforge.net/msg04...
+
+class OldSite(SQLObject):
+ class sqlmeta:
+ fromDatabase = True
+ table = 'site'
+
def update():
Location.createTable(ifNotExists=True)
FileGroup.createTable(ifNotExists=True)
- Schema.createTable(ifNotExists=True)
- if Schema.select().count() == 0:
- Schema(version=u'1.4')
+
+ if 'email_on_drop' not in OldSite.sqlmeta.columns and \
+ 'email_on_add' not in OldSite.sqlmeta.columns:
+ OldSite.addColumn(changeSchema=True, BoolCol("emailOnDrop", default=False))
+ OldSite.addColumn(changeSchema=True, BoolCol("emailOnAdd", default=False))
+ for s in Site.select():
+ s.emailOnDrop=False
+ s.emailOnAdd=False
commit 085da9317fdcce435a771e27b13d0fe3b48fd558
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Oct 10 08:06:45 2010 -0500
SCHEMA CHANGE: add FileGroups to model
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index fad3938..0c42781 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -809,6 +809,7 @@ class FileDetail(SQLObject):
md5 = UnicodeCol(default=None)
sha256 = UnicodeCol(default=None)
sha512 = UnicodeCol(default=None)
+ fileGroups = SQLRelatedJoin('FileGroup')
class RepositoryRedirect(SQLObject):
class sqlmeta:
@@ -878,6 +879,12 @@ class Location(SQLObject):
name = UnicodeCol(alternateID=True, length=UnicodeColKeyLength)
hosts = SQLRelatedJoin('Host')
+class FileGroup(SQLObject):
+ class sqlmeta:
+ cacheValues = False
+ name = UnicodeCol(alternateID=True, length=UnicodeColKeyLength)
+ files = SQLRelatedJoin('FileDetail')
+
class Schema(SQLObject):
class sqlmeta:
cacheValues = False
diff --git a/server/mirrormanager/schema_updates/update_1_4_0.py b/server/mirrormanager/schema_updates/update_1_4_0.py
index df9c43b..65784c8 100644
--- a/server/mirrormanager/schema_updates/update_1_4_0.py
+++ b/server/mirrormanager/schema_updates/update_1_4_0.py
@@ -1,7 +1,8 @@
-from mirrormanager.model import Location, Schema
+from mirrormanager.model import Location, FileGroup, Schema
def update():
Location.createTable(ifNotExists=True)
+ FileGroup.createTable(ifNotExists=True)
Schema.createTable(ifNotExists=True)
if Schema.select().count() == 0:
Schema(version=u'1.4')
13 years, 6 months
Branch 'v1.4' - server/mirrormanager
by Matt Domsch
server/mirrormanager/schema_updates/update_1_4_0.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
New commits:
commit 43cc8919f7e64d900e53e1ca8dd78686ebd11ca5
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Oct 7 23:34:51 2010 -0500
s/zone/location/g in schema_updates too
diff --git a/server/mirrormanager/schema_updates/update_1_4_0.py b/server/mirrormanager/schema_updates/update_1_4_0.py
index 382f238..df9c43b 100644
--- a/server/mirrormanager/schema_updates/update_1_4_0.py
+++ b/server/mirrormanager/schema_updates/update_1_4_0.py
@@ -1,7 +1,7 @@
-from mirrormanager.model import Zone, Schema
+from mirrormanager.model import Location, Schema
def update():
- Zone.createTable(ifNotExists=True)
+ Location.createTable(ifNotExists=True)
Schema.createTable(ifNotExists=True)
if Schema.select().count() == 0:
Schema(version=u'1.4')
13 years, 6 months
Branch 'v1.4' - 2 commits - mirrorlist-server/mirrorlist_client.wsgi mirrorlist-server/mirrorlist_server.py server/mirrormanager
by Matt Domsch
mirrorlist-server/mirrorlist_client.wsgi | 2 -
mirrorlist-server/mirrorlist_server.py | 24 ++++++++++----------
server/mirrormanager/mirrorlist.py | 10 ++++----
server/mirrormanager/model.py | 10 ++++----
server/mirrormanager/schema_updates/update_1_4_0.py | 7 ++++-
5 files changed, 28 insertions(+), 25 deletions(-)
New commits:
commit 333a7a5b03a4c5a389e57c3994952ef10b67e530
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Oct 7 22:19:46 2010 -0500
s/zone/location/g everywhere
diff --git a/mirrorlist-server/mirrorlist_client.wsgi b/mirrorlist-server/mirrorlist_client.wsgi
index d7466d1..148b975 100755
--- a/mirrorlist-server/mirrorlist_client.wsgi
+++ b/mirrorlist-server/mirrorlist_client.wsgi
@@ -70,7 +70,7 @@ def real_client_ip(xforwardedfor):
return xforwardedfor.split(',')[-1].strip()
def request_setup(environ, request):
- fields = ['repo', 'arch', 'country', 'path', 'netblock', 'zone']
+ fields = ['repo', 'arch', 'country', 'path', 'netblock', 'location']
d = {}
request_data = request.GET
for f in fields:
diff --git a/mirrorlist-server/mirrorlist_server.py b/mirrorlist-server/mirrorlist_server.py
index 8cb2d4f..5ebe198 100755
--- a/mirrorlist-server/mirrorlist_server.py
+++ b/mirrorlist-server/mirrorlist_server.py
@@ -64,7 +64,7 @@ hcurl_cache = {}
asn_host_cache = {}
internet2_tree = radix.Radix()
global_tree = radix.Radix()
-zone_cache = {}
+location_cache = {}
def lookup_ip_asn(tree, ip):
""" @t is a radix tree
@@ -343,11 +343,11 @@ def do_geoip(kwargs, cache, clientCountry, header):
hostresults = trim_by_client_country(hostresults, clientCountry)
return (header, hostresults)
-def do_zone(kwargs, header):
+def do_location(kwargs, header):
hostresults = set()
- if 'zone' in kwargs and kwargs['zone'] in zone_cache:
- hostresults = set(zone_cache[kwargs['zone']])
- header += "Using zone %s " % kwargs['zone']
+ if 'location' in kwargs and kwargs['location'] in location_cache:
+ hostresults = set(location_cache[kwargs['location']])
+ header += "Using location %s " % kwargs['location']
return (header, hostresults)
def add_host_to_cache(cache, hostid, hcurl):
@@ -472,7 +472,7 @@ def do_mirrorlist(kwargs):
ordered_mirrorlist = cache.get('ordered_mirrorlist', default_ordered_mirrorlist)
done = 0
- zone_results = set()
+ location_results = set()
netblock_results = set()
asn_results = set()
internet2_results = set()
@@ -481,7 +481,7 @@ def do_mirrorlist(kwargs):
continent_results = set()
global_results = set()
- header, zone_results = do_zone(kwargs, header)
+ header, location_results = do_location(kwargs, header)
requested_countries = []
if kwargs.has_key('country'):
@@ -573,7 +573,7 @@ def do_mirrorlist(kwargs):
random.shuffle(l)
return l
- zone_hosts = _random_shuffle(zone_results)
+ location_hosts = _random_shuffle(location_results)
netblock_hosts = _random_shuffle(netblock_results)
asn_hosts = _random_shuffle(asn_results)
internet2_hosts = _random_shuffle(internet2_results)
@@ -582,7 +582,7 @@ def do_mirrorlist(kwargs):
continent_hosts = shuffle(continent_results)
global_hosts = shuffle(global_results)
- allhosts = uniqueify(zone_hosts + netblock_hosts + asn_hosts + internet2_hosts + country_hosts + geoip_hosts + continent_hosts + global_hosts)
+ allhosts = uniqueify(location_hosts + netblock_hosts + asn_hosts + internet2_hosts + country_hosts + geoip_hosts + continent_hosts + global_hosts)
hosts_and_urls = append_path(allhosts, cache, file, pathIsDirectory=pathIsDirectory)
if 'metalink' in kwargs and kwargs['metalink']:
@@ -632,7 +632,7 @@ def read_caches():
global file_details_cache
global hcurl_cache
global asn_host_cache
- global zone_cache
+ global location_cache
data = {}
try:
@@ -666,8 +666,8 @@ def read_caches():
hcurl_cache = data['hcurl_cache']
if 'asn_host_cache' in data:
asn_host_cache = data['asn_host_cache']
- if 'zone_cache' in data:
- zone_cache = data['zone_cache']
+ if 'location_cache' in data:
+ location_cache = data['location_cache']
del data
setup_continents()
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 3b6eed9..f82ebde 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -1,4 +1,4 @@
-from mirrormanager.model import Directory, Host, RepositoryRedirect, CountryContinentRedirect, Repository, HostCategoryUrl, Zone
+from mirrormanager.model import Directory, Host, RepositoryRedirect, CountryContinentRedirect, Repository, HostCategoryUrl, Location
from IPy import IP
import sha
import pprint
@@ -260,10 +260,10 @@ def hcurl_cache():
cache[hcurl.id] = hcurl.url
return cache
-def zone_cache():
+def location_cache():
cache = {}
- for zone in Zone.select():
- cache[zone.name] = [host.id for host in zone.hosts]
+ for location in Location.select():
+ cache[location.name] = [host.id for host in location.hosts]
return cache
def populate_all_caches():
@@ -285,7 +285,7 @@ def dump_caches():
'file_details_cache':file_details_cache(),
'hcurl_cache':hcurl_cache(),
'asn_host_cache':asn_host_cache(),
- 'zone_cache':zone_cache()}
+ 'location_cache':location_cache()}
try:
f = open('/var/lib/mirrormanager/mirrorlist_cache.pkl', 'w')
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index cde263a..fad3938 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -194,7 +194,7 @@ class Host(SQLObject):
acl_ips = MultipleJoin('HostAclIp', orderBy='ip')
categories = MultipleJoin('HostCategory')
exclusive_dirs = MultipleJoin('DirectoryExclusiveHost')
- zones = SQLRelatedJoin('Zone')
+ locations = SQLRelatedJoin('Location')
def destroySelf(self):
"""Cascade the delete operation"""
@@ -207,8 +207,8 @@ class Host(SQLObject):
b.destroySelf()
for ed in self.exclusive_dirs:
ed.destroySelf()
- for z in self.zones:
- self.removeZone(z)
+ for l in self.locations:
+ self.removeLocation(l)
SQLObject.destroySelf(self)
@@ -869,9 +869,9 @@ def host_siteadmins(host):
return []
return _host_siteadmins(hcurl.url)
-class Zone(SQLObject):
+class Location(SQLObject):
"""For grouping hosts, perhaps across Site boundaries. User queries may request
- hosts from a particular Zone (such as an Amazon availability zone), which will be returned
+ hosts from a particular Location (such as an Amazon region), which will be returned
first in the mirror list. """
class sqlmeta:
cacheValues = False
commit eb011865574c485f0ce9b268e18bb054bd38174f
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Oct 5 22:31:57 2010 -0500
create Schema table on update
diff --git a/server/mirrormanager/schema_updates/update_1_4_0.py b/server/mirrormanager/schema_updates/update_1_4_0.py
index 7830acc..382f238 100644
--- a/server/mirrormanager/schema_updates/update_1_4_0.py
+++ b/server/mirrormanager/schema_updates/update_1_4_0.py
@@ -1,4 +1,7 @@
-from mirrormanager.model import Zone
+from mirrormanager.model import Zone, Schema
def update():
- zone = Zone.createTable(ifNotExists=True)
+ Zone.createTable(ifNotExists=True)
+ Schema.createTable(ifNotExists=True)
+ if Schema.select().count() == 0:
+ Schema(version=u'1.4')
13 years, 6 months
Changes to 'preupgrade'
by Matt Domsch
New branch 'preupgrade' available with the following commits:
commit d45b25dc99b0c4dd58b996985abb538ededd6ce3
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Oct 9 15:30:54 2008 -0500
initial thoughts on a Release table
13 years, 6 months
Changes to 'sourcerepos'
by Matt Domsch
New branch 'sourcerepos' available with the following commits:
commit 590b4932d94473b955ff71b3ea32dc41113c4735
Author: Matt Domsch <matt(a)domsch.com>
Date: Mon May 12 08:41:55 2008 -0500
initial steps towards per-arch source repositories
13 years, 6 months