[mirrormanager] v1.4: mirrorlist.py: fix file_details_cache() (aa3ffb9)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit aa3ffb96824a358f11dfadf071c21f3320252f76
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Feb 4 17:44:59 2013 -0600
mirrorlist.py: fix file_details_cache()
It helps if I dereference the right object to find the filename.
>---------------------------------------------------------------
server/mirrormanager/mirrorlist.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index b1e58f8..1985b1a 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -264,7 +264,7 @@ def file_details_cache():
cache[d.name] = {}
for fd in d.fileDetails:
details = dict(timestamp=fd.timestamp, sha1=fd.sha1, md5=fd.md5, sha256=fd.sha256, sha512=fd.sha512, size=fd.size)
- append_value_to_cache(cache[d.name], d.filename, details)
+ append_value_to_cache(cache[d.name], fd.filename, details)
return cache
def hcurl_cache():
11 years, 3 months
[mirrormanager] v1.4: mirrorlist.py: fix shrink() (4ad599f)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 4ad599f985efa6c6943a6a872c628cc5d8ba6566
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Feb 4 17:38:00 2013 -0600
mirrorlist.py: fix shrink()
The current shrink() routine could fail if the end value was an
integer, because then assignment of an already-recorded value would
appear to be string character re-assignment.
Fix the syntax to be explicit that we are updating the original dict
passed in, with the keys we've walked through to find the actual
value'd object, and then set the value to the already-recorded value.
>---------------------------------------------------------------
server/mirrormanager/mirrorlist.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 96bb5bc..b1e58f8 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -63,7 +63,7 @@ def shrink(mc):
c = mc[d][subcache]
s = hashlib.sha1(pp.pformat(c)).hexdigest()
if s in matches:
- d[c] = matches[s]
+ mc[d][subcache] = matches[s]
else:
matches[s] = c
return mc
11 years, 3 months
[mirrormanager] v1.4: move-devel-to-release: add required Category option (ba77761)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit ba77761f0c4d39a685fb7b4ed0ac4c791e96f650
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Feb 3 22:29:36 2013 -0600
move-devel-to-release: add required Category option
We need to be able to move Fedora Secondary Arches content, as well as
the usual Fedora Linux content. Make --category a required option,
and list the possible category names in --help and if an invalid name
is passed.
>---------------------------------------------------------------
server/move-devel-to-release | 23 +++++++++++++++++------
1 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/server/move-devel-to-release b/server/move-devel-to-release
index c482ac8..73a0dd9 100755
--- a/server/move-devel-to-release
+++ b/server/move-devel-to-release
@@ -14,14 +14,18 @@ hub = __connection__ = None
options = None
-category = 'Fedora Linux'
-
# moving from pub/fedora/linux/development/14/x86_64/os to pub/fedora/linux/releases/14/Everything/x86_64/os
def doit():
global options
- c = Category.byName(category)
+ try:
+ c = Category.byName(options.category)
+ except SQLObjectNotFound:
+ sys.stderr.write("Category '%s' not found, exiting" % options.category)
+ print_categories(sys.stderr)
+ sys.stderr.flush()
+ sys.exit(1)
topdir = c.topdir.name
oldpattern = os.path.join('development', options.version)
newpattern = os.path.join('releases', options.version, 'Everything')
@@ -30,7 +34,6 @@ def doit():
if oldRe.search(d.name):
for r in d.repositories:
t = d.name.replace(oldpattern, newpattern)
- #print "trying to find %s" % t
try:
new_d = Directory.byName(t)
r.directory = new_d
@@ -38,9 +41,14 @@ def doit():
r.sync()
print "%s => %s" % (d.name, t)
except:
- #print "Unable to find a directory in [%s] for %s" % (options.archiveCategory, d.name)
raise
+def print_categories(fileno):
+ fileno.write("Available categories:\n")
+ for c in Category.select():
+ fileno.write("\t%s\n" % c.name)
+ fileno.flush()
+
def main():
global options
parser = OptionParser(usage=sys.argv[0] + " [options]")
@@ -49,10 +57,13 @@ def main():
help="TurboGears config file to use (default=/etc/mirrormanager/prod.cfg)")
parser.add_option("--version",
dest="version", action="store", type="string", help="OS version to move (e.g. '14') [required]", default=None)
+ parser.add_option("--category",
+ dest="category", action="store", type="string", help="Category (e.g. 'Fedora Linux') [required]", default=None)
(options, args) = parser.parse_args()
- if options.version is None:
+ if options.version is None or options.category is None:
parser.print_help()
+ print_categories(sys.stdout)
sys.exit(1)
turbogears.update_config(configfile=options.config,
11 years, 3 months
[mirrormanager] v1.4: Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4 (fff6005)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit fff600549db26f177856d7ea6129587d9db6c1ae
Merge: 26cab4f aa7caa0
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Sun Feb 3 22:18:47 2013 -0600
Merge branch 'v1.4' of ssh+git://git.fedorahosted.org/git/mirrormanager into v1.4
>---------------------------------------------------------------
mirrorlist-server/supervisor/mirrorlist-server.ini | 1 +
server/mirrormanager/mirrorlist.py | 8 ++++----
server/refresh_mirrorlist_cache | 9 ++++++++-
server/update-mirrorlist-server | 4 ++++
4 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/mirrorlist-server/supervisor/mirrorlist-server.ini b/mirrorlist-server/supervisor/mirrorlist-server.ini
index c5aad22..7aeca58 100644
--- a/mirrorlist-server/supervisor/mirrorlist-server.ini
+++ b/mirrorlist-server/supervisor/mirrorlist-server.ini
@@ -8,3 +8,4 @@ startretries=5
stopsignal=TERM
stopwaitsecs=10
user=mirrormanager
+log_stderr=true
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index ebfd300..96bb5bc 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -1,7 +1,7 @@
from mirrormanager.model import Directory, Host, RepositoryRedirect, CountryContinentRedirect, Repository, HostCategoryUrl, Location, Version, Category, NetblockCountry, setup_directory_category_cache
import os
from IPy import IP
-import sha
+import hashlib
import pprint
import dns.resolver
from mirrormanager.lib import append_value_to_cache
@@ -61,7 +61,7 @@ def shrink(mc):
for d in mc:
for subcache in subcaches:
c = mc[d][subcache]
- s = sha.sha(pp.pformat(c))
+ s = hashlib.sha1(pp.pformat(c)).hexdigest()
if s in matches:
d[c] = matches[s]
else:
@@ -323,7 +323,7 @@ def populate_all_caches():
populate_directory_cache()
import cPickle as pickle
-def dump_caches():
+def dump_caches(filename):
data = {'mirrorlist_cache':global_caches['mirrorlist_cache'],
'host_netblock_cache':global_caches['host_netblock_cache'],
'host_country_allowed_cache':global_caches['host_country_allowed_cache'],
@@ -341,7 +341,7 @@ def dump_caches():
'netblock_country_cache':netblock_country_cache()}
try:
- f = open('/var/lib/mirrormanager/mirrorlist_cache.pkl', 'w')
+ f = open(filename, 'w')
pickle.dump(data, f)
f.close()
except:
diff --git a/server/refresh_mirrorlist_cache b/server/refresh_mirrorlist_cache
index 06cb5b2..5139b28 100755
--- a/server/refresh_mirrorlist_cache
+++ b/server/refresh_mirrorlist_cache
@@ -14,16 +14,23 @@ import mirrormanager.zonecreator
from turbogears.database import PackageHub
hub = __connection__ = None
+options = None
+
def doit():
+ global options
populate_all_caches()
- dump_caches()
+ dump_caches(options.output)
mirrormanager.zonecreator.writeZoneFiles()
def main():
+ global options
parser = OptionParser(usage=sys.argv[0] + " [options]")
parser.add_option("-c", "--config",
dest="config", default='dev.cfg',
help="TurboGears config file to use")
+ parser.add_option("-o", "--output",
+ dest="output", default='/var/lib/mirrormanager/mirrorlist_cache.pkl',
+ help="output file")
(options, args) = parser.parse_args()
diff --git a/server/update-mirrorlist-server b/server/update-mirrorlist-server
index b49781a..b95eba5 100755
--- a/server/update-mirrorlist-server
+++ b/server/update-mirrorlist-server
@@ -11,6 +11,10 @@ echo $$ > ${LOCKFILE}
trap "rm -f ${LOCKFILE}" QUIT TERM INT HUP EXIT
cd ${MM_DIR}
+rm -rf ${CACHEDIR}/old
+mkdir -p ${CACHEDIR}/old
+cp -ar ${CACHEDIR}/* ${CACHEDIR}/old/
+
${MM_DIR}/update-static-content ${MIRRORLIST_DIR}
${MM_DIR}/refresh_mirrorlist_cache -c /etc/mirrormanager/prod.cfg
${MM_DIR}/get_internet2_netblocks ${CACHEDIR}/i2_netblocks.txt
11 years, 3 months
[mirrormanager] v1.4: handle multiple Repositories pointing at the same Directory (26cab4f)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 26cab4fd9c2d32ee4e1da97177f70200fd23cf73
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Thu Jan 31 09:20:07 2013 -0600
handle multiple Repositories pointing at the same Directory
I didn't think this could happen, but apparently it can, and there are
no constraints against such. So deal with it.
>---------------------------------------------------------------
server/mirrormanager/model.py | 6 +++---
server/move-devel-to-release | 3 +--
server/move-to-archive | 3 +--
3 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index d7323ce..32c3502 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -731,7 +731,7 @@ class Directory(SQLObject):
readable = BoolCol(default=True)
ctime = BigIntCol(default=0)
categories = RelatedJoin('Category')
- repository = SingleJoin('Repository') # zero or one repository, set if this dir contains a yum repo
+ repositories = MultipleJoin('Repository') # zero or one repository, set if this dir contains a yum repo
host_category_dirs = MultipleJoin('HostCategoryDir')
fileDetails = MultipleJoin('FileDetail', orderBy=['filename', '-timestamp'])
exclusive_hosts = MultipleJoin('DirectoryExclusiveHost')
@@ -739,8 +739,8 @@ class Directory(SQLObject):
def destroySelf(self):
for c in self.categories:
self.removeCategory(c)
- if self.repository is not None:
- self.repository.destroySelf()
+ for r in self.repositories:
+ r.destroySelf()
# don't destroy a whole category if only deleting a directory
for hcd in self.host_category_dirs:
hcd.destroySelf()
diff --git a/server/move-devel-to-release b/server/move-devel-to-release
index 4eb00e0..c482ac8 100755
--- a/server/move-devel-to-release
+++ b/server/move-devel-to-release
@@ -28,8 +28,7 @@ def doit():
oldRe = re.compile(oldpattern)
for d in c.directories:
if oldRe.search(d.name):
- r = d.repository
- if r is not None:
+ for r in d.repositories:
t = d.name.replace(oldpattern, newpattern)
#print "trying to find %s" % t
try:
diff --git a/server/move-to-archive b/server/move-to-archive
index fc5d634..5c1c1cc 100755
--- a/server/move-to-archive
+++ b/server/move-to-archive
@@ -27,8 +27,7 @@ def doit():
dirRe = re.compile(options.directoryRe)
for d in c.directories:
if dirRe.search(d.name):
- r = d.repository
- if r is not None:
+ for r in d.repositories:
t = archiveTopdir + d.name[len(originalTopdir):]
print "trying to find %s" % t
try:
11 years, 3 months