[mirrormanager] v1.4: https://fedorahosted.org/fedora-infrastructure/ticket/3880 (e1048d5)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit e1048d5a38057b9d6f2d5ae21e343f783382585d
Author: Adrian Reber <adrian(a)lisas.de>
Date: Thu Jul 11 10:41:14 2013 +0200
https://fedorahosted.org/fedora-infrastructure/ticket/3880
If a URL is added to a Host Category which already exists in the
database, MirrorManager reports Success even though it has not
actually added it to the database. This patch displays an error
telling the user that the URL already exists in the database.
>---------------------------------------------------------------
server/mirrormanager/controllers.py | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 3a8feb1..28efdc7 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -790,7 +790,8 @@ class HostCategoryUrlController(controllers.Controller, identity.SecureResource,
del kwargs['hcid']
HostCategoryUrl(host_category=hc, **kwargs)
except: # probably sqlite IntegrityError but we can't catch that for some reason...
- turbogears.flash("Error: entity already exists")
+ turbogears.flash("Error: URL already in use by another host")
+ raise turbogears.redirect("/host_category/%s" % hcid)
turbogears.flash("Success: HostCategoryURL created.")
raise turbogears.redirect("/host_category/%s" % hcid)
10 years, 10 months
[mirrormanager] v1.4: fix for "expected a bool or an int" (a24b663)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit a24b663271365a6666e164337889c0410adde8aa
Author: Adrian Reber <adrian(a)lisas.de>
Date: Thu Jul 11 09:34:39 2013 +0200
fix for "expected a bool or an int"
This should fix following traceback.
File "/home/staff/adrian/mm/server/mirrormanager/model.py", line 239, in _uploaded_config
self.user_active = config['host']['user_active']
File "<string>", line 1, in <lambda>
File "/usr/lib/python2.7/site-packages/sqlobject/main.py", line 1035, in _SO_setValue
dbValue = from_python(value, self._SO_validatorState)
File "/usr/lib/python2.7/site-packages/sqlobject/col.py", line 690, in to_python
(self.name, type(value), value), value, state)
valid: expected a bool or an int in the BoolCol 'user_active', got <type 'str'> '1' instead
>---------------------------------------------------------------
server/mirrormanager/model.py | 5 ++++-
1 files changed, 4 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 31d3ccc..6d88011 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -288,7 +288,10 @@ class Host(SQLObject):
# handle the optional arguments
if config['host'].has_key('user_active'):
- self.user_active = config['host']['user_active']
+ if config['host']['user_active'] in ['true', '1', 't', 'y', 'yes']:
+ self.user_active = True
+ else:
+ self.user_active = False
# fill in the host category data (HostCategory and HostCategoryURL)
# the category names in the config have been lowercased
10 years, 10 months
[mirrormanager] v1.4: umdl: add debug message when creating a Repository (c556827)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit c556827ef8e0c9443f4bf76bb276b27cf819ae34
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Jul 11 14:14:33 2013 -0500
umdl: add debug message when creating a Repository
>---------------------------------------------------------------
server/update-master-directory-list | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index 3c12194..e8dafb2 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -263,6 +263,8 @@ def make_repository(dir, dname, category, product_id, topdir):
# anything meaningful. So simply have it match dir.name,
# which can't conflict.
repo = Repository(name=dir.name, category=category, version=ver, arch=arch, directory=dir, prefix=prefix)
+ sys.stdout.write('Created Repository %s -> Directory %s\n' % (repo, dir))
+ sys.stdout.flush()
except:
# repo exists, but it may be one we want to move
#try:
10 years, 10 months
[mirrormanager] v1.4: fix move-devel-to-release (6a91ccd)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 6a91ccdfc4908b287220b50bccfd99e3b3e541d3
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Jul 11 14:11:45 2013 -0500
fix move-devel-to-release
In the upgrade from MM 1.3. to 1.4 in Fedora Infrastructure, it was
noted that we could not make the Repository() objects simply "move"
from development/ to releases/ as we thought, because there were
already Repository objects on the releases/ directories, but with
incorrect data. We needed to update those, and zero out fields on the
ones in development/. That's what this patch does.
In MM 1.4, it should be that Repository objects won't get created on
releases/ directories. So we'll have to revisit this about when
Fedora 20 branches from rawhide to be sure it works the way we want.
>---------------------------------------------------------------
server/move-devel-to-release | 56 +++++++++++++++++++++++++++--------------
1 files changed, 37 insertions(+), 19 deletions(-)
diff --git a/server/move-devel-to-release b/server/move-devel-to-release
index 89513b3..5d2e8a0 100755
--- a/server/move-devel-to-release
+++ b/server/move-devel-to-release
@@ -8,7 +8,7 @@ from optparse import OptionParser
from sqlobject import *
import turbogears
sys.path.append('/usr/share/mirrormanager/server')
-from mirrormanager.model import Category, Directory
+from mirrormanager.model import Category, Directory, Version
from turbogears.database import PackageHub
hub = __connection__ = None
@@ -16,6 +16,19 @@ hub = __connection__ = None
options = None
# moving from pub/fedora/linux/development/14/x86_64/os to pub/fedora/linux/releases/14/Everything/x86_64/os
+# by now, UMDL would have created a new repository object at the target directory anyhow
+# except it's version and arch values are wrong
+# and we want to zero out the values in the current repository object
+
+def fixup_repos(version, fromrepo, torepo):
+ fromrepo_arch = fromrepo.arch
+ fromrepo.arch = None
+ fromrepo.sync()
+
+ torepo.prefix = fromrepo.prefix
+ torepo.arch = fromrepo_arch
+ torepo.version = version
+ torepo.sync()
def doit():
@@ -27,28 +40,33 @@ def doit():
print_categories(sys.stderr)
sys.stderr.flush()
sys.exit(1)
- topdir = c.topdir.name
+
+ try:
+ v = Version.selectBy(product=c.product, name=options.version)[0]
+ except:
+ raise # fixme
oldpattern = os.path.join('development', options.version)
newpattern = os.path.join('releases', options.version, 'Everything')
oldRe = re.compile(oldpattern)
- for d in c.directories:
+ for r in c.repositories:
+ d = r.directory
if oldRe.search(d.name):
- for r in d.repositories:
- t = d.name.replace(oldpattern, newpattern)
- try:
- new_d = Directory.byName(t)
- except SQLObjectNotFound:
- sys.stderr.write("target Directory(%) not found, ignoring.\n")
- sys.stderr.flush()
- break
-
- try:
- r.directory = new_d
- r.category = c
- r.sync()
- print "%s => %s" % (d.name, t)
- except:
- raise
+ t = d.name.replace(oldpattern, newpattern)
+ try:
+ new_d = Directory.byName(t)
+ except SQLObjectNotFound:
+ sys.stderr.write("target Directory(%s) not found, ignoring.\n" % t)
+ sys.stderr.flush()
+ continue
+
+ l = len(new_d.repositories)
+ if l == 0:
+ continue # uh oh, umdl hasn't found this directory yet. Pretty unlikely...
+ if l > 1:
+ raise ValueError # uh oh, how'd we wind up with 2 repos pointing at this directory?
+
+ fixup_repos(v, r, new_d.repositories[0])
+ print "%s => %s" % (d.name, t)
def print_categories(fileno):
fileno.write("Available categories:\n")
10 years, 10 months
[mirrormanager] v1.4: model: add backref Category->Repositories (88ee40d)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 88ee40d3b0890072bb460d0c0149c8fd968e7630
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jul 8 22:18:48 2013 -0500
model: add backref Category->Repositories
>---------------------------------------------------------------
server/mirrormanager/model.py | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 54a23fc..31d3ccc 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -791,6 +791,7 @@ class Category(SQLObject):
publiclist = BoolCol(default=True)
GeoDNSDomain = UnicodeCol(default=None)
directories = RelatedJoin('Directory', orderBy='name') # all the directories that are part of this category
+ repositories = MultipleJoin('Repository')
hostCategories = MultipleJoin('HostCategory')
def destroySelf(self):
10 years, 10 months
[mirrormanager] v1.4: move-devel-to-release: Don't traceback if a Directory doesn't exist yet (e912501)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit e91250158f9777a8526778f6bce89edebe10cd4b
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jul 8 12:27:04 2013 -0500
move-devel-to-release: Don't traceback if a Directory doesn't exist yet
In cases like secondary architectures, we may want to run
move-devel-to-release for some architectures, but the remaining arches
may not yet be released (content in
pub/fedora-secondary/releases/$version/Everything/$arch). Instead of
tracing back, just print a message on stderr, and let the remaining
(valid) architectures be moved.
>---------------------------------------------------------------
server/move-devel-to-release | 6 ++++++
1 files changed, 6 insertions(+), 0 deletions(-)
diff --git a/server/move-devel-to-release b/server/move-devel-to-release
index 8905ccb..89513b3 100755
--- a/server/move-devel-to-release
+++ b/server/move-devel-to-release
@@ -37,6 +37,12 @@ def doit():
t = d.name.replace(oldpattern, newpattern)
try:
new_d = Directory.byName(t)
+ except SQLObjectNotFound:
+ sys.stderr.write("target Directory(%) not found, ignoring.\n")
+ sys.stderr.flush()
+ break
+
+ try:
r.directory = new_d
r.category = c
r.sync()
10 years, 10 months
[mirrormanager] v1.4: cleanup imports (4286727)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit 4286727bfeed29049b871963b760fb90073b6408
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Mon Jul 8 12:23:49 2013 -0500
cleanup imports
>---------------------------------------------------------------
server/move-devel-to-release | 7 ++++---
1 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/server/move-devel-to-release b/server/move-devel-to-release
index 73a0dd9..8905ccb 100755
--- a/server/move-devel-to-release
+++ b/server/move-devel-to-release
@@ -3,11 +3,12 @@ __requires__ = 'TurboGears[future]'
import pkg_resources
pkg_resources.require("TurboGears")
+import sys, re, os
+from optparse import OptionParser
from sqlobject import *
import turbogears
-from mirrormanager.model import *
-from optparse import OptionParser
-import sys, re, os
+sys.path.append('/usr/share/mirrormanager/server')
+from mirrormanager.model import Category, Directory
from turbogears.database import PackageHub
hub = __connection__ = None
10 years, 10 months
[mirrormanager] v1.4: netblock_country_cache key should be an IP() object, not a string (c2c89ac)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit c2c89ace0840178bd217c83dcc07abd1c692e6a3
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jul 3 13:20:03 2013 -0500
netblock_country_cache key should be an IP() object, not a string
the other such caches put an IP object here, so mirrorlist_server.py
expects it to be such, and tracebacks loading the cache otherwise.
>---------------------------------------------------------------
server/mirrormanager/mirrorlist.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index 09c391e..b869ce0 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -287,7 +287,7 @@ def netblock_country_cache():
except:
continue
# guaranteed to be unique by database constraints
- cache[ip.strNormal()] = (n.country)
+ cache[ip] = (n.country)
return cache
10 years, 10 months
[mirrormanager] v1.4: remove set-but-unused variables (f4c85f7)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit f4c85f72c9373478ff5fbc8c6406a737032c5924
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Wed Jul 3 00:05:56 2013 -0500
remove set-but-unused variables
>---------------------------------------------------------------
server/mirrormanager/controllers.py | 3 +--
server/mirrormanager/model.py | 1 -
2 files changed, 1 insertions(+), 3 deletions(-)
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index f7cbbb5..3a8feb1 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -494,7 +494,6 @@ class HostCategoryController(controllers.Controller, identity.SecureResource, co
def read(self, hostcategory):
downstream_siteadmin_check(hostcategory.my_site(), identity)
submit_action = "/host_category/%s/update" % hostcategory.id
- disabled_fields=self.disabled_fields()
return dict(form=host_category_form_read, values=hostcategory, action=submit_action, disabled_fields=self.disabled_fields(), host=hostcategory.host)
@expose(template="kid:mirrormanager.templates.hostcategory")
@@ -599,7 +598,7 @@ class HostListitemController(controllers.Controller, identity.SecureResource, co
except InvalidData, msg:
turbogears.flash(msg)
raise turbogears.redirect("/host/%s" % host.id)
- except Exception, e: # probably sqlite IntegrityError but we can't catch that for some reason...
+ except Exception: # probably sqlite IntegrityError but we can't catch that for some reason...
etype, value, tb = sys.exc_info()
s = traceback.format_exception(etype, value, tb)
lines = ''
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 666ff14..54a23fc 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -764,7 +764,6 @@ class Directory(SQLObject):
t = int(time.time())
max_stale = config.get('mirrormanager.max_stale_days', 3)
max_propogation = config.get('mirrormanager.max_propogation_days', 2)
- fd = {}
stale = t - (60*60*24*max_stale)
propogation = t - (60*60*24*max_propogation)
10 years, 10 months
[mirrormanager] v1.4: run_rsync(): return the rsync exit code, log nonzero exits (ff8b74f)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : v1.4
>---------------------------------------------------------------
commit ff8b74f31c0eea9632f2b97da4dc2a2f8411eb60
Author: Matt Domsch <Matt_Domsch(a)dell.com>
Date: Tue Jul 2 23:40:34 2013 -0500
run_rsync(): return the rsync exit code, log nonzero exits
>---------------------------------------------------------------
server/crawler_perhost | 5 ++---
server/mirrormanager/lib.py | 4 ++--
server/update-master-directory-list | 8 +++++---
3 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/server/crawler_perhost b/server/crawler_perhost
index a6fc9bb..b546bab 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -457,10 +457,9 @@ def try_percategory(trydirs, url, host_category_dirs, hc, host, categoryPrefixLe
url += '/'
result, listing = run_rsync(url, '--no-motd')
- if not result:
- print >>logfile, 'rsync failed'
+ if result > 0:
+ print >>logfile, 'rsync returned exit code %d' % result
logfile.flush()
- return None
# put the rsync listing in a dict for easy access
while True:
diff --git a/server/mirrormanager/lib.py b/server/mirrormanager/lib.py
index d94119f..4efc33e 100644
--- a/server/mirrormanager/lib.py
+++ b/server/mirrormanager/lib.py
@@ -94,7 +94,6 @@ def append_value_to_cache(cache, key, value):
def run_rsync(rsyncpath, extra_rsync_args=None):
tmpfile = tempfile.SpooledTemporaryFile()
- result = True
cmd = "rsync --temp-dir=/tmp -r --exclude=.snapshot --exclude='*.~tmp~'"
if extra_rsync_args is not None:
cmd += ' ' + extra_rsync_args
@@ -106,12 +105,13 @@ def run_rsync(rsyncpath, extra_rsync_args=None):
p = subprocess.Popen(cmd, shell=True, stdin=devnull,
stdout=tmpfile, stderr=devnull, close_fds=True, bufsize=-1)
p.wait()
+ result = p.returncode
except Exception, e:
msg = "Exception invoking rsync:\n"
msg += traceback.format_exc(e)
sys.stderr.write(msg+'\n')
sys.stderr.flush()
- result = False
+ result = p.returncode
tmpfile.flush()
tmpfile.seek(0)
return (result, tmpfile)
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index d380ffc..3c12194 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -438,10 +438,12 @@ def parse_rsync_listing(category, f):
def sync_directories_using_rsync(rsyncpath, category, extra_rsync_args=None):
result, output = run_rsync(rsyncpath, extra_rsync_args)
- if result is False:
- sys.stdout.write("rsync failed for Category %s: %s\n" % (category.name, output))
+ if result > 0:
+ sys.stdout.write("rsync returned exit code %d for Category %s: %s\n" % (result,
+ category.name,
+ output))
sys.stdout.flush()
- return
+ # still, try to use the output listing if we can
parse_rsync_listing(category, output)
def sync_directories_from_file(filename, category):
10 years, 10 months