[mirrormanager] master: fix and simplify umdl version and arch caching and lookup (e959197)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit e959197deccc6861910ad08b845e8bd403898f0c
Author: Matt Domsch <matt(a)domsch.com>
Date: Sat Dec 7 10:32:18 2013 -0600
fix and simplify umdl version and arch caching and lookup
>---------------------------------------------------------------
server/mirrormanager/repomap.py | 2 +-
server/update-master-directory-list | 40 +++++++++++++---------------------
2 files changed, 16 insertions(+), 26 deletions(-)
diff --git a/server/mirrormanager/repomap.py b/server/mirrormanager/repomap.py
index 97b0e29..15d7d22 100644
--- a/server/mirrormanager/repomap.py
+++ b/server/mirrormanager/repomap.py
@@ -53,7 +53,7 @@ def repo_prefix(path, category, ver):
version = u'unknown'
if not isRawhide and ver is not None:
- version = ver['name']
+ version = ver.name
if isEpel:
# epel-
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index cc72baa..b1a75f9 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -84,52 +84,42 @@ def create_version_from_path(category, path):
return ver
-arch_cache = dict()
-version_cache = []
+arch_cache = None
+version_cache = None
def setup_arch_version_cache():
global arch_cache
- arch_cache = dict()
- for a in list(Arch.select()):
- arch_cache[a.name] = dict(name=a.name,
- id=a.id)
+ if arch_cache is None:
+ arch_cache = list(Arch.select())
global version_cache
- version_cache = []
- for v in list(Version.select(orderBy='-id')):
- version_cache.append(dict(id = v.id,
- name = v.name,
- product_id = v.product.id,
- ordered_mirrorlist = v.ordered_mirrorlist)
- )
+ if version_cache is None:
+ version_cache = list(Version.select(orderBy='-id'))
def guess_ver_arch_from_path(category, product_id, path):
arch = None
if 'SRPMS' in path:
- arch = arch_cache['source']
+ arch = Arch.byName('source')
else:
- for aname, v in arch_cache.iteritems():
- s = '.*(^|/)%s(/|$).*' % (aname)
+ for a in arch_cache:
+ s = '.*(^|/)%s(/|$).*' % (a.name)
if re.compile(s).match(path):
- arch = aname
+ arch = a
break
ver = None
# newest versions/IDs first, also handles stupid Fedora 9.newkey hack.
for v in version_cache:
- if v['product_id'] != product_id: continue
- s = '.*(^|/)%s(/|$).*' % (v['name'])
+ if v.product.id != product_id: continue
+ s = '.*(^|/)%s(/|$).*' % (v.name)
if re.compile(s).match(path):
ver = v
break
# create Versions if we can figure it out...
if ver is None:
- v = create_version_from_path(category, path)
- if v:
- ver = dict(id = v.id, name = v.name,
- product_id = product_id,
- ordered_mirrorlist = v.ordered_mirrorlist)
+ ver = create_version_from_path(category, path)
+ if ver:
version_cache.append(ver)
return (ver, arch)
@@ -273,7 +263,7 @@ def make_repository(dir, dname, category, product_id, topdir):
# anything meaningful. So simply have it match dir.name,
# which can't conflict.
repo = Repository(name=dir.name, category=category, version=ver, arch=arch, directory=dir, prefix=prefix)
- logger.info('Created Repository %s -> Directory %s' % (repo, dir))
+ logger.info('Created Repository(prefix=%s, version=%s, arch=%s, category=%s) -> Directory %s' % (prefix, ver.name, arch.name, category.name, dir.name))
except:
# repo exists, but it may be one we want to move
#try:
10 years, 4 months
[mirrormanager] master: import datetime, not datetime.datetime (293b754)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 293b75476ca2b76139401c336f29e250cfa1c222
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Dec 6 23:35:22 2013 -0600
import datetime, not datetime.datetime
Modules get confused when one bit of code (say, model.py) imports
datetime.datetime (the 'type'), while other code (say, umdl) imports
datetime (the module). It leads to runtime error:
AttributeError: type object 'datetime.datetime' has no attribute 'datetime'
Change all imports to import the module, not the type, and fix all callers.
>---------------------------------------------------------------
mirrorlist-server/mirrorlist_client.wsgi | 1 -
mirrorlist-server/test/server_tester.py | 10 +++++-----
server/crawler_perhost | 10 +++++-----
server/mirrormanager/model.py | 14 +++++++-------
4 files changed, 17 insertions(+), 18 deletions(-)
diff --git a/mirrorlist-server/mirrorlist_client.wsgi b/mirrorlist-server/mirrorlist_client.wsgi
index 8855595..5f49cda 100755
--- a/mirrorlist-server/mirrorlist_client.wsgi
+++ b/mirrorlist-server/mirrorlist_client.wsgi
@@ -17,7 +17,6 @@ from string import zfill, atoi, strip, replace
from paste.wsgiwrappers import *
import gzip
import cStringIO
-from datetime import datetime, timedelta
socketfile = '/var/run/mirrormanager/mirrorlist_server.sock'
select_timeout = 60 # seconds
diff --git a/mirrorlist-server/test/server_tester.py b/mirrorlist-server/test/server_tester.py
index 8b28e37..16d4663 100755
--- a/mirrorlist-server/test/server_tester.py
+++ b/mirrorlist-server/test/server_tester.py
@@ -7,7 +7,7 @@
import socket, os
import cPickle as pickle
from string import zfill, atoi
-from datetime import datetime
+import datetime
socketfile = '/var/run/mirrormanager/mirrorlist_server.sock'
@@ -17,9 +17,9 @@ connectTime = None
def do_mirrorlist(d):
try:
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
- start = datetime.utcnow()
+ start = datetime.datetime.utcnow()
s.connect(socketfile)
- end = datetime.utcnow()
+ end = datetime.datetime.utcnow()
except:
raise
@@ -77,7 +77,7 @@ while True:
client_ip = u"%s.%s.%s.%s" % (random.randint(0,255), random.randint(0,255), random.randint(0,255), random.randint(0,255))
d['client_ip'] = client_ip
- start = datetime.utcnow()
+ start = datetime.datetime.utcnow()
result = do_mirrorlist(d)
- end = datetime.utcnow()
+ end = datetime.datetime.utcnow()
print "[%s] connect: %s total: %s" % (pid, connectTime, (end-start))
diff --git a/server/crawler_perhost b/server/crawler_perhost
index cf3c936..c470621 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -3,7 +3,7 @@ __requires__ = 'TurboGears[future]'
import pkg_resources
pkg_resources.require("TurboGears")
-from datetime import datetime
+import datetime
import ftplib
from ftplib import FTP
import hashlib
@@ -332,7 +332,7 @@ def sync_hcds(host, host_category_dirs):
stats = dict(up2date = 0, not_up2date = 0, unchanged = 0,
unknown = 0, newdir = 0, deleted_on_master = 0)
current_hcds = {}
- now = datetime.utcnow()
+ now = datetime.datetime.utcnow()
host.lastCrawled = now
keys = host_category_dirs.keys()
keys = sorted(keys, key = lambda t: t[1].name)
@@ -488,13 +488,13 @@ def try_percategory(trydirs, url, host_category_dirs, hc, host, categoryPrefixLe
if not url.endswith('/'):
url += '/'
- rsync_start_time = datetime.utcnow()
+ rsync_start_time = datetime.datetime.utcnow()
try:
result, listing = run_rsync(url, '--no-motd')
except:
logger.warning('Failed to run rsync.', exc_info = True)
return False
- rsync_stop_time = datetime.utcnow()
+ rsync_stop_time = datetime.datetime.utcnow()
msg = "rsync time: %s" % str(rsync_stop_time - rsync_start_time)
logger.info(msg)
if result == 10:
@@ -648,7 +648,7 @@ Date: %s
def mark_not_up2date(exc, host, reason="Unknown"):
host.set_not_up2date()
- host.lastCrawled = datetime.utcnow()
+ host.lastCrawled = datetime.datetime.utcnow()
msg = "Host marked not up2date: %s" % reason
logger.warning(msg)
if exc is not None:
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index da67f2c..0c72672 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -2,7 +2,7 @@ from sqlobject import *
from sqlobject.converters import sqlrepr
from sqlobject.sqlbuilder import RLIKE, AND, OR, INNERJOINOn
from turbogears import identity, config
-from datetime import datetime
+import datetime
import time
from string import strip
import IPy
@@ -43,7 +43,7 @@ class Site(SQLObject):
private = BoolCol(default=False)
admin_active = BoolCol(default=True)
user_active = BoolCol(default=True)
- createdAt = DateTimeCol(default=datetime.utcnow())
+ createdAt = DateTimeCol(default=datetime.datetime.utcnow())
createdBy = UnicodeCol(default=None)
# allow all sites to pull from me
allSitesCanPullFromMe = BoolCol(default=False)
@@ -323,7 +323,7 @@ class Host(SQLObject):
def _set_config(self, config):
# really, we don't store the config anymore
self._config = None
- self.lastCheckedIn = datetime.utcnow()
+ self.lastCheckedIn = datetime.datetime.utcnow()
def checkin(self, config):
message = self._uploaded_config(config)
@@ -605,7 +605,7 @@ class HostStats(SQLObject):
class sqlmeta:
cacheValues = False
host = ForeignKey('Host')
- _timestamp = DateTimeCol(default=datetime.utcnow())
+ _timestamp = DateTimeCol(default=datetime.datetime.utcnow())
type = UnicodeCol(default=None)
data = PickleCol(default=None)
@@ -988,7 +988,7 @@ class Visit(SQLObject):
visit_key = StringCol(length=40, alternateID=True,
alternateMethodName="by_visit_key")
- created = DateTimeCol(default=datetime.now)
+ created = DateTimeCol(default=datetime.datetime.now)
expiry = DateTimeCol()
def lookup_visit(cls, visit_key):
@@ -1017,7 +1017,7 @@ class Group(SQLObject):
group_name = UnicodeCol(length=16, alternateID=True,
alternateMethodName="by_group_name")
display_name = UnicodeCol(length=255)
- created = DateTimeCol(default=datetime.now)
+ created = DateTimeCol(default=datetime.datetime.now)
# collection of all users belonging to this group
users = RelatedJoin("User", intermediateTable="user_group",
@@ -1044,7 +1044,7 @@ class User(SQLObject):
alternateMethodName="by_email_address")
display_name = UnicodeCol(length=255)
password = UnicodeCol(length=40)
- created = DateTimeCol(default=datetime.now)
+ created = DateTimeCol(default=datetime.datetime.now)
# groups this user belongs to
groups = RelatedJoin("Group", intermediateTable="user_group",
10 years, 4 months
[mirrormanager] master: add initialize-database --distro=CentOS (29b4460)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 29b446089eab87f9a4ac48c4c9808e703d9fa0b4
Author: Matt Domsch <matt(a)domsch.com>
Date: Sat Dec 7 09:47:26 2013 -0600
add initialize-database --distro=CentOS
to add CentOS-specific initialization
>---------------------------------------------------------------
server/initialize-database | 7 ++++++-
server/mirrormanager/initial.py | 5 +++++
2 files changed, 11 insertions(+), 1 deletions(-)
diff --git a/server/initialize-database b/server/initialize-database
index d8032fd..4acee12 100755
--- a/server/initialize-database
+++ b/server/initialize-database
@@ -8,7 +8,7 @@ pkg_resources.require("TurboGears")
import turbogears
from turbogears.database import PackageHub
-from mirrormanager.initial import user_group_setup, create_arches
+from mirrormanager.initial import user_group_setup, create_arches, setup_centos
hub = __connection__ = None
options = None
@@ -16,6 +16,8 @@ options = None
def doit():
user_group_setup()
create_arches()
+ if options.distro == u'CentOS':
+ setup_centos()
def main():
global options
@@ -23,6 +25,9 @@ def main():
parser.add_option("-c", "--config",
dest="config", default='/etc/mirrormanager/prod.cfg',
help="TurboGears config file to use (default=/etc/mirrormanager/prod.cfg)")
+ parser.add_option("-d", "--distro",
+ dest="distro", default=u'Fedora', choices=[u'Fedora', u'CentOS']
+ help="Distro to for which initialize")
(options, args) = parser.parse_args()
diff --git a/server/mirrormanager/initial.py b/server/mirrormanager/initial.py
index ceebb03..616d1b5 100644
--- a/server/mirrormanager/initial.py
+++ b/server/mirrormanager/initial.py
@@ -41,3 +41,8 @@ def create_arches():
_do_create(arch, True)
for arch in secondary_arches:
_do_create(arch, False)
+
+def setup_centos():
+ p = Product(name=u'CentOS')
+ d = Directory(name=u'centos')
+ c = Category(name=u'CentOS', product=p, topdir=d, canonicalhost=u'rsync://msync.centos.org/CentOS')
10 years, 4 months
[mirrormanager] master: SCHEMA CHANGE: max_connections is not UNSIGNED (dcb5169)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit dcb51693e31ca826e280196fee3c8d0f668e0009
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Dec 6 23:14:24 2013 -0600
SCHEMA CHANGE: max_connections is not UNSIGNED
postgresql can't handle UNSIGNED. And we don't really care if it's UNSIGNED.
>---------------------------------------------------------------
server/mirrormanager/model.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index ae14378..da67f2c 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -190,7 +190,7 @@ class Host(SQLObject):
internet2_clients = BoolCol(default=False)
asn = IntCol(default=None)
asn_clients = BoolCol(default=True)
- max_connections = IntCol(default=1, notNone=True, unsigned=True)
+ max_connections = IntCol(default=1, notNone=True)
last_crawl_duration = BigIntCol(default=0)
countries_allowed = MultipleJoin('HostCountryAllowed')
netblocks = MultipleJoin('HostNetblock', orderBy='netblock')
10 years, 4 months
[mirrormanager] master: repomap: add CentOS repository patterns (372be57)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 372be5790e2f6e00040a30fae0e033a33394bffe
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Dec 6 22:02:54 2013 -0600
repomap: add CentOS repository patterns
>---------------------------------------------------------------
server/mirrormanager/repomap.py | 14 +++++++++++++-
1 files changed, 13 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/repomap.py b/server/mirrormanager/repomap.py
index f5dab9f..97b0e29 100644
--- a/server/mirrormanager/repomap.py
+++ b/server/mirrormanager/repomap.py
@@ -7,6 +7,15 @@ def is_development(path):
return m.group(1)
return None
+def centos_prefix(path):
+ choices = (u'os', u'updates', u'extras', u'centosplus',
+ u'contrib', u'fasttrack', u'cr', u'addons', u'xen4')
+ for c in choices:
+ pattern = u'/' + c + u'/'
+ if pattern in path:
+ return c
+ return None
+
def repo_prefix(path, category, ver):
prefix = None
@@ -39,7 +48,7 @@ def repo_prefix(path, category, ver):
isRrpmfusionFreeFedora = (category.name == u'RPMFUSION free Fedora')
isRrpmfusionNonfreeEl = (category.name == u'RPMFUSION nonfree EL')
isRrpmfusionNonfreeFedora = (category.name == u'RPMFUSION nonfree Fedora')
-
+ isCentOS = (category.name == u'CentOS')
isRhel = (category.name == u'RHEL')
version = u'unknown'
@@ -278,4 +287,7 @@ def repo_prefix(path, category, ver):
if prefix and isBeta:
prefix = u'%s-beta' % prefix
+ elif isCentOS:
+ prefix = centos_prefix(path)
+
return prefix
10 years, 4 months
[mirrormanager] master: repomap: add prefix=fedora-install-* handling (ea7d051)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit ea7d05196094cfd8cdab6536bf1e7c359eb1497e
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Dec 6 19:16:29 2013 -0600
repomap: add prefix=fedora-install-* handling
Fedora's installer started using fedora-install-* repositories that we
had previously been ignoring. We can't ignore them anymore.
>---------------------------------------------------------------
server/mirrormanager/repomap.py | 27 ++++++++++++++++++---------
1 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/server/mirrormanager/repomap.py b/server/mirrormanager/repomap.py
index 548f558..f5dab9f 100644
--- a/server/mirrormanager/repomap.py
+++ b/server/mirrormanager/repomap.py
@@ -27,6 +27,7 @@ def repo_prefix(path, category, ver):
if not isUpdatesTesting:
isUpdatesReleased = u'updates' in path
isEverything = u'Everything' in path
+ isFedora = u'Fedora' in path
isEpel = (category.name == u'Fedora EPEL')
@@ -65,15 +66,23 @@ def repo_prefix(path, category, ver):
elif isFedoraLinux or isFedoraSecondary or isFedoraArchive:
if isReleases or isDevelopment:
- if isReleases and not isEverything: prefix = None
- # fedora-
- elif isDebug:
- prefix = u'fedora-debug-%s' % version
- elif isSource:
- prefix = u'fedora-source-%s' % version
- else:
- prefix=u'fedora-%s' % version
-
+ if isReleases:
+ if isEverything:
+ # fedora-
+ if isDebug:
+ prefix = u'fedora-debug-%s' % version
+ elif isSource:
+ prefix = u'fedora-source-%s' % version
+ else:
+ prefix=u'fedora-%s' % version
+ elif isFedora:
+ if isDebug or isSource:
+ # ignore releases/$version/Fedora/$arch/debug/
+ # ignore releases/$version/Fedora/source/SRPMS/
+ prefix = None
+ else:
+ # fedora-install-
+ prefix = u'fedora-install-%s' % version
elif isUpdatesReleased:
# updates-released-
if isDebug:
10 years, 4 months
Moving the MirrorManager master branch reference
by Matt Domsch
I'm going to move the git branch pointer for "master" from where it is
now (effectively on a v1.3 branch commit) up to what is at the tip of
the v1.4 branch today. As such, the v1.4 branch will match the master
branch, until such a time when master gets new development that I
don't want on v1.4 (not in the forseeable future).
Sorry for any inconvenience this causes.
Thanks,
Matt
--
Matt Domsch
Distinguished Engineer, Technology Strategist
Dell | Office of the CTO
10 years, 4 months