[mirrormanager] master: import_centos: fix for centos mirror CSV file, initial import (56f07c6)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 56f07c664e0b579652ad11a65be79a80c5c54193
Author: Matt Domsch <matt(a)domsch.com>
Date: Mon Dec 16 14:47:17 2013 -0600
import_centos: fix for centos mirror CSV file, initial import
Ralph Angenendt provide me a CSV file of the public CentOS mirrors
to import into MM. The import_centos program needed significant
updates to be able to properly import this file, which is no surprise
given it had never been executed before on any file, much less
on this CSV file.
>---------------------------------------------------------------
server/import_centos | 173 +++++++++++++++++++++++++++++++-------------------
1 files changed, 108 insertions(+), 65 deletions(-)
diff --git a/server/import_centos b/server/import_centos
index 4f16d64..0ab6ea0 100755
--- a/server/import_centos
+++ b/server/import_centos
@@ -42,7 +42,7 @@ internet2_tree = radix.Radix()
global_tree = radix.Radix()
gipv4=None
gipv6=None
-
+admin = None
def generate_password():
# from http://en.wikipedia.org/wiki/Random_password_generator
@@ -98,14 +98,23 @@ def asn(ip):
asn = lookup_ip_asn(global_tree, ip)
return asn
+def hostname_from_url(url):
+ if '://' not in url: return None
+ protocol, rest = url.split('://', 1)
+ rest = u'//' + rest
+ hostport,path = urllib.splithost(rest)
+ host, port = urllib.splitnport(hostport)
+ return host
+
def hostnames_from_urls(*urls):
hostnames = set()
for url in urls:
- hostport,path = urllib.splithost(url)
- host, port = urllib.splitnport(hostport)
- hostnames.add(host)
+ hostname = hostname_from_url(url)
+ if hostname:
+ hostnames.add(hostname)
return hostnames
+
def ips_from_hostnames(hostnames):
result = set()
for hostname in hostnames:
@@ -116,7 +125,7 @@ def ips_from_hostnames(hostnames):
def urls_in_internet2(*urls):
rc = False
- hostnames = hostnames_from_urls(urls)
+ hostnames = hostnames_from_urls(*urls)
ips = ips_from_hostnames(hostnames)
for ip in ips:
if internet2(ip):
@@ -125,12 +134,12 @@ def urls_in_internet2(*urls):
def asn_from_urls(*urls):
result = set()
- hostnames = hostnames_from_urls(urls)
+ hostnames = hostnames_from_urls(*urls)
ips = ips_from_hostnames(hostnames)
for ip in ips:
- asn = asn(ip)
- if asn is not None:
- result.add(asn)
+ ASN = asn(ip)
+ if ASN is not None:
+ result.add(ASN)
result = list(result)
if len(result) > 1:
@@ -139,7 +148,7 @@ def asn_from_urls(*urls):
elif len(result) == 1:
result = result[0]
else:
- result = []
+ result = None
return result
def open_geoip_databases():
@@ -188,7 +197,7 @@ def convert_teredo_v4(ip):
return IP(v4addr)
def lookup_geoip(ip):
- clientCounry = None
+ clientCountry = None
# attempt IPv6, then IPv6 6to4 as IPv4, then Teredo, then IPv4
try:
if ip.version() == 6:
@@ -208,7 +217,7 @@ def lookup_geoip(ip):
return clientCountry
def country_from_urls(*urls):
- hostnames = hostnames_from_urls(urls)
+ hostnames = hostnames_from_urls(*urls)
ips = ips_from_hostnames(hostnames)
for ip in ips:
country = lookup_geoip(ip)
@@ -216,72 +225,104 @@ def country_from_urls(*urls):
return country
return None
-# fixme make sure the CSV file has this header:
-# "site_name","http_url","ftp_url","rsync_url","bandwidth","display_name","user_name","site_url","country"
-# "LMU Muenchen, Dpt. Biologie 2, IT-Gruppe","http://centos.bio.lmu.de/","ftp://centos.bio.lmu.de/centos/","rsync://centos.bio.lmu.de/CentOS/","100Mbit","Ralph Angenendt","centos-mirror-lmu(a)strg-alt-entf.org","http://zi.bio.lmu.de/","de"
-#
-#These are the fields: Name of the mirror's sponsor, then the mirror URLs
-#(http,ftp and rsync). Next field is bandwidth (where we have *tons* of
-#mirrors without any entry - mostly older entries). Then "Contact Name",
-#"Contact Mail", "URL to sponsor" and the countrycode (here de for Germany).
+def one_hostname_from_urls(http, ftp, rsync):
+ if len(http):
+ return hostname_from_url(http)
+ elif len(ftp):
+ return hostname_from_url(ftp)
+ elif len(rsync):
+ return hostname_from_url(rsync)
+ else:
+ return None
+# "Sponsor"," Location Major"," Location Minor"," http-URL"," ftp-URL"," rsync-URL"," Bandwidth (if available)"," state (current mostly)"," Sponsor URL"," cc"," continent"
def process(row):
- # Create User
- # avoid duplicate users
- try:
- new_user = True
- user = User(user_name=row['user_name'],
- email_address = row['user_name'],
- display_name = row['display_name'],
- password = generate_password())
- except:
- user = User.byName(row['user_name'])
- new_user = False
# Create Site
# avoid duplicate sites
- try:
- site = Site(name=row['site_name'],
- password=generate_password(),
- orgUrl=row['site_url'])
- except:
- site = Site.byName(row['site_name'])
+ http_url = row['http-URL'].strip().rstrip('/')
+ ftp_url = row['ftp-URL'].strip().rstrip('/')
+ rsync_url = row['rsync-URL'].strip().rstrip('/')
+ try:
+ site = Site(name=row['Sponsor'],
+ password=generate_password(),
+ orgUrl=row.get('Sponsor URL'))
+ print "Site %s created" % (site.name)
+ except:
+ return # no duplicates
+
# Create SiteAdmin
# if user and site were both duplicates, this could fail
try:
- siteAdmin = SiteAdmin(user, site)
+ siteAdmin = SiteAdmin(admin, site)
except:
pass
# Create Host
# fixme: be sure to filter out duplicate hosts from the CSV file before calling this program
+ hostname = one_hostname_from_urls(http_url, ftp_url, rsync_url)
+ if hostname is None:
+ print "no hostname for row %s" % row
+ return
try:
- host = Host(name=hostname,
- site=site,
- country=row['country'].upper() or country_from_urls(row['http_url'], row['ftp_url'], row['rsync_url']),
- bandwidth_int = row['bandwidth'],
- internet2 = urls_in_internet2(row['http_url'], row['ftp_url'], row['rsync_url']),
- asn = asn_from_urls(row['http_url'], row['ftp_url'], row['rsync_url']),
+ country = row['cc'].upper() or country_from_urls(http_url, ftp_url, rsync_url)
+ internet2 = urls_in_internet2(http_url, ftp_url, rsync_url)
+ asn = asn_from_urls(http_url, ftp_url, rsync_url)
+ bandwidth_int = row['Bandwidth (if available)']
+ bandwidth_int = bandwidth_int.strip()
+ if len(bandwidth_int):
+ bandwidth_int = int(bandwidth_int)
+ else:
+ bandwidth_int = 100
+ host = Host(name = hostname,
+ site = site,
+ country = country,
+ internet2 = internet2,
+ asn = asn,
+ bandwidth_int = bandwidth_int
)
+ print "Host %s created" % host
except:
# duplicate hosts means we've gone this far already somehow before.
# don't bother doing any more
- return user, False
+ raise
# Create HostCategory
category = Category.byName("CentOS")
- host_category = host_category(host=host,
- category=category,
- upstream=centos_upstream)
+ host_category = HostCategory(host=host,
+ category=category)
+
# Create HostCategoryURLs
- if row['http_url']:
- http_url = HostCategoryUrl(host_category=host_category,
- url = row['http_url'])
- if row['ftp_url']:
- ftp_url = HostCategoryUrl(host_category=host_category,
- url = row['ftp_url'])
- if row['rsync_url']:
- rsync_url = HostCategoryUrl(host_category=host_category,
- url = row['rsync_url'])
- return user, new_user
+ if http_url:
+ if http_url.startswith('http'):
+ try:
+ http_url = HostCategoryUrl(host_category=host_category,
+ url = http_url)
+ except:
+ print "ERROR: HTTP URL for %s is a duplicate" % row
+
+ else:
+ print "ERROR: HTTP URL for %s is invalid" % row
+
+ if ftp_url:
+ if ftp_url.startswith('ftp'):
+ try:
+ ftp_url = HostCategoryUrl(host_category=host_category,
+ url = ftp_url)
+ except:
+ print "ERROR: HTTP URL for %s is a duplicate" % row
+
+ else:
+ print "ERROR: FTP URL for %s is invalid" % row
+
+ if rsync_url:
+ if rsync_url.startswith('rsync'):
+ try:
+ rsync_url = HostCategoryUrl(host_category=host_category,
+ url = rsync_url)
+ except:
+ print "ERROR: RSYNC URL for %s is a duplicate" % row
+
+ else:
+ print "ERROR: RSYNC URL for %s is invalid" % row
def email_user(user):
subject = "Welcome to the CentOS MirrorManager"
@@ -342,12 +383,13 @@ def doit(options, args):
f = sys.stdin
else:
f = open(options.filename, 'r')
- csv_reader = reader(f, dialect='excel')
+ csv_reader = csv.DictReader(f, dialect='excel')
for row in csv_reader:
- user, new_user = process(row)
- if new_user:
- email_user(user)
-
+ print row
+# user, new_user = process(row)
+ process(row)
+# if new_user:
+# email_user(user)
def main():
parser = OptionParser(usage=sys.argv[0] + " [options]")
@@ -367,12 +409,13 @@ def main():
modulename="mirrormanager.config")
global hub
global __connection__
- global repos
hub = PackageHub("mirrormanager")
__connection__ = hub
- doit(options, args)
+ global admin
+ admin = User.selectBy(user_name=u'admin')[0]
+ doit(options, args)
if __name__ == "__main__":
sys.exit(main())
10 years, 4 months
[mirrormanager] master: change selinux context on /var/log/mirrormanager/crawler/ too (ebcaa70)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit ebcaa70c663aff2ca79a08539fd910025a5647d3
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Dec 13 08:44:40 2013 -0600
change selinux context on /var/log/mirrormanager/crawler/ too
>---------------------------------------------------------------
mirrormanager.spec.in | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/mirrormanager.spec.in b/mirrormanager.spec.in
index 8c30fe3..2e7f3c6 100644
--- a/mirrormanager.spec.in
+++ b/mirrormanager.spec.in
@@ -62,6 +62,7 @@ getent passwd mirrormanager >/dev/null || \
useradd -r -g mirrormanager -d /var/lib/mirrormanager -s /sbin/nologin \
-c "MirrorManager" mirrormanager
semanage fcontext -a -t httpd_sys_content_t '/var/lib/mirrormanager/mirrorlists(/.*)?'
+semanage fcontext -a -t httpd_sys_content_t '/var/log/mirrormanager/crawler(/.*)?'
exit 0
%files
10 years, 4 months
[mirrormanager] master: set selinux context on /var/lib/mirrormanager/mirrorlists/ (9b187ac)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 9b187ac1b4bfe66f186172df5fa9318a6093312b
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Dec 13 08:34:30 2013 -0600
set selinux context on /var/lib/mirrormanager/mirrorlists/
otherwise we can't serve the publiclist in Enforcing mode
>---------------------------------------------------------------
mirrormanager.spec.in | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/mirrormanager.spec.in b/mirrormanager.spec.in
index f24b9da..8c30fe3 100644
--- a/mirrormanager.spec.in
+++ b/mirrormanager.spec.in
@@ -14,6 +14,7 @@ BuildRequires: python >= 2.6
Requires: TurboGears, python-IPy, python-GeoIP, wget, yum, psmisc, python-dns, python-fedora >= 0.3.14, python-fedora-turbogears, py-radix, mod_wsgi, python-TurboMail
Requires: supervisor
Requires(pre): shadow-utils
+Requires(pre): policycoreutils-python
%description
MirrorManager tracks all the content provided on a master mirror
@@ -60,6 +61,7 @@ getent group mirrormanager >/dev/null || groupadd -r mirrormanager
getent passwd mirrormanager >/dev/null || \
useradd -r -g mirrormanager -d /var/lib/mirrormanager -s /sbin/nologin \
-c "MirrorManager" mirrormanager
+semanage fcontext -a -t httpd_sys_content_t '/var/lib/mirrormanager/mirrorlists(/.*)?'
exit 0
%files
10 years, 4 months
[mirrormanager] master: silence stdout from cronjobs (800cf33)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 800cf3343a17e54079c943907f14034047ab9d8a
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Dec 13 08:03:37 2013 -0600
silence stdout from cronjobs
get_internet2_netblocks routinely fails not able to get the router tables,
printint out an ugly gzip failure message. That's ok, we'll get
the tables on a future run then, no problem.
The other cronjobs now write their logs into /var/log/mirrormanager
from inside the app, we don't need the cron shell to do so also.
>---------------------------------------------------------------
cron/mirrormanager.cron | 6 ++++--
server/update-mirrorlist-server | 4 ++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/cron/mirrormanager.cron b/cron/mirrormanager.cron
index 8779795..e089fef 100644
--- a/cron/mirrormanager.cron
+++ b/cron/mirrormanager.cron
@@ -8,10 +8,12 @@
55 0 * * * mirrormanager /usr/share/mirrormanager/server/update-static-content >> /var/log/mirrormanager/static-content.log 2>&1
# update master directory list
-0 */2 * * * mirrormanager cd /usr/share/mirrormanager/server && ./update-master-directory-list -c /etc/mirrormanager/prod.cfg >> /var/log/mirrormanager/umdl.log 2>&1
+# logs sent to /var/log/mirrormanager/umdl.log by default
+0 */2 * * * mirrormanager cd /usr/share/mirrormanager/server && ./update-master-directory-list -c /etc/mirrormanager/prod.cfg > /dev/null
# Get the global netblocks list once a day
30 0 * * * mirrormanager cd /usr/share/mirrormanager/server && ./get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt > /dev/null 2>&1
# run the crawler twice a day
-0 */12 * * * mirrormanager cd /usr/share/mirrormanager/server && ./crawler -c /etc/mirrormanager/prod.cfg --threads 75 >> /var/log/mirrormanager/crawler.log 2>&1
+# logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default
+0 */12 * * * mirrormanager cd /usr/share/mirrormanager/server && ./crawler -c /etc/mirrormanager/prod.cfg --threads 75 > /dev/null 2>&1
diff --git a/server/update-mirrorlist-server b/server/update-mirrorlist-server
index 62a4f92..a15673f 100755
--- a/server/update-mirrorlist-server
+++ b/server/update-mirrorlist-server
@@ -12,10 +12,10 @@ trap "rm -f ${LOCKFILE}" QUIT TERM INT HUP EXIT
cd ${MM_DIR}
rm -rf ${CACHEDIR}/old
mkdir -p ${CACHEDIR}/old
-cp -ar ${CACHEDIR}/* ${CACHEDIR}/old/
+cp -ar ${CACHEDIR}/* ${CACHEDIR}/old/ 2>/dev/null
${MM_DIR}/refresh_mirrorlist_cache -c /etc/mirrormanager/prod.cfg
-${MM_DIR}/get_internet2_netblocks ${CACHEDIR}/i2_netblocks.txt
+${MM_DIR}/get_internet2_netblocks ${CACHEDIR}/i2_netblocks.txt 2>/dev/null
kill -HUP $(cat /var/run/mirrormanager/mirrorlist_server.pid)
exit 0
10 years, 4 months
[mirrormanager] master: reduce TG-induced DeprecationWarnings (2dec3bc)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 2dec3bcfc79f8004c92b180432938fd8bfaab59e
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Dec 13 00:16:29 2013 -0600
reduce TG-induced DeprecationWarnings
The import of the turbogears module throws a warning
/usr/lib/python2.6/site-packages/tw/core/view.py:223: DeprecationWarning: object.__new__() takes no parameters
due to object.__new__ changing in Python 2.6.
This was annoying. So, shut it up, only in import
of turbogears.
>---------------------------------------------------------------
server/crawler | 6 ++++--
server/crawler_perhost | 6 +++++-
server/generate-publiclist | 5 ++++-
server/generate-worldmap | 5 ++++-
server/import_centos | 5 ++++-
server/initialize-database | 5 ++++-
server/manage-repo-redirects | 5 ++++-
server/mirrormanager.wsgi | 6 ++++--
server/move-devel-to-release | 5 ++++-
server/move-to-archive | 5 ++++-
server/refresh_mirrorlist_cache | 5 ++++-
server/update-database-schema | 5 ++++-
server/update-master-directory-list | 5 ++++-
13 files changed, 53 insertions(+), 15 deletions(-)
diff --git a/server/crawler b/server/crawler
index 3b2aacf..81e472b 100755
--- a/server/crawler
+++ b/server/crawler
@@ -12,8 +12,10 @@ import signal
from subprocess import Popen
import sys
import time
-
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
sys.path.append('/usr/share/mirrormanager/server')
from mirrormanager.model import Host
diff --git a/server/crawler_perhost b/server/crawler_perhost
index c470621..199e3e1 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -18,9 +18,13 @@ import socket
import time
from urlparse import urlsplit
import urlgrabber
+import warnings
from sqlobject import SQLObjectNotFound
-import turbogears
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
+
from turbogears import config
sys.path.append('/usr/share/mirrormanager/server')
from mirrormanager.model import Directory, Category, HostCategoryDir, Host, HostCategory
diff --git a/server/generate-publiclist b/server/generate-publiclist
index a965c35..b075fad 100755
--- a/server/generate-publiclist
+++ b/server/generate-publiclist
@@ -12,7 +12,10 @@ import optparse
import os
import sys
import time
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
import turbogears.util as tg_util
from turbogears import config
from turbogears.view import engines
diff --git a/server/generate-worldmap b/server/generate-worldmap
index 3fd9800..3a684e8 100755
--- a/server/generate-worldmap
+++ b/server/generate-worldmap
@@ -14,7 +14,10 @@ pkg_resources.require("TurboGears")
from sqlobject import *
import sys, os, string
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from mirrormanager.model import *
from optparse import OptionParser
import GeoIP
diff --git a/server/import_centos b/server/import_centos
index 265bb37..4f16d64 100755
--- a/server/import_centos
+++ b/server/import_centos
@@ -15,7 +15,10 @@ pkg_resources.require("TurboGears")
from sqlobject import *
import sys
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from mirrormanager.model import *
from mirrormanager.mirrorlist import name_to_ips
from optparse import OptionParser
diff --git a/server/initialize-database b/server/initialize-database
index 4acee12..7142486 100755
--- a/server/initialize-database
+++ b/server/initialize-database
@@ -6,7 +6,10 @@ __requires__ = 'TurboGears[future]'
import pkg_resources
pkg_resources.require("TurboGears")
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from turbogears.database import PackageHub
from mirrormanager.initial import user_group_setup, create_arches, setup_centos
diff --git a/server/manage-repo-redirects b/server/manage-repo-redirects
index 62d5032..c8507e4 100755
--- a/server/manage-repo-redirects
+++ b/server/manage-repo-redirects
@@ -11,7 +11,10 @@ pkg_resources.require("TurboGears")
from sqlobject import *
import sys
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from mirrormanager.model import *
from optparse import OptionParser
diff --git a/server/mirrormanager.wsgi b/server/mirrormanager.wsgi
index 7006c85..9236b5f 100755
--- a/server/mirrormanager.wsgi
+++ b/server/mirrormanager.wsgi
@@ -12,8 +12,10 @@ else:
import pkg_resources
pkg_resources.require("TurboGears<1.2")
pkg_resources.require("CherryPy<3.0")
-
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from turbogears import startup
import cherrypy
import cherrypy._cpwsgi
diff --git a/server/move-devel-to-release b/server/move-devel-to-release
index 5d2e8a0..8a63f1a 100755
--- a/server/move-devel-to-release
+++ b/server/move-devel-to-release
@@ -6,7 +6,10 @@ pkg_resources.require("TurboGears")
import sys, re, os
from optparse import OptionParser
from sqlobject import *
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
sys.path.append('/usr/share/mirrormanager/server')
from mirrormanager.model import Category, Directory, Version
diff --git a/server/move-to-archive b/server/move-to-archive
index 5c1c1cc..787ef33 100755
--- a/server/move-to-archive
+++ b/server/move-to-archive
@@ -4,7 +4,10 @@ import pkg_resources
pkg_resources.require("TurboGears")
from sqlobject import *
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from mirrormanager.model import *
from optparse import OptionParser
import sys, re
diff --git a/server/refresh_mirrorlist_cache b/server/refresh_mirrorlist_cache
index d81c0d8..4e00a43 100755
--- a/server/refresh_mirrorlist_cache
+++ b/server/refresh_mirrorlist_cache
@@ -5,7 +5,10 @@ pkg_resources.require("TurboGears")
import sys
import optparse
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from turbogears.database import PackageHub
sys.path.append('/usr/share/mirrormanager/server')
from mirrormanager.mirrorlist import populate_all_caches, dump_caches
diff --git a/server/update-database-schema b/server/update-database-schema
index d4fc053..fc3761d 100755
--- a/server/update-database-schema
+++ b/server/update-database-schema
@@ -4,7 +4,10 @@ import pkg_resources
pkg_resources.require("TurboGears")
import sys
-import turbogears
+import warnings
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from optparse import OptionParser
from turbogears.database import PackageHub
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index 48fd087..7c66ee5 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -15,9 +15,12 @@ import yum.repoMDObject
import datetime
import time
import hashlib
+import warnings
from sqlobject import *
-import turbogears
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ import turbogears
from turbogears import config
sys.path.append('/usr/share/mirrormanager/server')
10 years, 4 months
[mirrormanager] master: remove Fedora login form from CentOS, use quickstart login. (5567c57)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 5567c5700b32df880bb9b7547e3e8caaa9412427
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Dec 12 15:26:45 2013 -0600
remove Fedora login form from CentOS, use quickstart login.
>---------------------------------------------------------------
server/mirrormanager/controllers.py | 55 ++++++++++++++++++++--
server/mirrormanager/templates/centos/login.html | 19 +++++++-
2 files changed, 69 insertions(+), 5 deletions(-)
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 786cfa4..756ce69 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -7,7 +7,7 @@ import traceback
admin_group = config.get('mirrormanager.admin_group', 'sysadmin')
import turbogears
-from turbogears import controllers, expose, validate, redirect, widgets
+from turbogears import controllers, expose, validate, redirect, widgets, visit
from turbogears import validators, error_handler
from turbogears import identity
@@ -937,11 +937,51 @@ class Root(controllers.RootController):
includes[i] += u'/'
return project_dict('rsync_acl', includes=includes, excludes=excludes, message=message)
- @expose(template="genshi:mirrormanager.templates.login", allow_json=True)
- def login(self, forward_url=None, previous_url=None, *args, **kw):
+ def fedora_login(self, forward_url=None, previous_url=None, *args, **kw):
login_dict = fc_login(forward_url, previous_url, args, kw)
return project_dict('login', template_engine='genshi', **login_dict)
+
+ def nonfedora_login(self, forward_url=None, *args, **kw):
+ """Show the login form or forward user to previously requested page."""
+
+ if forward_url:
+ if isinstance(forward_url, list):
+ forward_url = forward_url.pop(0)
+ else:
+ del cherrypy.request.params['forward_url']
+
+ new_visit = visit.current()
+ if new_visit:
+ new_visit = new_visit.is_new
+
+ if (not new_visit and not identity.current.anonymous
+ and identity.was_login_attempted()
+ and not identity.get_identity_errors()):
+ redirect(forward_url or '/', kw)
+
+ if identity.was_login_attempted():
+ if new_visit:
+ msg = _(u"Cannot log in because your browser "
+ "does not support session cookies.")
+ else:
+ msg = _(u"The credentials you supplied were not correct or "
+ "did not grant access to this resource.")
+ elif identity.get_identity_errors():
+ msg = _(u"You must provide your credentials before accessing "
+ "this resource.")
+ else:
+ msg = _(u"Please log in.")
+ if not forward_url:
+ forward_url = cherrypy.request.headers.get("Referer", "/")
+
+ # we do not set the response status here anymore since it
+ # is now handled in the identity exception.
+ return project_dict('login', template_engine='genshi', logging_in=True, message=msg,
+ forward_url=forward_url, previous_url=cherrypy.request.path_info,
+ original_parameters=cherrypy.request.params)
+
+
@expose(allow_json=True)
def logout(self):
return fc_logout()
@@ -987,4 +1027,11 @@ class Root(controllers.RootController):
raise redirect("/")
-
+ if config.get('identity.provider') == 'sqlobjectcsrf':
+ @expose(template="genshi:mirrormanager.templates.login", allow_json=True)
+ def login(self, **kw):
+ return self.nonfedora_login(**kw)
+ else:
+ @expose(template="genshi:mirrormanager.templates.login", allow_json=True)
+ def login(self, **kw):
+ return self.fedora_login(**kw)
diff --git a/server/mirrormanager/templates/centos/login.html b/server/mirrormanager/templates/centos/login.html
index b7f9131..dfe4ed4 100644
--- a/server/mirrormanager/templates/centos/login.html
+++ b/server/mirrormanager/templates/centos/login.html
@@ -13,6 +13,23 @@
<body>
<h1 class="icon48 password">Login</h1>
- <loginform>${message}</loginform>
+ <div id="loginform">
+ <p py:content="message">Login message.</p>
+ <form action="${tg.url(previous_url)}" method="post" class="loginfields">
+ <h2><span>Login</span></h2>
+ <div>
+ <label for="user_name">Username:</label><input type="text" id="user_name" name="user_name" class="text"></input><br/>
+ <label for="password">Password:</label><input type="password" id="password" name="password" class="text"></input>
+ <input type="submit" id="submit" name="login" value="Login"/>
+ <input py:if="forward_url" type="hidden" name="forward_url"
+ value="${forward_url}"/>
+ <py:for each="name,values in original_parameters.items()">
+ <input py:for="value in isinstance(values, list) and values or [values]"
+ py:if="isinstance(value, basestring)"
+ type="hidden" name="${name}" value="${value}"/>
+ </py:for>
+ </div>
+ </form>
+ </div>
</body>
</html>
10 years, 4 months
[mirrormanager] master: remove Fedora-isms from the CentOS templates (e3c92d3)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit e3c92d3fd026d41cd68b166383a82c9afbb73948
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Dec 12 14:31:32 2013 -0600
remove Fedora-isms from the CentOS templates
>---------------------------------------------------------------
server/mirrormanager/templates/centos/help.kid | 4 +-
server/mirrormanager/templates/centos/host.kid | 2 +-
server/mirrormanager/templates/centos/master.kid | 12 +-------
.../mirrormanager/templates/centos/publiclist.kid | 2 +-
server/mirrormanager/templates/centos/site.kid | 29 --------------------
5 files changed, 5 insertions(+), 44 deletions(-)
diff --git a/server/mirrormanager/templates/centos/help.kid b/server/mirrormanager/templates/centos/help.kid
index 94d37e2..87417d9 100644
--- a/server/mirrormanager/templates/centos/help.kid
+++ b/server/mirrormanager/templates/centos/help.kid
@@ -47,8 +47,8 @@ Control List. As such, each Host should create one or more <strong>ACL
IPs</strong>. Hosts can also be temporarily marked inactive.
</p>
<p>
-Hosts carry content by <strong>Category</strong>. Fedora's categories include
-Fedora Linux (Fedora 7 and newer). Hosts expose a Category via one or
+Hosts carry content by <strong>Category</strong>. CentOS has a single Category, "CentOS".
+Hosts expose a Category via one or
more <strong>URLs</strong> (public URLs for anonymous http/ftp/rsync, or private URLs
for use by other mirrors only).
</p>
diff --git a/server/mirrormanager/templates/centos/host.kid b/server/mirrormanager/templates/centos/host.kid
index 80e4c78..3a2d3cb 100644
--- a/server/mirrormanager/templates/centos/host.kid
+++ b/server/mirrormanager/templates/centos/host.kid
@@ -115,7 +115,7 @@ privileges.
<a href="${tg.url('/host_category/0/new?hostid=' + str(values.id))}">[Add Category]</a>
</div>
<p>
-Hosts carry categories of software. Example Fedora categories include Fedora and Fedora Archive.
+Hosts carry categories of software. CentOS has only category "CentOS".
</p>
<div py:if="values.categories is not None">
<ul>
diff --git a/server/mirrormanager/templates/centos/master.kid b/server/mirrormanager/templates/centos/master.kid
index 275e3a2..d4d1b60 100644
--- a/server/mirrormanager/templates/centos/master.kid
+++ b/server/mirrormanager/templates/centos/master.kid
@@ -84,18 +84,8 @@ if admin_group in tg.identity.groups:
<div id="bottom">
<div py:if="tg.config('mirrormanager.copyright', 'fedora') == 'fedora'" id="footer">
<p class="copy">Copyright © 2007 Red Hat, Inc. and others. All Rights Reserved.
- Please send any comments or corrections to the <a href="mailto:webmaster@fedoraproject.org">websites team</a>.
+ Please send any comments or corrections to the <a href="http://lists.centos.org/mailman/listinfo/CentOS-mirror">websites team</a>.
</p>
- <p class="disclaimer">
- The Fedora Project is maintained and driven by the community and sponsored by Red Hat. This is a community maintained site. Red Hat is not responsible for content.
- </p>
- <ul>
- <li class="first"><a href="http://fedoraproject.org/wiki/Legal">Legal</a></li>
- <li><a href="http://fedoraproject.org/wiki/Legal/TrademarkGuidelines">Trademark Guidelines</a></li>
- </ul>
- </div>
- <div py:if="tg.config('mirrormanager.copyright', 'fedora') != 'fedora'" id="footer">
- Some other copyright???
</div>
</div>
</div>
diff --git a/server/mirrormanager/templates/centos/publiclist.kid b/server/mirrormanager/templates/centos/publiclist.kid
index 8b1279d..d2d1f53 100644
--- a/server/mirrormanager/templates/centos/publiclist.kid
+++ b/server/mirrormanager/templates/centos/publiclist.kid
@@ -46,7 +46,7 @@ donations.
<p>
To become a public ${tg.config('mirrormanager.projectname','Fedora')} mirror, please see <a
py:attrs="{'href': tg.config('mirrormanager.mirrorwiki', 'http://fedoraproject.org/wiki/Infrastructure/Mirroring')}">
-our wiki page on Mirroring</a>.
+our page on Mirroring</a>.
</p>
<p>
diff --git a/server/mirrormanager/templates/centos/site.kid b/server/mirrormanager/templates/centos/site.kid
index 4aafe56..3917ed2 100644
--- a/server/mirrormanager/templates/centos/site.kid
+++ b/server/mirrormanager/templates/centos/site.kid
@@ -24,35 +24,6 @@ Created At: ${values.createdAt}<br></br>
Created By: ${values.createdBy}<br></br>
</div>
-<p>
-Mirror server administrators are not required to sign the Fedora
-<a href="http://fedoraproject.org/wiki/Legal/Licenses/CLA">Contributor
-License Agreement</a>.
-</p>
-<h2>Export Compliance</h2>
-<p>
-By downloading Fedora software, you acknowledge that you understand
-all of the following: Fedora software and technical information may be
-subject to the U.S. Export Administration Regulations (the “EAR”) and
-other U.S. and foreign laws and may not be exported, re-exported or
-transferred (a) to any country listed in Country Group E:1 in
-Supplement No. 1 to part 740 of the EAR (currently, Cuba, Iran, North
-Korea, Sudan and Syria); (b) to any prohibited destination or to any end
-user who has been prohibited from participating in U.S. export
-transactions by any federal agency of the U.S. government; or (c) for
-use in connection with the design, development or production of
-nuclear, chemical or biological weapons, or rocket systems, space
-launch vehicles, or sounding rockets, or unmanned air vehicle
-systems. You may not download Fedora software or technical information
-if you are located in one of these countries or otherwise subject to
-these restrictions. You may not provide Fedora software or technical
-information to individuals or entities located in one of these
-countries or otherwise subject to these restrictions. You are also
-responsible for compliance with foreign law requirements applicable to
-the import, export and use of Fedora software and technical
-information.
-</p>
-
${form(value=values, action=tg.url(action), disabled_fields=disabled_fields)}
<div py:if="values is not None">
10 years, 4 months
[mirrormanager] master: fix mirrorlist pickle generation (4398137)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 4398137fcbc4e24a49b0c8251a9f97368ca5e81a
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Dec 12 14:22:38 2013 -0600
fix mirrorlist pickle generation
it was failing to look up a parent directory, which is odd, as the parent
should have been seen first. in reality, we don't care about
the ordered_mirrorlist setting for much of anything, especially
non-repository directories, so, just nuke that lookup.
move the parent_dir() function to lib.py.
>---------------------------------------------------------------
server/mirrormanager/lib.py | 8 ++++++++
server/mirrormanager/mirrorlist.py | 8 +-------
server/update-master-directory-list | 11 +----------
3 files changed, 10 insertions(+), 17 deletions(-)
diff --git a/server/mirrormanager/lib.py b/server/mirrormanager/lib.py
index e0fc6e0..b78884a 100644
--- a/server/mirrormanager/lib.py
+++ b/server/mirrormanager/lib.py
@@ -120,3 +120,11 @@ def project_dict(formname, template_engine="kid", **kw):
d['tg_template'] = "%s:%s" % (template_engine, template)
d.update(**kw)
return d
+
+def parent_dir(path):
+ sdir = path.split('/')[:-1]
+ try:
+ parent = os.path.join(*sdir)
+ except TypeError: #
+ parent = u''
+ return parent
diff --git a/server/mirrormanager/mirrorlist.py b/server/mirrormanager/mirrorlist.py
index ca45a1e..e67c9c2 100644
--- a/server/mirrormanager/mirrorlist.py
+++ b/server/mirrormanager/mirrorlist.py
@@ -4,7 +4,7 @@ from IPy import IP
import hashlib
import pprint
import dns.resolver
-from mirrormanager.lib import append_value_to_cache
+from mirrormanager.lib import append_value_to_cache, parent_dir
global_caches = dict(
# key is directoryname
@@ -141,12 +141,6 @@ def populate_directory_cache():
# any of them will do, so just look at the first one
category_id = directory_category_cache[directory_id][0]
- # repodata/ directories aren't themselves repositories, their parent dir is
- # we're walking the list in order, so the parent will be added to the cache before the child
- if directoryname.endswith('/repodata'):
- parent = os.path.dirname(directoryname) # parent
- cache[directoryname]['ordered_mirrorlist'] = cache[parent]['ordered_mirrorlist']
-
cache[directoryname]['subpath'] = directoryname[category_topdir_cache[category_id]:]
del repo
diff --git a/server/update-master-directory-list b/server/update-master-directory-list
index 43006a5..48fd087 100755
--- a/server/update-master-directory-list
+++ b/server/update-master-directory-list
@@ -23,7 +23,7 @@ from turbogears import config
sys.path.append('/usr/share/mirrormanager/server')
from mirrormanager.model import *
from mirrormanager.repomap import *
-from mirrormanager.lib import manage_pidfile, remove_pidfile, run_rsync
+from mirrormanager.lib import manage_pidfile, remove_pidfile, run_rsync, parent_dir
pidfile='/var/run/mirrormanager/umdl.pid'
@@ -279,15 +279,6 @@ def nuke_gone_directories(diskpath, category, excludes):
logger.info("Deleting gone directory %s" % (d.name))
d.destroySelf()
-def parent_dir(path):
- sdir = path.split('/')[:-1]
- try:
- parent = os.path.join(*sdir)
- except TypeError: #
- parent = u''
- return parent
-
-
def ctime_from_rsync(date, hms):
year, month, day = date.split('/')
hour, minute, second = hms.split(':')
10 years, 4 months
[mirrormanager] master: fix lib.py project_dict() (b7da2a1)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit b7da2a15e0d1d4dcf0b1cd6142e5768181a16a9f
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Dec 12 09:32:42 2013 -0600
fix lib.py project_dict()
>---------------------------------------------------------------
server/mirrormanager/lib.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/server/mirrormanager/lib.py b/server/mirrormanager/lib.py
index 94c4099..e0fc6e0 100644
--- a/server/mirrormanager/lib.py
+++ b/server/mirrormanager/lib.py
@@ -116,7 +116,7 @@ def project_template(formname):
def project_dict(formname, template_engine="kid", **kw):
d = dict()
- template = project_template(formname, template_engine)
+ template = project_template(formname)
d['tg_template'] = "%s:%s" % (template_engine, template)
d.update(**kw)
return d
10 years, 4 months
[mirrormanager] master: generate-publiclist and lib: use project_template() (3e6922e)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 3e6922eee801331260c66fe7f7cf2ceadcecff26
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Dec 12 07:05:28 2013 -0600
generate-publiclist and lib: use project_template()
we need the project_template() work to run when generating the static publiclist pages.
>---------------------------------------------------------------
server/generate-publiclist | 3 ++-
server/mirrormanager/lib.py | 12 ++++++++----
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/server/generate-publiclist b/server/generate-publiclist
index 23c5a8c..a965c35 100755
--- a/server/generate-publiclist
+++ b/server/generate-publiclist
@@ -19,6 +19,7 @@ from turbogears.view import engines
from turbogears.database import PackageHub
sys.path.append("/usr/share/mirrormanager/server")
from mirrormanager.model import Product, publiclist_arches, publiclist_hosts
+from mirrormanager.lib import project_template
hub = __connection__ = None
logger = None
@@ -148,7 +149,7 @@ def generate_all():
email = config.get('mirrormanager.report_problems_to_email', 'webmaster')
utcnow = dt.datetime.utcnow()
projectname = config.get('mirrormanager.projectname','Fedora')
- template = engine.load_template('mirrormanager.templates.publiclist')
+ template = engine.load_template(project_template('publiclist'))
generate_one_html(engine, template, pvaMatrix, projectname, products,
arches, email, utcnow)
for p in products:
diff --git a/server/mirrormanager/lib.py b/server/mirrormanager/lib.py
index 36ff56c..94c4099 100644
--- a/server/mirrormanager/lib.py
+++ b/server/mirrormanager/lib.py
@@ -108,11 +108,15 @@ def run_rsync(rsyncpath, extra_rsync_args=None):
tmpfile.seek(0)
return (result, tmpfile)
+
+def project_template(formname):
+ project = config.get("mirrormanager.project_template", "fedora")
+ template = "mirrormanager.templates.%s.%s" % (project, formname)
+ return template
+
def project_dict(formname, template_engine="kid", **kw):
d = dict()
- project = config.get("mirrormanager.project_template", "fedora")
- template = "%s:mirrormanager.templates.%s.%s" % (template_engine,
- project, formname)
- d['tg_template'] = template
+ template = project_template(formname, template_engine)
+ d['tg_template'] = "%s:%s" % (template_engine, template)
d.update(**kw)
return d
10 years, 4 months