[mirrormanager] master: crawler: fix os.waitpid(), capture more debugging (cbe153f)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit cbe153f8ee8ca00c48f971366021da465f67bd82
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 19 23:23:18 2013 -0500
crawler: fix os.waitpid(), capture more debugging
The first argument to os.waitpid() was 0, meaning wait until _any_
process in our process group (in our case, the rsync calls by
crawler_perhost) would complete. That's not what we want! We only
care about our own children (the crawler_perhost process itself)
completing. This bug was causing all invocations using rsync to fail,
and of course, only when run under the master crawler, not when
crawler_perhost was run stand-alone, which made it hard to track
down. Ugh.
To help with the tracking, start capturing crawler_perhost stdout and
stderr.
Also be a little more verbose when removing a completed child from the
active_children list.
>---------------------------------------------------------------
server/crawler | 8 +++++---
1 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/server/crawler b/server/crawler
index c8fbd19..c565cd3 100755
--- a/server/crawler
+++ b/server/crawler
@@ -53,7 +53,7 @@ class ForkingMaster:
options = os.WNOHANG
while True:
try:
- pid, status = os.waitpid(0, options)
+ pid, status = os.waitpid(-1, options)
except OSError as e:
logger.debug(str(e))
if e.errno == errno.ECHILD:
@@ -69,14 +69,16 @@ class ForkingMaster:
for p in self.active_children:
if p.pid == pid:
self.stop_time(p)
- logger.debug("Removing child pid %d" % p.pid)
+ logger.debug("Removing child pid %d, leaving %d active children" % (p.pid, len(self.active_children)-1))
self.active_children.remove(p)
return True
def process_request(self, command, args, host):
"""Fork a new subprocess to process the request."""
logger.info("Starting crawler %s: %s" % (host.name, args))
- p = Popen(args, executable=command, stdin=self.devnull, stdout=self.devnull, stderr=self.devnull, close_fds=True)
+ stdout = open(os.path.join(options.logdir, '%d-stdout.log' % host.id), 'a')
+ stderr = open(os.path.join(options.logdir, '%d-stderr.log' % host.id), 'a')
+ p = Popen(args, executable=command, stdin=self.devnull, stdout=stdout, stderr=stderr, close_fds=True)
self.start_time(p, host.id)
logger.debug("Adding child pid %d" % p.pid)
self.active_children.append(p)
10 years, 9 months
[mirrormanager] master: crawler: fix wait_for_completion() (d63ca58)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit d63ca58079239e319e0af90c39d3da2567582a81
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 19 16:47:27 2013 -0500
crawler: fix wait_for_completion()
>---------------------------------------------------------------
server/crawler | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/server/crawler b/server/crawler
index d0d89f1..c8fbd19 100755
--- a/server/crawler
+++ b/server/crawler
@@ -90,7 +90,9 @@ class ForkingMaster:
def wait_for_completion(self):
self.max_children = 0
- self.wait_for_available_slot()
+ while len(self.active_children):
+ self.collect_children()
+ time.sleep(1)
def start_time(self, p, hostid):
now = dt.datetime.utcnow()
10 years, 9 months
[mirrormanager] master: fix use of orderBy in select statements (a6e0472)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit a6e04720c0e2e94340f800bd56fd1645ad9eb471
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 19 16:37:34 2013 -0500
fix use of orderBy in select statements
proper use is select().orderBy(), not select(orderBy=...). Otherwise
failure ensues.
>---------------------------------------------------------------
server/errorcheck | 2 +-
server/generate-publiclist | 2 +-
server/mirrormanager/controllers.py | 6 +++---
server/mirrormanager/model.py | 4 ++--
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/server/errorcheck b/server/errorcheck
index a3ca134..cb11a99 100755
--- a/server/errorcheck
+++ b/server/errorcheck
@@ -450,7 +450,7 @@ def doit():
if len(options.site):
sites = [Site.get(id) for id in options.site]
else:
- sites = Site.select(orderBy='id')
+ sites = Site.select().orderBy('id')
for site in sites:
try:
diff --git a/server/generate-publiclist b/server/generate-publiclist
index 0e1e22f..64c923c 100755
--- a/server/generate-publiclist
+++ b/server/generate-publiclist
@@ -176,7 +176,7 @@ def generate_one_html(engine, pvaMatrix, productname=None, vername=None, archnam
template_data = dict(page_title=title, hosts=hosts, numhosts=len(hosts), bandwidth=bandwidth,
pvaMatrix=pvaMatrix,
pva=pva,
- products=list(Product.select(orderBy='name')),
+ products=list(Product.select().orderBy('name')),
arches=publiclist_arches(),
report_problems_to_email = config.get('mirrormanager.report_problems_to_email', 'webmaster'),
utcnow = dt.datetime.utcnow())
diff --git a/server/mirrormanager/controllers.py b/server/mirrormanager/controllers.py
index 28efdc7..a1d2696 100644
--- a/server/mirrormanager/controllers.py
+++ b/server/mirrormanager/controllers.py
@@ -237,7 +237,7 @@ class SiteAdminController(controllers.Controller, identity.SecureResource, conte
##############################################
class SiteToSiteFields(widgets.WidgetsList):
def get_sites_options():
- return [(s.id, s.name) for s in Site.select(orderBy='name')]
+ return [(s.id, s.name) for s in Site.select().orderBy('name')]
sites = widgets.MultipleSelectField(options=get_sites_options, size=15,
validator=validators.NotEmpty())
@@ -433,7 +433,7 @@ class HostController(controllers.Controller, identity.SecureResource, content):
##################################################################33
class HostCategoryFieldsNew(widgets.WidgetsList):
def get_category_options():
- return [(c.id, c.name) for c in Category.select(orderBy='name')]
+ return [(c.id, c.name) for c in Category.select().orderBy('name')]
category = widgets.SingleSelectField(options=get_category_options,
validator=validators.NotEmpty())
always_up2date = widgets.CheckBox(default=False, help_text="Set to force belief that the whole category is always in sync.")
@@ -856,7 +856,7 @@ class Root(controllers.RootController):
@identity.require(identity.not_anonymous())
def help(self):
if admin_group in identity.current.groups:
- sites = Site.select(orderBy='name')
+ sites = Site.select().orderBy('name')
else:
sites = user_sites(identity)
return {}
diff --git a/server/mirrormanager/model.py b/server/mirrormanager/model.py
index 6d88011..b988e54 100644
--- a/server/mirrormanager/model.py
+++ b/server/mirrormanager/model.py
@@ -661,7 +661,7 @@ class Arch(SQLObject):
primaryArch = BoolCol(default=True)
def publiclist_arches():
- return list(Arch.selectBy(publiclist=True, orderBy=['-primaryArch', 'name']))
+ return list(Arch.selectBy(publiclist=True).orderBy(['-primaryArch', 'name']))
# e.g. 'fedora' and 'epel'
class Product(SQLObject):
@@ -681,7 +681,7 @@ class Product(SQLObject):
@staticmethod
def selectFieldOptions():
- return [(p.id, p.name) for p in Product.select(orderBy='name')]
+ return [(p.id, p.name) for p in Product.select().orderBy('name')]
class Version(SQLObject):
10 years, 9 months
[mirrormanager] master: publiclist: add utcnow timestamp to bottom of each page (8cd0f6d)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 8cd0f6dd59d40d1947b9e51310fbf57b728bf495
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 19 16:02:38 2013 -0500
publiclist: add utcnow timestamp to bottom of each page
>---------------------------------------------------------------
server/generate-publiclist | 4 +++-
server/mirrormanager/templates/publiclist.kid | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/server/generate-publiclist b/server/generate-publiclist
index 0c27be8..0e1e22f 100755
--- a/server/generate-publiclist
+++ b/server/generate-publiclist
@@ -5,6 +5,7 @@ pkg_resources.require("TurboGears")
from sqlobject import *
import sys, os, string, errno
+import datetime as dt
import turbogears
import turbogears.util as tg_util
from turbogears import view, database, errorhandling, config
@@ -177,7 +178,8 @@ def generate_one_html(engine, pvaMatrix, productname=None, vername=None, archnam
pva=pva,
products=list(Product.select(orderBy='name')),
arches=publiclist_arches(),
- report_problems_to_email = config.get('mirrormanager.report_problems_to_email', 'webmaster'))
+ report_problems_to_email = config.get('mirrormanager.report_problems_to_email', 'webmaster'),
+ utcnow = dt.datetime.utcnow())
t=engine.load_template('mirrormanager.templates.publiclist')
out_html = _process_output(template_data, template=t, format='html', engine=engine)
diff --git a/server/mirrormanager/templates/publiclist.kid b/server/mirrormanager/templates/publiclist.kid
index 9dab87b..8b1279d 100644
--- a/server/mirrormanager/templates/publiclist.kid
+++ b/server/mirrormanager/templates/publiclist.kid
@@ -102,7 +102,8 @@ and development networks globally.
<div id="footer">
<p class="disclaimer">
Listing generated
- by <a href="http://mirrormanager.org/">MirrorManager</a>.
+ by <a href="http://mirrormanager.org/">MirrorManager</a> at
+ ${utcnow} UTC.
Please direct questions or problems with this list to ${report_problems_to_email}.
</p>
</div>
10 years, 9 months
[mirrormanager] master: crawler_perhost: drop threading, use SIGALRM. Don't nuke on FTP timeout. (491f6b9)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 491f6b9566972bbd395f668ecf0e23b6f6c11c8b
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 19 15:15:22 2013 -0500
crawler_perhost: drop threading, use SIGALRM. Don't nuke on FTP timeout.
>---------------------------------------------------------------
server/crawler_perhost | 20 +++++++++++---------
1 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/server/crawler_perhost b/server/crawler_perhost
index 283ac79..cb04cca 100755
--- a/server/crawler_perhost
+++ b/server/crawler_perhost
@@ -17,7 +17,6 @@ from smtplib import SMTP
import socket
import time
from urlparse import urlsplit
-import threading
import urlgrabber
from sqlobject import SQLObjectNotFound
@@ -659,6 +658,12 @@ def select_host_categories_to_scan(options):
return result
+def sigalrm_handler(signal, stackframe):
+ logger.warning("SIGALRM received, exiting.")
+ global must_dienow
+ must_dienow = True
+
+
def per_host(host, options):
"""Canary mode looks for 2 things:
directory.path ends in 'iso' or directory.path ends in 'repodata'. In
@@ -676,15 +681,14 @@ def per_host(host, options):
http_debuglevel = 2
ftp_debuglevel = 2
- timer = threading.Timer(options.timeout_minutes * 60.0, dienow)
+
if options.timeout_minutes > 0:
- timer.start()
+ signal.alarm(options.timeout_minutes * 60)
hoststate = hostState(http_debuglevel=http_debuglevel, ftp_debuglevel=ftp_debuglevel)
for hc in select_host_categories_to_scan(options):
if must_dienow:
- timer.cancel()
raise TimeoutException
if hc.always_up2date:
@@ -703,7 +707,6 @@ def per_host(host, options):
try:
has_all_files = try_percategory(trydirs, categoryUrl, host_category_dirs, hc, host, categoryPrefixLen, options)
except TimeoutException:
- timer.cancel()
raise
if type(has_all_files) == type(True):
@@ -719,7 +722,6 @@ def per_host(host, options):
try_later_delay = 1
for d in trydirs:
if must_dienow:
- timer.cancel()
raise TimeoutException
if not d.readable:
@@ -762,9 +764,9 @@ def per_host(host, options):
rc = 1
break
- timer.cancel()
hoststate.close()
+ signal.alarm(0) # we got this far, don't alarm out now!
if rc == 0:
if len(host_category_dirs) > 0:
sync_hcds(host, host_category_dirs)
@@ -817,7 +819,7 @@ def main():
__connection__ = hub
os.chdir('/tmp')
- signal.signal(signal.SIGHUP, signal.SIG_IGN)
+ signal.signal(signal.SIGALRM, sigalrm_handler)
global logger
if options.logfile is not None:
@@ -845,7 +847,7 @@ def main():
try:
rc = per_host(host.id, options)
except TimeoutException:
- mark_not_up2date(None, host.id, "Crawler timed out before completing. Host is likely overloaded.")
+# mark_not_up2date(None, host.id, "Crawler timed out before completing. Host is likely overloaded.")
rc = 2
logger.info("Ending crawl")
10 years, 9 months
[mirrormanager] master: crawler: rethink child reaping, other fixes (e812c19)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit e812c19a8202a92cdcfbfcac8d6792454e5ff878
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 19 15:14:34 2013 -0500
crawler: rethink child reaping, other fixes
>---------------------------------------------------------------
server/crawler | 117 +++++++++++++++++++++++++++++++++++++++-----------------
1 files changed, 82 insertions(+), 35 deletions(-)
diff --git a/server/crawler b/server/crawler
index 9842ae5..d0d89f1 100755
--- a/server/crawler
+++ b/server/crawler
@@ -4,9 +4,11 @@ import pkg_resources
pkg_resources.require("TurboGears")
import datetime as dt
+import errno
import logging
from optparse import OptionParser
import os
+import signal
from subprocess import Popen
import sys
import time
@@ -26,69 +28,102 @@ logger = None
class ForkingMaster:
def __init__(self, max_children = 10):
- self.active_children = None
+ self.active_children = []
self.max_children = max_children
self.devnull = open('/dev/null', 'rw')
self.timings = {}
+
+ def check_timedout_children(self):
+ now = dt.datetime.utcnow()
+ for child in self.active_children:
+ if child.kill_time < now:
+ try:
+ os.kill(child.pid, signal.SIGKILL) # SIGTERM wasn't enough
+ logger.info('Killed process %d' % child.pid)
+ except: # the process could be gone
+ pass
+ else:
+ # items lower on this list are newer, no need to check
+ break
+ return None
+
def collect_children(self):
"""Internal routine to wait for died children."""
- while self.active_children:
- if len(self.active_children) < self.max_children:
- options = os.WNOHANG
- else:
- # If the maximum number of children are already
- # running, block while waiting for a child to exit
- options = 0
+ options = os.WNOHANG
+ while True:
try:
pid, status = os.waitpid(0, options)
- except os.error:
+ except OSError as e:
+ logger.debug(str(e))
+ if e.errno == errno.ECHILD:
+ logger.debug("Got ECHILD. Number of active children: %d/%d" % (len(self.active_children), self.max_children))
+ return False
pid = None
- if not pid: break
- self.active_children.remove(pid)
- self.stop_time(pid)
+
+ if not pid:
+ # no child was ready, see if any should be killed
+ self.check_timedout_children()
+ return
+ # a child should be reaped
+ for p in self.active_children:
+ if p.pid == pid:
+ self.stop_time(p)
+ logger.debug("Removing child pid %d" % p.pid)
+ self.active_children.remove(p)
+ return True
def process_request(self, command, args, host):
"""Fork a new subprocess to process the request."""
- self.collect_children()
- logging.info("Starting crawler %s: %s" % (host.name, args))
+ logger.info("Starting crawler %s: %s" % (host.name, args))
p = Popen(args, executable=command, stdin=self.devnull, stdout=self.devnull, stderr=self.devnull, close_fds=True)
- self.start_time(p.pid, host.id)
- # Parent process
- if self.active_children is None:
- self.active_children = []
- self.active_children.append(p.pid)
+ self.start_time(p, host.id)
+ logger.debug("Adding child pid %d" % p.pid)
+ self.active_children.append(p)
+ logger.debug("Number of active children now: %d" % len(self.active_children))
+
+ def wait_for_available_slot(self):
+ logger.debug("Waiting for a slot: Number of active children: %d/%d" % (len(self.active_children), self.max_children))
+ while len(self.active_children) >= self.max_children:
+ self.collect_children()
+ time.sleep(1)
def wait_for_completion(self):
self.max_children = 0
- self.collect_children()
+ self.wait_for_available_slot()
- def start_time(self, pid, hostid):
- self.timings[pid] = dict()
- self.timings[pid]['start'] = dt.datetime.utcnow()
- self.timings[pid]['hostid'] = hostid
+ def start_time(self, p, hostid):
+ now = dt.datetime.utcnow()
+ p.kill_time = now + dt.timedelta(seconds=(options.timeout_minutes * 60))
+ self.timings[p.pid] = dict()
+ self.timings[p.pid]['start'] = now
+ self.timings[p.pid]['hostid'] = hostid
- def stop_time(self, pid):
- self.timings[pid]['stop'] = dt.datetime.utcnow()
+ def stop_time(self, p):
+ self.timings[p.pid]['stop'] = dt.datetime.utcnow()
- diff = self.timings[pid]['stop'] - self.timings[pid]['start']
- host = Host.get(self.timings[pid]['hostid'])
+ diff = self.timings[p.pid]['stop'] - self.timings[p.pid]['start']
+ host = Host.get(self.timings[p.pid]['hostid'])
logger.info('Host %s (id=%s) crawl time %s' % (host.name, host.id, str(diff)))
- del self.timings[pid]
+ del self.timings[p.pid]
+
+
def doit():
master = ForkingMaster(max_children=options.threads)
- command = '/usr/share/mirrormanager/server/crawler_perhost'
- commonargs = [ command, '-c', options.config, '--timeout-minutes', '90']
+ commonargs = [ options.crawler_perhost, '-c', options.config, '--debug']
if options.canary:
commonargs.append('--canary')
-
numhosts = Host.selectBy(private=False).count()
i = 0
- for h in Host.selectBy(private=False).orderBy('id'):
+ for h in list(Host.selectBy(private=False).orderBy('id')):
i += 1
try:
+ if h.id < options.startid: continue
+ if h.id >= options.stopid: break
+ master.wait_for_available_slot()
+
if not (h.admin_active and h.user_active and h.site.user_active and h.site.admin_active):
continue
if h.site.private:
@@ -100,7 +135,7 @@ def doit():
hostargs.extend(['--logfile', logfilename])
args = commonargs + hostargs
logger.debug('starting crawler for host %s (id=%d) %d/%d' % (h.name, h.id, i, numhosts))
- master.process_request(command, args, h)
+ master.process_request(options.crawler_perhost, args, h)
except AssertionError: # someone deleted our host while we were looking at it
continue
@@ -125,7 +160,7 @@ def main():
parser = OptionParser(usage=sys.argv[0] + " [options]")
parser.add_option("-c", "--config",
- dest="config", default='dev.cfg',
+ dest="config", default='/etc/mirrormanager/prod.cfg',
help="TurboGears config file to use")
parser.add_option("--include-private",
@@ -138,9 +173,21 @@ def main():
parser.add_option("-l", "--logdir", type="string", metavar="DIR",
dest="logdir", default='/var/log/mirrormanager/crawler',
help="write individual host logfiles to DIR")
+ parser.add_option("--timeout-minutes", type="int",
+ dest="timeout_minutes", default=90,
+ help="per-host timeout, in minutes")
parser.add_option("--logfile", type="string", metavar="FILE",
dest="logfile", default='/var/log/mirrormanager/crawler.log',
help="write logfile to FILE")
+ parser.add_option("--startid", type="int", metavar="ID",
+ dest="startid", default=0,
+ help="Start crawling at host ID (default=0)")
+ parser.add_option("--stopid", type="int", metavar="ID",
+ dest="stopid", default=sys.maxint,
+ help="Stop crawling before host ID (default=maxint)")
+ parser.add_option("--crawler_perhost", type="string", metavar="FILE",
+ dest="crawler_perhost", default='/usr/share/mirrormanager/server/crawler_perhost',
+ help="Per-host crawler executable (default=/usr/share/mirrormanager/server/crawler_perhost")
parser.add_option("--canary",
dest="canary", action="store_true", default=False,
10 years, 9 months
[mirrormanager] master: don't SIGHUP crawler_perhost anymore (7055579)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 705557931cb25e7696cffc46d51f12fb844473ff
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Jul 18 23:37:16 2013 -0500
don't SIGHUP crawler_perhost anymore
>---------------------------------------------------------------
server/logrotate.conf | 1 -
1 files changed, 0 insertions(+), 1 deletions(-)
diff --git a/server/logrotate.conf b/server/logrotate.conf
index dc9dbe6..1eed251 100644
--- a/server/logrotate.conf
+++ b/server/logrotate.conf
@@ -16,6 +16,5 @@
dateext
rotate 15
postrotate
- /usr/bin/killall -HUP crawler_perhost
endscript
}
10 years, 9 months
[mirrormanager] master: crawler: drop to 90 minute timeout, fix logger line endings (d62e372)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit d62e37263583ed42b0d74cfcfa779116d13a68ea
Author: Matt Domsch <matt(a)domsch.com>
Date: Thu Jul 18 14:32:56 2013 -0500
crawler: drop to 90 minute timeout, fix logger line endings
>---------------------------------------------------------------
server/crawler | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/server/crawler b/server/crawler
index 28c2136..9842ae5 100755
--- a/server/crawler
+++ b/server/crawler
@@ -79,7 +79,7 @@ class ForkingMaster:
def doit():
master = ForkingMaster(max_children=options.threads)
command = '/usr/share/mirrormanager/server/crawler_perhost'
- commonargs = [ command, '-c', options.config, '--timeout-minutes', '360']
+ commonargs = [ command, '-c', options.config, '--timeout-minutes', '90']
if options.canary:
commonargs.append('--canary')
@@ -107,7 +107,7 @@ def doit():
master.wait_for_completion()
def setup_logger():
- fmt = "%(asctime)s %(message)s\n"
+ fmt = "%(asctime)s %(message)s"
formatter = logging.Formatter(fmt=fmt)
formatter.converter = time.gmtime
logger = logging.getLogger('crawler')
10 years, 9 months
[mirrormanager] master: more crawler cleanups (3a91716)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit 3a9171673320c29daf17a1c50681ff4ee3f7ca07
Author: Matt Domsch <matt(a)domsch.com>
Date: Tue Jul 16 23:16:26 2013 -0500
more crawler cleanups
>---------------------------------------------------------------
server/crawler | 71 +++++++++++++++++++++++++++++++------------------------
1 files changed, 40 insertions(+), 31 deletions(-)
diff --git a/server/crawler b/server/crawler
index 072be5b..28c2136 100755
--- a/server/crawler
+++ b/server/crawler
@@ -9,22 +9,23 @@ from optparse import OptionParser
import os
from subprocess import Popen
import sys
+import time
-from sqlobject import *
import turbogears
sys.path.append('/usr/share/mirrormanager/server')
-from mirrormanager.model import *
+from mirrormanager.model import Host
from mirrormanager.lib import manage_pidfile, remove_pidfile
pidfile='/var/run/mirrormanager/crawler.pid'
+options = None
from turbogears.database import PackageHub
hub = __connection__ = None
logger = None
class ForkingMaster:
- def __init__(self, max_children=10):
+ def __init__(self, max_children = 10):
self.active_children = None
self.max_children = max_children
self.devnull = open('/dev/null', 'rw')
@@ -75,34 +76,49 @@ class ForkingMaster:
logger.info('Host %s (id=%s) crawl time %s' % (host.name, host.id, str(diff)))
del self.timings[pid]
-def doit(options, include_private=False, threads=10, config=None, logdir=None):
- master = ForkingMaster(max_children=threads)
+def doit():
+ master = ForkingMaster(max_children=options.threads)
command = '/usr/share/mirrormanager/server/crawler_perhost'
- commonargs = [ command, '-c', config]
+ commonargs = [ command, '-c', options.config, '--timeout-minutes', '360']
if options.canary:
commonargs.append('--canary')
- for h in Host.select():
- commonargs = [ command, '-c', config]
- if options.canary:
- commonargs.append('--canary')
- if not h.is_active():
- continue
- if h.is_private() and not (include_private or (h.internet2 and h.internet2_clients)):
- continue
- commonargs.append('--include-private')
-
- args = commonargs + ['--hostid=%s' % h.id]
- if logdir is not None:
- logfilename = os.path.join(logdir, str(h.id) + '.log')
- args.append('--logfile=%s' % logfilename)
- master.process_request(command, args, h)
+ numhosts = Host.selectBy(private=False).count()
+ i = 0
+ for h in Host.selectBy(private=False).orderBy('id'):
+ i += 1
+ try:
+ if not (h.admin_active and h.user_active and h.site.user_active and h.site.admin_active):
+ continue
+ if h.site.private:
+ continue
+
+ hostargs = []
+ hostargs.extend(['--hostid', str(h.id)])
+ logfilename = os.path.join(options.logdir, str(h.id) + '.log')
+ hostargs.extend(['--logfile', logfilename])
+ args = commonargs + hostargs
+ logger.debug('starting crawler for host %s (id=%d) %d/%d' % (h.name, h.id, i, numhosts))
+ master.process_request(command, args, h)
+ except AssertionError: # someone deleted our host while we were looking at it
+ continue
master.wait_for_completion()
+def setup_logger():
+ fmt = "%(asctime)s %(message)s\n"
+ formatter = logging.Formatter(fmt=fmt)
+ formatter.converter = time.gmtime
+ logger = logging.getLogger('crawler')
+ handler = logging.handlers.WatchedFileHandler(options.logfile, "a+b")
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+ logger.setLevel(logging.DEBUG)
+ return logger
def main():
+ global options
if manage_pidfile(pidfile):
print "another instance is running, try again later."
return 1
@@ -140,16 +156,9 @@ def main():
__connection__ = hub
global logger
- if options.logfile is not None:
- fmt ='%(asctime)s %(message)s'
- datefmt ='%m/%d/%Y %I:%M:%S %p'
- formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
- logger = logging.getLogger('crawler_perhost')
- handler = logging.handlers.WatchedFileHandler(options.logfile, "a+b")
- handler.setFormatter(formatter)
- logger.addHandler(handler)
-
- doit(options, include_private=options.include_private, threads=options.threads, config=options.config, logdir=options.logdir)
+ logger = setup_logger()
+
+ doit()
remove_pidfile(pidfile)
return 0
10 years, 9 months
[mirrormanager] master: crawler: don't crawl private mirrors - really (b631ea4)
by Matt Domsch
Repository : http://git.fedorahosted.org/cgit/
On branch : master
>---------------------------------------------------------------
commit b631ea4e2f77d3eef8d4a1b6e5433928150065e5
Author: Matt Domsch <matt(a)domsch.com>
Date: Tue Jul 16 10:38:48 2013 -0500
crawler: don't crawl private mirrors - really
two bugs fixed:
1) changing the value of include_private inside the loop means all
subsequent mirrors, private or not, get crawled. Whoops
2) testing for if h.is_private instead of is_private() (the result of
the function) means that was always evaluating true. Stupid.
>---------------------------------------------------------------
server/crawler | 6 ++----
1 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/server/crawler b/server/crawler
index eb2941a..072be5b 100755
--- a/server/crawler
+++ b/server/crawler
@@ -88,11 +88,9 @@ def doit(options, include_private=False, threads=10, config=None, logdir=None):
commonargs.append('--canary')
if not h.is_active():
continue
- if include_private or (h.internet2 and h.internet2_clients):
- commonargs.append('--include-private')
- include_private = True
- if not include_private and h.is_private():
+ if h.is_private() and not (include_private or (h.internet2 and h.internet2_clients)):
continue
+ commonargs.append('--include-private')
args = commonargs + ['--hostid=%s' % h.id]
if logdir is not None:
10 years, 9 months