Repository :
http://git.fedorahosted.org/cgit/
On branch : master
---------------------------------------------------------------
commit cbe153f8ee8ca00c48f971366021da465f67bd82
Author: Matt Domsch <matt(a)domsch.com>
Date: Fri Jul 19 23:23:18 2013 -0500
crawler: fix os.waitpid(), capture more debugging
The first argument to os.waitpid() was 0, meaning wait until _any_
process in our process group (in our case, the rsync calls by
crawler_perhost) would complete. That's not what we want! We only
care about our own children (the crawler_perhost process itself)
completing. This bug was causing all invocations using rsync to fail,
and of course, only when run under the master crawler, not when
crawler_perhost was run stand-alone, which made it hard to track
down. Ugh.
To help with the tracking, start capturing crawler_perhost stdout and
stderr.
Also be a little more verbose when removing a completed child from the
active_children list.
---------------------------------------------------------------
server/crawler | 8 +++++---
1 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/server/crawler b/server/crawler
index c8fbd19..c565cd3 100755
--- a/server/crawler
+++ b/server/crawler
@@ -53,7 +53,7 @@ class ForkingMaster:
options = os.WNOHANG
while True:
try:
- pid, status = os.waitpid(0, options)
+ pid, status = os.waitpid(-1, options)
except OSError as e:
logger.debug(str(e))
if e.errno == errno.ECHILD:
@@ -69,14 +69,16 @@ class ForkingMaster:
for p in self.active_children:
if p.pid == pid:
self.stop_time(p)
- logger.debug("Removing child pid %d" % p.pid)
+ logger.debug("Removing child pid %d, leaving %d active
children" % (p.pid, len(self.active_children)-1))
self.active_children.remove(p)
return True
def process_request(self, command, args, host):
"""Fork a new subprocess to process the
request."""
logger.info("Starting crawler %s: %s" % (host.name, args))
- p = Popen(args, executable=command, stdin=self.devnull, stdout=self.devnull,
stderr=self.devnull, close_fds=True)
+ stdout = open(os.path.join(options.logdir, '%d-stdout.log' % host.id),
'a')
+ stderr = open(os.path.join(options.logdir, '%d-stderr.log' % host.id),
'a')
+ p = Popen(args, executable=command, stdin=self.devnull, stdout=stdout,
stderr=stderr, close_fds=True)
self.start_time(p, host.id)
logger.debug("Adding child pid %d" % p.pid)
self.active_children.append(p)