Author: tmckay
Date: 2012-05-14 23:15:47 +0000 (Mon, 14 May 2012)
New Revision: 5375
Modified:
trunk/cumin/bin/cumin
trunk/cumin/bin/cumin-admin
trunk/cumin/bin/cumin-data
trunk/cumin/bin/cumin-report
trunk/cumin/bin/cumin-web
trunk/cumin/etc/sysvinit-cumin
trunk/cumin/python/cumin/config.py
trunk/cumin/python/cumin/util.py
Log:
Comprehensive changes to the shutdown mechanism using stdin inside of signals.
* Master communicates with children over stdin (redirected to a pipe).
* If a child sees input on stdin matching the --es option value, it will exit.
* Children launch shutdown routines in a daemon thread and call join() from the main
thread
* with a timeout of --tm. This guarantees exit after --tm seconds.
* The master sets the --tm option for children based on its own --timeout option.
* The master will exit after --timeout seconds, even if children have not exited
* since they are guaranteed to do so. This allows initd to to track the pid of
* the master script with confidence.
* Initd script uses the killproc form that sends a SIGTERM and checks the pid
* with a delay before issuing SIGKILL. The delay is the same value that is
* passed as the --timeout option to the master.
* Artificial 3 second delay between stop and start on restart has been removed.
BZ800065
Modified: trunk/cumin/bin/cumin
===================================================================
--- trunk/cumin/bin/cumin 2012-05-14 13:32:02 UTC (rev 5374)
+++ trunk/cumin/bin/cumin 2012-05-14 23:15:47 UTC (rev 5375)
@@ -18,6 +18,9 @@
from parsley.loggingex import *
log = logging.getLogger("cumin.master")
+# Termination string to children
+exit_s = "exit"
+
def call_sys_exit(sig,frame):
sys.exit()
@@ -32,8 +35,8 @@
return 1
return 0
-def get_args(app, section, init_only, console, extra_options=""):
- args = [app, "--section="+section.strip()]
+def get_args(app, section, init_only, console, es, timeout, extra_options):
+ args = [app, "--section="+section.strip(), "--es="+es.strip(),
"--tm="+str(timeout)]
if init_only:
args.append("--init-only")
if not console:
@@ -87,6 +90,12 @@
parser.add_option("--console", dest="console",
action="store_true", default=False,
help="Log to stderr rather than master.log, no IO
redirection for children.")
+ parser.add_option("--timeout", dest="timeout",
default="5", type=int,
+ help="Time that the master will wait for child "\
+ "processes to exit before exiting itself. Also
"\
+ "the time child processes will allow themselves
"\
+ "to shutdown before calling sys.exit")
+
parser.add_option("--web-options", dest="web_options",
default="", type=str,
help="Additional options string to pass to web
instances."\
"\nEnclose in quotes, options must be --option form,
splits on spaces."\
@@ -142,17 +151,26 @@
apps = []
if len(options.webs) > 0:
for instance in options.webs.split(','):
- args, prog_string = get_args("cumin-web", instance,
options.init_only, console, options.web_options)
+ args, prog_string = get_args("cumin-web", instance,
+ options.init_only, console,
+ exit_s, options.timeout,
+ options.web_options)
apps.append([None, args, prog_string])
if len(options.datas) > 0:
for instance in options.datas.split(','):
- args, prog_string = get_args("cumin-data", instance,
options.init_only, console, options.data_options)
+ args, prog_string = get_args("cumin-data", instance,
+ options.init_only, console,
+ exit_s, options.timeout,
+ options.data_options)
apps.append([None, args, prog_string])
if len(options.reports) > 0:
for instance in options.reports.split(','):
- args, prog_string = get_args("cumin-report", instance,
options.init_only, console, options.report_options)
+ args, prog_string = get_args("cumin-report", instance,
+ options.init_only, console,
+ exit_s, options.timeout,
+ options.report_options)
apps.append([None, args, prog_string])
# Launch and babysit, do not restart if options.init_only is set
@@ -162,7 +180,7 @@
slow_down = 40 # slow down polling after initial period
try:
def start(app, verb):
- app[PROCESS] = subprocess.Popen(app[ARGS])
+ app[PROCESS] = subprocess.Popen(app[ARGS], stdin=subprocess.PIPE)
if app[PROCESS]:
log.info(verb+"ed subprocess (pid %s): %s" %\
(app[PROCESS].pid, app[PROG_STRING]))
@@ -212,15 +230,23 @@
else:
start(app, "Restart")
finally:
- # Try a ctrl-C first
- log.info("Send SIGTERM to all children")
+ log.info("Write termination string to all children")
complete = 0
for app in apps:
if app[PROCESS]:
complete += 1
- os.kill(app[PROCESS].pid, signal.SIGTERM)
+ try:
+ app[PROCESS].stdin.write(exit_s+"\n")
+ except:
+ log.debug("Writing termination failed for subprocess (%s)"
\
+ % app[PROCESS].pid)
- # Give children 10 seconds to exit, then bail
+ # Wait for children to exit so we can record the result and so that
+ # the initd script can track the pid correctly. Exit regardless after
+ # timeout+0.5 seconds, even if the children are still running. Each child
+ # will have a hard exit after timeout seconds from its own main thread,
+ # even if its shutdown activities have not completed, so in most
+ # cases the children *should* exit within timeout+0.5 seconds
then = datetime.now()
while complete != 0:
for app in apps:
@@ -235,13 +261,12 @@
break
sleep(0.25)
- if datetime.now() - then > timedelta(seconds=5):
- log.warn("Timed out waiting for children, send SIGKILL")
+ if datetime.now() - then > timedelta(seconds=options.timeout+0.5):
+ log.warn("Timed out waiting for children, exiting")
for app in apps: # just to be paranoid
if app[PROCESS]:
- log.warn("Send SIGKILL to subprocess (%s)"\
+ log.warn("Subprocess (%s) failed to stop"\
% app[PROCESS].pid)
- os.kill(app[PROCESS].pid, signal.SIGKILL)
break
return return_code
Modified: trunk/cumin/bin/cumin-admin
===================================================================
--- trunk/cumin/bin/cumin-admin 2012-05-14 13:32:02 UTC (rev 5374)
+++ trunk/cumin/bin/cumin-admin 2012-05-14 23:15:47 UTC (rev 5375)
@@ -41,6 +41,7 @@
# Get options
parser = CuminOptionParser()
+ parser.remove_option("--es")
parser.set_usage(generate_usage())
opts, args = parser.parse_args()
Modified: trunk/cumin/bin/cumin-data
===================================================================
--- trunk/cumin/bin/cumin-data 2012-05-14 13:32:02 UTC (rev 5374)
+++ trunk/cumin/bin/cumin-data 2012-05-14 23:15:47 UTC (rev 5375)
@@ -216,6 +216,8 @@
stats = mint.update_thread.stats
count = 0
+ exit_msg = "Exiting on termination string from master"
+
if opts.print_stats:
print "[Starred columns are the number of events per second]"
@@ -226,11 +228,20 @@
count += 1
stats.print_values()
-
- sleep(5)
+ if opts.es:
+ if poll_stdin(5, value=opts.es):
+ log.debug(exit_msg)
+ break
+ else:
+ sleep(5)
else:
while True:
- sleep(86400)
+ if opts.es:
+ if poll_stdin(86400, value=opts.es):
+ log.debug(exit_msg)
+ break
+ else:
+ sleep(86400)
except KeyboardInterrupt:
log.info("Received shutdown signal")
@@ -260,15 +271,29 @@
print_exc()
return_code = CuminErrors.UNHANDLED_ERROR
- if mint:
- mint.stop()
- if pipeThread:
- pipeThread.stop()
- log.info("about to call logging shutdown")
- logging.shutdown()
+ def shutdown():
+ if mint:
+ mint.stop()
+ if pipeThread:
+ pipeThread.stop()
+ log.info("about to call logging shutdown")
+ logging.shutdown()
+
+ # Run shutdown with a timeout in the main thread.
+ # This guarantees exit.
+ t = ShutdownThread(shutdown)
+ t.start()
+
+ # If we get some error before options are parsed opts.tm won't be set
+ try:
+ tm = opts.tm
+ except:
+ tm = 5
+ t.join(tm)
return adjust_return(passed_init, return_code)
def make_ctrl_c(sig, frame):
+ log.debug("Signal received, raising KeyboardInterrupt")
raise KeyboardInterrupt
if __name__ == "__main__":
Modified: trunk/cumin/bin/cumin-report
===================================================================
--- trunk/cumin/bin/cumin-report 2012-05-14 13:32:02 UTC (rev 5374)
+++ trunk/cumin/bin/cumin-report 2012-05-14 23:15:47 UTC (rev 5375)
@@ -193,6 +193,8 @@
stats = mint.update_thread.stats
count = 0
+ exit_msg = "Exiting on termination string from master"
+
if opts.print_stats:
print "[Starred columns are the number of events per second]"
@@ -203,11 +205,20 @@
count += 1
stats.print_values()
-
- sleep(5)
+ if opts.es:
+ if poll_stdin(5, value=opts.es):
+ log.debug(exit_msg)
+ break
+ else:
+ sleep(5)
else:
while True:
- sleep(86400)
+ if opts.es:
+ if poll_stdin(86400, value=opts.es):
+ log.debug(exit_msg)
+ break
+ else:
+ sleep(86400)
except KeyboardInterrupt:
log.info("Received shutdown signal")
@@ -237,12 +248,26 @@
print_exc()
return_code = CuminErrors.UNHANDLED_ERROR
- if mint:
- mint.stop()
- if pipeThread:
- pipeThread.stop()
- log.info("about to call logging shutdown")
- logging.shutdown()
+ def shutdown():
+ if mint:
+ mint.stop()
+ if pipeThread:
+ pipeThread.stop()
+ log.info("about to call logging shutdown")
+ logging.shutdown()
+
+ # Run shutdown with a timeout in the main thread.
+ # This guarantees exit.
+ t = ShutdownThread(shutdown)
+ t.start()
+
+ # If we get some error before options are parsed opts.tm won't be set
+ try:
+ tm = opts.tm
+ except:
+ tm = 5
+ t.join(tm)
+
return adjust_return(passed_init, return_code)
def make_ctrl_c(sig, frame):
Modified: trunk/cumin/bin/cumin-web
===================================================================
--- trunk/cumin/bin/cumin-web 2012-05-14 13:32:02 UTC (rev 5374)
+++ trunk/cumin/bin/cumin-web 2012-05-14 23:15:47 UTC (rev 5375)
@@ -233,7 +233,13 @@
cumin.start()
while True:
# print_threads()
- sleep(1)
+ if opts.es:
+ if poll_stdin(1, value=opts.es):
+ log.debug("Exiting on termination string from master")
+ break
+ else:
+ sleep(1)
+
if not cumin.server_alive():
print "web server has stopped, exiting..."
log.error("Web server process has stopped")
@@ -274,12 +280,25 @@
traceback.print_exc()
return_code = CuminErrors.UNHANDLED_ERROR
- if cumin:
- cumin.stop()
- if pipeThread:
- pipeThread.stop()
- log.info("about to call logging shutdown")
- logging.shutdown()
+ def shutdown():
+ if cumin:
+ cumin.stop()
+ if pipeThread:
+ pipeThread.stop()
+ log.info("about to call logging shutdown")
+ logging.shutdown()
+
+ # Run shutdown with a timeout in the main thread.
+ # This guarantees exit.
+ t = ShutdownThread(shutdown)
+ t.start()
+
+ # If we get some error before options are parsed opts.tm won't be set
+ try:
+ tm = opts.tm
+ except:
+ tm = 5
+ t.join(tm)
return adjust_return(passed_init, return_code)
def make_ctrl_c(sig, frame):
Modified: trunk/cumin/etc/sysvinit-cumin
===================================================================
--- trunk/cumin/etc/sysvinit-cumin 2012-05-14 13:32:02 UTC (rev 5374)
+++ trunk/cumin/etc/sysvinit-cumin 2012-05-14 23:15:47 UTC (rev 5375)
@@ -18,6 +18,7 @@
servicename=cumin
RETVAL=0
+DELAY=5
start() {
@@ -86,14 +87,14 @@
# Launch for real now...Since we already did an init check assume success
# Squelch all IO, let cumin redirect.
- daemon --user cumin --check $servicename $processname $CUMIN_OPTIONS < /dev/null
&> /dev/null \&
+ daemon --user cumin --check $servicename $processname "--timeout="$DELAY
$CUMIN_OPTIONS < /dev/null &> /dev/null \&
echo_success && touch /var/lock/subsys/$servicename && echo
}
stop() {
echo -n $"Stopping cumin: "
- killproc $servicename -TERM
+ killproc -d $(($DELAY-1)) $servicename
RETVAL=$?
echo
if [ $RETVAL -eq 0 ]; then
@@ -120,7 +121,6 @@
;;
restart)
stop
- sleep 3
start
;;
condrestart)
Modified: trunk/cumin/python/cumin/config.py
===================================================================
--- trunk/cumin/python/cumin/config.py 2012-05-14 13:32:02 UTC (rev 5374)
+++ trunk/cumin/python/cumin/config.py 2012-05-14 23:15:47 UTC (rev 5375)
@@ -271,7 +271,11 @@
OptionParser.__init__(self)
self.add_option("--init-only", action="store_true",
default=False)
+ self.add_option("--es", default="")
+ self.add_option("--tm", default="5", type=int)
+
+
# Defaults for these come later from the config
# section named in --section, via apply_defaults()
self.add_option("--debug", action="store_true")
Modified: trunk/cumin/python/cumin/util.py
===================================================================
--- trunk/cumin/python/cumin/util.py 2012-05-14 13:32:02 UTC (rev 5374)
+++ trunk/cumin/python/cumin/util.py 2012-05-14 23:15:47 UTC (rev 5375)
@@ -2,6 +2,7 @@
import os
import logging
import string
+import select
from crypt import crypt
from datetime import datetime, timedelta
@@ -156,8 +157,10 @@
return dvalue
class JobStatusInfo(object):
- stat_strings = ["Unexpanded", "Idle", "Running",
"Removed", "Completed", "Held", "Transferring",
"Suspended", "Submission error"]
- stat_colors = ["red", "clear", "green",
"black", "blue", "yellow", "green",
"yellow", "red"]
+ stat_strings = ["Unexpanded", "Idle", "Running",
"Removed", "Completed",
+ "Held", "Transferring", "Suspended",
"Submission error"]
+ stat_colors = ["red", "clear", "green",
"black", "blue",
+ "yellow", "green", "yellow",
"red"]
@classmethod
def get_status_string(cls, stat):
# Allow strings to be passed in for flexibility.
@@ -253,3 +256,30 @@
if indicator:
retval = retval + "..."
return retval
+
+def poll_stdin(timeout=5, value=""):
+ '''
+ Use select() to poll for input on sys.stdin. Return
+ True if there is input and it matches value.
+
+ timeout - seconds, how long to block for input
+ value - compare the input string to this value.
+ If value is "", all strings match.
+ '''
+ r, w, e = select.select([sys.stdin], [], [], timeout)
+ if r:
+ if value != "":
+ a = raw_input()
+ return a == value
+ return True
+ return False
+
+class ShutdownThread(Thread):
+ def __init__(self, routine):
+ super(ShutdownThread, self).__init__()
+
+ self.routine = routine
+ self.daemon = True
+
+ def run(self):
+ self.routine()