Author: tmckay
Date: 2011-11-09 19:36:31 +0000 (Wed, 09 Nov 2011)
New Revision: 5126
Modified:
trunk/cumin/bin/cumin
trunk/cumin/bin/cumin-admin
trunk/cumin/bin/cumin-data
trunk/cumin/bin/cumin-database
trunk/cumin/bin/cumin-web
trunk/cumin/python/cumin/admin.py
trunk/cumin/python/cumin/database.py
trunk/cumin/python/cumin/main.py
trunk/mint/python/mint/database.py
trunk/mint/python/mint/main.py
Log:
Incorporate environment checks from sysvinit into the cumin executables.
Cumin-web, cumin-data and cumin-admin will now check to make sure the
database is installed and running and that the schema version is correct.
Improved logging and error reporting in cumin executables.
Change cumin-database to refer users to 'cumin-database install' in most
case where the environment is not correct. Install will appropriately
fix most issues.
Modified: trunk/cumin/bin/cumin
===================================================================
--- trunk/cumin/bin/cumin 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/cumin/bin/cumin 2011-11-09 19:36:31 UTC (rev 5126)
@@ -146,8 +146,7 @@
for app in apps:
poll = app[PROCESS] and app[PROCESS].poll()
if poll is not None:
- log.warn("Subprocess exited (pid %s), status %s" %\
- (app[PROCESS].pid, str(poll)))
+
# If the low bit is set on the return code, the
# process got an error during init checks.
# Exit and shut down any processes that have already
@@ -155,17 +154,23 @@
# Note, signals that cause termination will result in
# a negative error code, we treat those as "normal"
if poll > 0 and poll & 1:
- log.error("Subprocess failed init checks (pid %s),"\
- " stopping cumin" % app[PROCESS].pid)
+ log.error("Subprocess failed init checks (pid %s), "\
+ "status %s, check subprocess logs for
details" %\
+ (app[PROCESS].pid, str(poll)))
+ log.info("Stopping cumin")
app[PROCESS] = None
return_code = 2
complete = len(apps)
break
- elif options.init_only:
- app[PROCESS] = None
- complete += 1
else:
- start(app, "Restart")
+ log.warn("Subprocess exited (pid %s), status %s, "\
+ "check subprocess logs for details" %\
+ (app[PROCESS].pid, str(poll)))
+ if options.init_only:
+ app[PROCESS] = None
+ complete += 1
+ else:
+ start(app, "Restart")
finally:
# Try a ctrl-C first
log.info("Send SIGINT to all children")
Modified: trunk/cumin/bin/cumin-admin
===================================================================
--- trunk/cumin/bin/cumin-admin 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/cumin/bin/cumin-admin 2011-11-09 19:36:31 UTC (rev 5126)
@@ -6,7 +6,7 @@
import subprocess
from subprocess import PIPE
-from psycopg2 import IntegrityError
+from psycopg2 import IntegrityError, OperationalError
home = os.environ.get("CUMIN_HOME",
os.path.normpath("/usr/share/cumin"))
sys.path.append(os.path.join(home, "python"))
@@ -16,6 +16,7 @@
from cumin import *
from cumin.config import *
from cumin.util import *
+from cumin.admin import SchemaMissing
def main():
uid = os.getuid()
@@ -70,14 +71,41 @@
app = Cumin(config.get_home(), broker_uris, opts.database, authmech=authmech)
app.check()
- app.init()
+ # Normally, a schema version check during Cumin init will raise
+ # an exception if the schema version doesn't match. However,
+ # cumin-admin is the mechanism for fixing the schema so we
+ # have to avoid the check here!
+ try:
+ app.init(schema_version_check=False)
+ except OperationalError:
+ print "Can't talk to the database"
+ error("Run 'cumin-database check' as root for more
information")
+
conn = app.database.get_connection()
cursor = conn.cursor()
+
+ # Okay, do the schema version check here and disallow
+ # all but schema commands if the version is wrong
+ if name not in ["check_schema",
+ "create_schema",
+ "drop_schema",
+ "print_schema",
+ "upgrade_schema"]:
+ err = True
+ try:
+ app.admin.check_schema(cursor)
+ err = False
+ except SchemaMissing:
+ print("The schema is missing, run 'cumin-admin
create-schema'")
+ except SchemaVersion, e:
+ print str(e)
+ print("Run 'cumin-admin upgrade-schema'")
+ if err:
+ error("Only schema commands are allowed until the schema is
repaired")
try:
handler(app, cursor, opts, args[1:])
-
conn.commit()
finally:
cursor.close()
@@ -191,7 +219,11 @@
print app.admin.get_schema(),
def handle_upgrade_schema(app, cursor, opts, args):
- curr = app.admin.get_schema_version(cursor)
+ try:
+ curr = app.admin.get_schema_version(cursor)
+ except SchemaMissing:
+ error("The schema is missing, run 'cumin-admin
create-schema'")
+
target = app.admin.get_target_schema_version()
if curr == target:
print "The schema is already version %s, nothing to do" % target
@@ -250,16 +282,20 @@
def handle_check_schema(app, cursor, opts, args):
try:
schema_version = app.admin.check_schema(cursor)
+ except SchemaMissing:
+ error("The schema is missing, run 'cumin-admin
create-schema'")
except Exception, e:
error(str(e))
print "The schema is OK (schema version %s)" % schema_version
def handle_drop_schema(app, cursor, opts, args):
- app.admin.drop_schema(cursor)
+ try:
+ app.admin.drop_schema(cursor)
+ print "The schema is dropped"
+ except SchemaMissing:
+ print "The schema has already been dropped"
- print "The schema is dropped"
-
def get_users(app, cursor):
user_cls = app.model.com_redhat_cumin.User
role_cls = app.model.com_redhat_cumin.Role
Modified: trunk/cumin/bin/cumin-data
===================================================================
--- trunk/cumin/bin/cumin-data 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/cumin/bin/cumin-data 2011-11-09 19:36:31 UTC (rev 5126)
@@ -10,6 +10,8 @@
from cumin.util import *
from mint import *
from parsley.loggingex import PipeLogThread
+from psycopg2 import OperationalError
+from cumin.admin import SchemaVersion, SchemaMissing
def restore_IO():
sys.stderr = sys.__stderr__
@@ -228,7 +230,7 @@
sleep(86400)
except KeyboardInterrupt:
- pass
+ log.info("Received SIGINT")
except SystemExit:
if "--help" not in sys.argv:
@@ -237,6 +239,19 @@
except EarlyReturn:
return_code = 1
+ except OperationalError:
+ # Failed to talk to the database on check()
+ log.info("Run 'cumin-database check' as root for more
information.")
+ return_code = 3
+
+ except SchemaMissing:
+ log.info("Run 'cumin-admin create-schema' as root")
+ return_code = 4
+
+ except SchemaVersion:
+ log.info("Run 'cumin-admin upgrade-schema' as root")
+ return_code = 5
+
except:
print_exc()
return_code = 2
Modified: trunk/cumin/bin/cumin-database
===================================================================
--- trunk/cumin/bin/cumin-database 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/cumin/bin/cumin-database 2011-11-09 19:36:31 UTC (rev 5126)
@@ -73,7 +73,7 @@
if [[ ! -f "$pghbaconf" ]]; then
echo "Error: The database is not initialized"
if [ "$1" != noadvice ]; then
- echo "Hint: Run 'cumin-database initialize'"
+ echo "Hint: Run 'cumin-database install'"
fi
return 1
fi
@@ -83,7 +83,7 @@
grep "$dbname" "$pghbaconf" &> /dev/null || {
echo "Error: The database is not configured"
if [ "$1" != noadvice ]; then
- echo "Hint: Run 'cumin-database configure'"
+ echo "Hint: Run 'cumin-database install'"
fi
return 1
}
@@ -93,7 +93,7 @@
psql -d cumin -U cumin -h localhost -c '\q' &> /dev/null || {
echo "Error: The database is not created"
if [ "$1" != noadvice ]; then
- echo "Hint: Run 'cumin-database create'"
+ echo "Hint: Run 'cumin-database install'"
fi
return 1
}
@@ -117,7 +117,7 @@
*)
echo "Error: The database is not created"
if [ "$1" != noadvice ]; then
- echo "Hint: Run 'cumin-database create'"
+ echo "Hint: Run 'cumin-database install'"
fi
return 1
;;
@@ -132,7 +132,7 @@
WARNING
This script installs a cumin database into the system postgresql
-instance.
+instance. It will take the following actions as necessary.
* It will stop and start the postgresql service.
Modified: trunk/cumin/bin/cumin-web
===================================================================
--- trunk/cumin/bin/cumin-web 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/cumin/bin/cumin-web 2011-11-09 19:36:31 UTC (rev 5126)
@@ -9,6 +9,8 @@
from cumin.config import *
from cumin.util import *
from parsley.loggingex import PipeLogThread
+from psycopg2 import OperationalError
+from cumin.admin import SchemaVersion, SchemaMissing
def restore_IO():
sys.stderr = sys.__stderr__
@@ -160,6 +162,7 @@
break
except KeyboardInterrupt:
+ log.info("Received SIGINT")
pass
except SystemExit:
@@ -169,6 +172,19 @@
except EarlyReturn:
return_code = 1
+ except OperationalError:
+ # Failed to talk to the database on check()
+ log.info("Run 'cumin-database check' as root for more
information.")
+ return_code = 3
+
+ except SchemaMissing:
+ log.info("Run 'cumin-admin create-schema' as root")
+ return_code = 4
+
+ except SchemaVersion:
+ log.info("Run 'cumin-admin upgrade-schema' as root")
+ return_code = 5
+
except:
print_exc()
return_code = 2
Modified: trunk/cumin/python/cumin/admin.py
===================================================================
--- trunk/cumin/python/cumin/admin.py 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/cumin/python/cumin/admin.py 2011-11-09 19:36:31 UTC (rev 5126)
@@ -1,12 +1,18 @@
from StringIO import StringIO
from util import *
-from psycopg2 import IntegrityError
+from psycopg2 import IntegrityError, ProgrammingError
log = logging.getLogger("cumin.admin")
schema_version = "1.1"
+class SchemaMissing(Exception):
+ pass
+
+class SchemaVersion(Exception):
+ pass
+
class CuminAdmin(object):
def __init__(self, app):
self.app = app
@@ -34,9 +40,13 @@
def get_schema_version(self, cursor):
cls = self.app.model.com_redhat_cumin.Info
- info = cls.get_object(cursor)
+ try:
+ info = cls.get_object(cursor)
+ except ProgrammingError:
+ info = None
+
if not info:
- raise Exception("The schema isn't there")
+ raise SchemaMissing("The schema isn't there")
return info.schema_version
def get_target_schema_version(self):
@@ -44,19 +54,28 @@
def check_schema(self, cursor):
cls = self.app.model.com_redhat_cumin.Info
- info = cls.get_object(cursor)
+ try:
+ info = cls.get_object(cursor)
+ except ProgrammingError:
+ info = None
if not info:
- raise Exception("The schema isn't there")
+ raise SchemaMissing("The schema isn't there")
if info.schema_version != schema_version:
args = (schema_version, info.schema_version)
msg = "Expected schema version %s; found version %s" % args
- raise Exception(msg)
+ raise SchemaVersion(msg)
return info.schema_version
def drop_schema(self, cursor):
+ cls = self.app.model.com_redhat_cumin.Info
+ try:
+ info = cls.get_object(cursor)
+ except ProgrammingError:
+ raise SchemaMissing("The schema isn't there")
+
writer = StringIO()
self.app.model.sql_model.write_drop_ddl(writer)
sql = writer.getvalue()
Modified: trunk/cumin/python/cumin/database.py
===================================================================
--- trunk/cumin/python/cumin/database.py 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/cumin/python/cumin/database.py 2011-11-09 19:36:31 UTC (rev 5126)
@@ -16,11 +16,37 @@
self.connection_args = dict()
self.thread_local = threading.local()
- def init(self):
+ def init(self, schema_version_check):
log.info("Initializing %s", self)
#m = re.match(r"^([^:]+)://([^@]+)@([^/]+)/(.+)$", self.uri)
+ conn = None
+ try:
+ conn = self.get_connection()
+ cursor = conn.cursor()
+ self.check_connection(cursor)
+ if schema_version_check:
+ self.app.admin.check_schema(cursor)
+ conn.close()
+ except psycopg2.OperationalError:
+ conn and conn.close()
+ log.error("Connection to database failed, is the database
running?")
+ raise
+ except Exception, e:
+ conn and conn.close()
+ log.error(str(e))
+ raise
+ def check(self):
+ # checks moved to init because the model has to be
+ # initialized before we can check the schema. And
+ # there is no point connecting to the database twice...
+ log.info("Checking %s", self)
+
+ def check_connection(self, cursor):
+ cursor.execute("select now()")
+ log.debug("Database is talking at '%s'", self.dsn)
+
def get_connection(self):
return psycopg2.connect(self.dsn)
Modified: trunk/cumin/python/cumin/main.py
===================================================================
--- trunk/cumin/python/cumin/main.py 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/cumin/python/cumin/main.py 2011-11-09 19:36:31 UTC (rev 5126)
@@ -111,6 +111,7 @@
raise Exception(msg % self.home)
self.model.check()
+ self.database.check()
def _init_grid_persona(self):
self.main_page = GridMainPage(self, "index.html")
@@ -163,7 +164,7 @@
inventory.Module(self, "inventory")
usergrid.Module(self, "usergrid")
- def init(self):
+ def init(self, schema_version_check=True):
log.info("Initializing %s", self)
# Create RPC interfaces for QMF and aviary.
@@ -216,7 +217,7 @@
self.model.init()
self.session.init()
- self.database.init()
+ self.database.init(schema_version_check)
self.server.init()
# Create the correct persona
Modified: trunk/mint/python/mint/database.py
===================================================================
--- trunk/mint/python/mint/database.py 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/mint/python/mint/database.py 2011-11-09 19:36:31 UTC (rev 5126)
@@ -14,22 +14,31 @@
def check(self):
log.info("Checking %s", self)
+ # checks moved to init because the model has to be
+ # initialized before we can check the schema. And
+ # there is no point connecting to the database twice...
- self.check_connection()
-
def init(self):
log.info("Initializing %s", self)
-
- def check_connection(self):
- conn = self.get_connection()
-
+ conn = None
try:
+ conn = self.get_connection()
cursor = conn.cursor()
- cursor.execute("select now()")
-
- log.debug("Database is talking at '%s'", self.dsn)
- finally:
+ self.check_connection(cursor)
+ self.app.admin.check_schema(cursor)
conn.close()
+ except psycopg2.OperationalError:
+ conn and conn.close()
+ log.error("Connection to database failed, is the database
running?")
+ raise
+ except Exception, e:
+ conn and conn.close()
+ log.error(str(e))
+ raise
+ def check_connection(self, cursor):
+ cursor.execute("select now()")
+ log.debug("Database is talking at '%s'", self.dsn)
+
def __repr__(self):
return self.__class__.__name__
Modified: trunk/mint/python/mint/main.py
===================================================================
--- trunk/mint/python/mint/main.py 2011-11-09 19:32:49 UTC (rev 5125)
+++ trunk/mint/python/mint/main.py 2011-11-09 19:36:31 UTC (rev 5126)
@@ -4,7 +4,7 @@
from session import MintSession
from update import UpdateThread
from vacuum import VacuumThread
-
+from cumin.admin import *
from util import *
log = logging.getLogger("mint.main")
@@ -16,7 +16,7 @@
self.session = MintSession(self, broker_uris)
self.database = MintDatabase(self, database_dsn)
-
+ self.admin = CuminAdmin(self)
self.update_thread = UpdateThread(self)
self.expire_enabled = True