Author: tmckay
Date: 2012-04-09 19:48:49 +0000 (Mon, 09 Apr 2012)
New Revision: 5287
Modified:
branches/play_plumage/cumin/model/plumage/plumage.xml
branches/play_plumage/cumin/python/cumin/model.py
branches/play_plumage/cumin/python/cumin/stat.py
branches/play_plumage/mint/python/mint/plumage/session.py
branches/play_plumage/mint/python/mint/plumage/update.py
branches/play_plumage/rosemary/python/rosemary/model.py
branches/play_plumage/rosemary/python/rosemary/sqltype.py
Log:
cumin-report which generates false data, complete with objects/samples in the db
Add obj.get_signature() and cls.get_object_by_signature()
Modified: branches/play_plumage/cumin/model/plumage/plumage.xml
===================================================================
--- branches/play_plumage/cumin/model/plumage/plumage.xml 2012-04-09 16:56:04 UTC (rev
5286)
+++ branches/play_plumage/cumin/model/plumage/plumage.xml 2012-04-09 19:48:49 UTC (rev
5287)
@@ -1,11 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
-<schema package="com.redhat.grid.plumage">
- <class name="OSUtil" qmf_headers="False">
- <property name="uuid" index="y" type="uuid"
access="RC" desc="UUID of System Image"/>
+<schema package="com.redhat.grid.plumage"
package_type="plumage">
+ <class name="OSUtil">
+ <property name="uuid" unique="y"
+ type="uuid" access="RC" desc="UUID of System
Image"/>
- <property name="osName" type="sstr" access="RO"
desc="Operating System Name" />
+ <property name="osName"
+ type="sstr" access="RO" desc="Operating System Name"
/>
- <statistic name="avail" type="uint32" unit="uint32"
/>
- <statistic name="used" type="uint32" unit="uint32"
/>
+ <statistic name="avail" type="uint32" unit="uint32"
/>
+ <statistic name="used" type="uint32" unit="uint32"
/>
</class>
</schema>
Modified: branches/play_plumage/cumin/python/cumin/model.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/model.py 2012-04-09 16:56:04 UTC (rev 5286)
+++ branches/play_plumage/cumin/python/cumin/model.py 2012-04-09 19:48:49 UTC (rev 5287)
@@ -320,20 +320,13 @@
class SamplesSqlAdapter(SqlAdapter):
qmf_update_col = '_qmf_update_time'
- def __init__(self, app, table, agent_id, object_id):
- super(SamplesSqlAdapter, self).__init__(app, table)
+ def __init__(self, app, cls, sig):
+ super(SamplesSqlAdapter, self).__init__(app, cls.sql_samples_table)
- assert ";" not in agent_id
- assert ";" not in object_id
+ filters = cls.get_sample_filters_by_signature(sig)
+ for f in filters:
+ self.query.add_filter(f)
- col = self.table._qmf_agent_id
- filter = SqlComparisonFilter(col, "'%s'" % agent_id)
- self.query.add_filter(filter)
-
- col = self.table._qmf_object_id
- filter = SqlComparisonFilter(col, "'%s'" % object_id)
- self.query.add_filter(filter)
-
def get_sql_options(self, options):
return options
Modified: branches/play_plumage/cumin/python/cumin/stat.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/stat.py 2012-04-09 16:56:04 UTC (rev 5286)
+++ branches/play_plumage/cumin/python/cumin/stat.py 2012-04-09 19:48:49 UTC (rev 5287)
@@ -154,10 +154,14 @@
object = self.object.get(session)
params = list()
- # open-flash-chart can't handle the : in the agent id. Net even if
- # it's converted to %3A, so we have to change it
- params.append("agent=%s" % object._qmf_agent_id.replace(":",
"|"))
- params.append("object_id=%s" % object._qmf_object_id)
+ # Ask the object for a list of values that uniquely identifies it.
+ # Signature is the name of a ListParameter that will hold the values
+ # as an ordered list. The values can be passed back to the class to
+ # retrieve the object (or its samples)
+ signature = object.get_signature()
+ for s in signature:
+ params.append("signature=%s" % s)
+
params.append("chart_id=%s" % self.render_id(session, None))
params.append("duration=%s" % self.duration.get(session))
params.append("rpkg=%s" % object._class._package._name)
@@ -340,12 +344,10 @@
self.rosemary_package = Parameter(app, "rpkg")
self.add_parameter(self.rosemary_package)
- self.agent_id = ModifiedAgentIdParameter(app, "agent")
- self.add_parameter(self.agent_id)
+ param = StringParameter(app, "sigparam")
+ self.object_signature = ListParameter(app, "signature", param)
+ self.add_parameter(self.object_signature)
- self.object_id = StringParameter(app, "object_id")
- self.add_parameter(self.object_id)
-
param = Parameter(app, "param")
self.stats = ListParameter(app, "stat", param)
self.add_parameter(self.stats)
@@ -389,15 +391,15 @@
return self.samples.get(session) and "text/plain" or
"image/png"
def get_object_property(self, session, property):
- agent_id = self.agent_id.get(session)
-
rpackage = self.rosemary_package.get(session)
rclass = self.rosemary_class.get(session)
+ signature = self.object_signature.get(session)
+
rosemary_package = self.app.model._packages_by_name[rpackage]
cls = rosemary_package._classes_by_name[rclass]
-
cursor = self.app.database.get_read_cursor()
- object = cls.get_object(cursor, _qmf_agent_id=agent_id)
+ object = cls.get_object_by_signature(cursor, signature)
+
return object.get_value(property)
def get_adapter_stats(self, session):
@@ -406,15 +408,9 @@
rosemary_package = self.app.model._packages_by_name[rpackage]
rosemary_class = rosemary_package._classes_by_name[rclass]
- table = rosemary_class.sql_samples_table
- agent_id = self.agent_id.get(session)
- object_id = self.object_id.get(session)
+ signature = self.object_signature.get(session)
+ adapter = SamplesSqlAdapter(self.app, rosemary_class, signature)
- assert agent_id
- assert object_id
-
- adapter = SamplesSqlAdapter(self.app, table, agent_id, object_id)
-
stats = [getattr(rosemary_class, x) for x in self.stats.get(session)]
return (adapter, stats)
@@ -480,12 +476,10 @@
self.updater.update_enabled = True
self.add_child(self.updater)
- self.object_id = StringParameter(app, "object_id")
- self.add_parameter(self.object_id)
+ param = StringParameter(app, "sigparam")
+ self.object_signature = ListParameter(app, "signature", param)
+ self.add_parameter(self.object_signature)
- self.agent_id = ModifiedAgentIdParameter(app, "agent")
- self.add_parameter(self.agent_id)
-
self.rosemary_class = Parameter(app, "rcls")
self.add_parameter(self.rosemary_class)
@@ -531,14 +525,13 @@
if not obj:
rpackage = self.widget.rosemary_package.get(session)
rclass = self.widget.rosemary_class.get(session)
+ signature = self.widget.object_signature.get(session)
rosemary_package = self.app.model._packages_by_name[rpackage]
rosemary_class = rosemary_package._classes_by_name[rclass]
- agent_id = self.widget.agent_id.get(session)
cursor = self.app.database.get_read_cursor()
+ obj = rosemary_class.get_object_by_signature(cursor, signature)
- obj = rosemary_class.get_object(cursor, _qmf_agent_id=agent_id)
-
self.set(session, obj)
return obj
Modified: branches/play_plumage/mint/python/mint/plumage/session.py
===================================================================
--- branches/play_plumage/mint/python/mint/plumage/session.py 2012-04-09 16:56:04 UTC (rev
5286)
+++ branches/play_plumage/mint/python/mint/plumage/session.py 2012-04-09 19:48:49 UTC (rev
5287)
@@ -2,6 +2,8 @@
from mint.util import MintDaemonThread
from time import sleep
+from update import ObjectUpdate
+from random import randrange
log = logging.getLogger("mint.plumage.session")
@@ -9,7 +11,8 @@
def __init__(self, app, server_url):
self.app = app
self.server_url = server_url
-
+ self.thread = PlumageSessionThread(app, server_url)
+
def check(self):
log.info("Checking %s", self)
@@ -18,25 +21,57 @@
def start(self):
log.info("Starting %s", self)
+ self.thread.start()
def stop(self):
log.info("Stopping %s", self)
+ self.thread.stop()
def __repr__(self):
return "%s(%s)" % (self.__class__.__name__, self.server_url)
class PlumageSessionThread(MintDaemonThread):
- def __init__(self, app):
- super(PumageSessionThread, self).__init__(app)
+ def __init__(self, app, server_url):
+ super(PlumageSessionThread, self).__init__(app)
+ self.server_url = server_url
def run(self):
+ uuid = 1
while True:
if self.stop_requested:
break
- # We create objects here
- sleep(5)
+ # We create objects here. Tag them with the right class,
+ # probably specified to us from a config option (with a corresponding
+ # query specification in the xml)
+ cls = self.app.model.com_redhat_grid_plumage.OSUtil
+ class OSUtil(object):
+ pass
+
+ # We need to get some data, and we need to turn it into some kind
+ # of update object.
+ os = OSUtil()
+
+ os.uuid = str(uuid)
+ os.osName = "Linux"
+
+ # random, 16 slots
+ os.avail = randrange(0, 17)
+
+ # some number of available used
+ os.used = randrange(0, os.avail+1)
+
+ obj = ObjectUpdate(self.app.model, os, cls)
+ self.app.update_thread.enqueue(obj)
+
+ sleep(1)
+
+ # five machines
+ uuid = (uuid + 1) % 6
+ if uuid == 0:
+ uuid = 1
+
'''
class MintConsole(Console):
def __init__(self, model):
Modified: branches/play_plumage/mint/python/mint/plumage/update.py
===================================================================
--- branches/play_plumage/mint/python/mint/plumage/update.py 2012-04-09 16:56:04 UTC (rev
5286)
+++ branches/play_plumage/mint/python/mint/plumage/update.py 2012-04-09 19:48:49 UTC (rev
5287)
@@ -1,6 +1,7 @@
import copy
import resource
import pickle
+from rosemary.sqltype import qmf_type_code_by_string
from psycopg2 import IntegrityError, TimestampFromTicks
from psycopg2.extensions import cursor as Cursor
@@ -14,6 +15,12 @@
sample_window_min = 60
sample_window_max = 60 * 5
+class UpdateDropped(Exception):
+ pass
+
+class MappingException(Exception):
+ pass
+
class UpdateThread(MintDaemonThread):
def __init__(self, app):
super(UpdateThread, self).__init__(app)
@@ -94,13 +101,14 @@
self.stats.sql_ops += 1
class Update(object):
- def __init__(self, model):
+ def __init__(self, model, cls):
self.model = model
+ self.cls = cls
def process(self, thread):
log.debug("Processing %s", self)
try:
- self.do_process(thread.cursor, thread.stats, thread.app.packages)
+ self.do_process(thread.cursor, thread.stats)
thread.conn.commit()
except UpdateDropped:
log.debug("Update dropped")
@@ -118,55 +126,34 @@
if thread.halt_on_error:
raise
- def do_process(self, cursor, stats, packages):
+ def do_process(self, cursor, stats):
raise Exception("Not implemented")
def __repr__(self):
return self.__class__.__name__
class ObjectUpdate(Update):
- def __init__(self, model, plumage_object):
- super(ObjectUpdate, self).__init__(model)
+ def __init__(self, model, plumage_object, cls):
+ super(ObjectUpdate, self).__init__(model, cls)
self.plumage_object = plumage_object
- def do_process(self, cursor, stats, bound_classes, bound_packages):
- cls = self.get_class()
- object_id = self.get_object_id()
-
+ def do_process(self, cursor, stats):
# Okay, we want to do a lookup by object id here.
- # If we can't find it, we create it. Agent is
- # a mint Agent class.
- # We need a lookup by field on machine.
-
+ # Let the class find the unique identifiers in the
+ # object based on the xml table description and do
+ # the lookup for us. Names in the record must match
+ # the names in the xml, or we need a mapping construct
+ # in the rosemary.xml to override.
+ sig = self.cls.get_signature(self.plumage_object)
try:
- obj = agent.get_object( cursor, cls, object_id)
- except RosemaryNotFound:
- obj = self.create_object(cursor, stats, cls)
+ obj = self.cls.get_object_by_signature(cursor, sig)
+ except:
+ self.create_object(cursor, stats, self.cls)
return
self.update_object(cursor, stats, obj)
- def maybe_drop_sample(self, obj):
- properties = self.qmf_object.getProperties()
- statistics = self.qmf_object.getStatistics()
-
- if not properties and statistics:
- # Just stats; do we want it?
- # if stats.enqueued - stats.dequeued > 500:
-
- now = time.time()
- update = self.qmf_object.getTimestamps()[0] / 1000000000
- sample = obj._sample_time
-
- if update < now - sample_window_max:
- # The sample is too old
- raise UpdateDropped()
-
- if sample and sample > now - sample_window_min:
- # The samples are too fidelitous
- raise UpdateDropped()
-
def create_object(self, cursor, stats, cls):
obj = cls.create_object(cursor)
@@ -177,7 +164,7 @@
object_columns.append(table._id)
- self.process_properties(obj, object_columns, cursor)
+ self.process_properties(obj, object_columns)
self.process_statistics(obj, object_columns, sample_columns)
statements = list()
@@ -204,13 +191,12 @@
object_columns = list()
sample_columns = list()
- self.process_properties(obj, object_columns, cursor)
+ self.process_properties(obj, object_columns)
self.process_statistics(obj, object_columns, sample_columns)
statements = list()
cls = obj._class
- # force a write if it's been too long, even if the values match
if object_columns:
sql = cls.sql_update_object.emit(object_columns)
statements.append(sql)
@@ -232,73 +218,56 @@
stats.objects_updated += 1
#stats.objects_updated_by_class[cls] += 1
- def process_properties(self, obj, columns, cursor):
+ def process_properties(self, obj, columns):
cls = obj._class
- # getProperties comes from console.py
- # list of tuples, first is an instance
- # which can be spoofed, second is a value for the property
-
- for prop, value in self.qmf_object.getProperties():
- try:
- col, nvalue = self.process_value(cls, prop, value)
- except MappingException, e:
- log.debug(e)
+ # get a list of property names from the cls
+ # loop through the list of properties, look for
+ # the value in the object
+ for name, prop in cls._properties_by_name.iteritems():
+ value = getattr(self.plumage_object, name)
+ if value is not None:
+ value = transform_value(prop, value)
+ if value == getattr(obj, prop.sql_column.name):
continue
- # XXX This optimization will be obsolete when QMF does it
- # instead
- if nvalue == getattr(obj, col.name):
- continue
+ setattr(obj, prop.sql_column.name, value)
+ columns.append(prop.sql_column)
- setattr(obj, col.name, nvalue)
- columns.append(col)
-
- def process_value(self, cls, prop, value):
- try:
- col = cls._properties_by_name[prop.name].sql_column
- except KeyError:
- raise MappingException("Property %s is unknown" % prop)
-
- if value is not None:
- value = transform_value(prop, value)
-
- return col, value
-
def process_statistics(self, obj, update_columns, insert_columns):
build_columns = list()
saw_change = False
+ cls = obj._class
-
- # getStatistics comes from console.py
- # list of tuples, first is an instance
- # which can be spoofed, second is a value for the property
-
- for stat, value in self.qmf_object.getStatistics():
- try:
- col = obj._class._statistics_by_name[stat.name].sql_column
- except KeyError:
- log.debug("Statistic %s is unknown", stat)
-
- continue
-
+ # get a list of statistic names from the cls
+ # loop through the list of properties, look for
+ # the value in the object
+ for name, stat in cls._statistics_by_name.iteritems():
+ value = getattr(self.plumage_object, name)
if value is not None:
value = transform_value(stat, value)
# Don't write unchanged values
- #
- # XXX This optimization will be obsolete when QMF does it
- # instead
- if value != getattr(obj, col.name):
- setattr(obj, col.name, value)
- update_columns.append(col)
+ if value != getattr(obj, stat.sql_column.name):
+ setattr(obj, stat.sql_column.name, value)
+ update_columns.append(stat.sql_column)
saw_change = True
# If we do end up seeing a value change, we will
# need to insert an entire row so build it up
- build_columns.append(col)
+ build_columns.append(stat.sql_column)
- if saw_change:
+ if saw_change:
+ # Okay, we saw some changed statistics. We also
+ # need to include here all the property columns marked
+ # unique since they are reflected in the sample tables
+ # in addition to the statistics.
+ key_cols = []
+ for p in cls._properties:
+ if p.unique:
+ key_cols.append(p.sql_column)
+
+ insert_columns.extend(key_cols)
insert_columns.extend(build_columns)
def execute_sql(self, cursor, text, args):
@@ -318,14 +287,14 @@
log.error("object properties:")
# see process_properties
- for item in sorted(self.qmf_object.getProperties()):
- log.error(" %-34s %r", *item)
+ #for item in sorted(self.qmf_object.getProperties()):
+ # log.error(" %-34s %r", *item)
- log.error("object statistics:")
+ #log.error("object statistics:")
# see process statistics
- for item in sorted(self.qmf_object.getStatistics()):
- log.error(" %-34s %r", *item)
+ #for item in sorted(self.qmf_object.getStatistics()):
+ # log.error(" %-34s %r", *item)
raise
@@ -360,4 +329,5 @@
transformers[15] = transform_pickle
def transform_value(attr, value):
- return transformers[attr.type](value)
+ t = qmf_type_code_by_string[attr.type]
+ return transformers[t](value)
Modified: branches/play_plumage/rosemary/python/rosemary/model.py
===================================================================
--- branches/play_plumage/rosemary/python/rosemary/model.py 2012-04-09 16:56:04 UTC (rev
5286)
+++ branches/play_plumage/rosemary/python/rosemary/model.py 2012-04-09 19:48:49 UTC (rev
5287)
@@ -4,6 +4,7 @@
from sqltype import *
import util
from cumin.util import xml_escape
+from parsley.collectionsex import defaultdict
log = logging.getLogger("rosemary.model")
@@ -66,7 +67,8 @@
attr = elem.get("package")
if attr:
- pkg = RosemaryPackage(self, attr)
+ pkg_type = elem.get("package_type", None)
+ pkg = RosemaryPackage(self, attr, pkg_type)
pkg.load(elem)
# This on the other hand is a tag itself. It should
@@ -76,7 +78,8 @@
# stuff
# </package>
for child in elem.findall("package"):
- pkg = RosemaryPackage(self, child.get("name"))
+ pkg_type = child.get("package_type", None)
+ pkg = RosemaryPackage(self, child.get("name"), pkg_type)
pkg.load(child)
def extend(self, elem):
@@ -100,16 +103,20 @@
if len(objects) > 0:
youngest = objects[0]
for obj in objects:
- if obj._qmf_update_time > youngest._qmf_update_time:
+ if obj.get_timestamp() > youngest.get_timestamp():
youngest = obj
except:
pass
return youngest
class RosemaryPackage(object):
- def __init__(self, model, name):
+ def __init__(self, model, name, pkg_type):
self._model = model
self._name = name
+ self.pkg_type = pkg_type
+ if self.pkg_type in (None, ""):
+ self.pkg_type = "qmf"
+ self.xml_class = self.get_xml_class()
# Set in the extension xml file. This marks a package as
# containing persistent data which should not be deleted
@@ -147,9 +154,26 @@
groups_by_name[name] = child
for child in elem.findall("class"):
- cls = RosemaryClass(self, child.get("name"),
child.get("qmf_headers"))
+ cls = self.xml_class(self, child.get("name"))
cls.load(child, groups_by_name)
+ def get_xml_class(self):
+ # Split the pkg_type specifier on last .
+ # Last item is the class type to use for this package.
+ # The convention is that type, capitalized and appended with "_class",
+ # names a class corresponding to the xml definition.
+ # Previous items name a module under rosemary.
+ try:
+ mdl, cls = self.pkg_type.rsplit(".", 1)
+ except ValueError:
+ # no dot, just the class is given
+ # default module to 'model'
+ mdl = "model"
+ cls = self.pkg_type
+ m = sys.modules["rosemary." + mdl.lower()]
+ cls = m.__dict__.get(cls.capitalize() + "_class")
+ return cls
+
def extend(self, elem):
persistent = elem.get("persistent")
if persistent is not None:
@@ -172,7 +196,7 @@
return "%s(%s)" % args
class RosemaryClass(object):
- def __init__(self, package, name, qmf_headers):
+ def __init__(self, package, name):
self._package = package
self._name = name
@@ -215,35 +239,20 @@
self._indexes = list()
self._indexes_by_name = dict()
- self._id = RosemaryAttribute(self, "_id")
+ self.add_unique_to_samples = False
- self.qmf_headers = qmf_headers is None or qmf_headers.lower() ==
"true"
- if self.qmf_headers:
- self.add_headers()
+ # Ditionary of tuples, indicates which
+ # columns are unique (either single columns
+ # or groups of columns)
+ self._unique_col_groups = defaultdict(list)
- def add_headers(self):
- name = "_qmf_agent_id"
- self._qmf_agent_id = RosemaryHeader(self, name, sql_text)
- self._qmf_agent_id.title = "Agent ID"
+ # Similar for samples, indicate which indices
+ # should be generated for the sample tables
+ self._sample_index_col_groups = defaultdict(list)
- name = "_qmf_object_id"
- self._qmf_object_id = RosemaryHeader(self, name, sql_text)
- self._qmf_object_id.title = "Object ID"
+ self._id = RosemaryAttribute(self, "_id")
+ self.qmf_headers = False
- name = "_qmf_create_time"
- self._qmf_create_time = RosemaryHeader(self, name, sql_timestamp)
- self._qmf_create_time.title = "Create Time"
-
- name = "_qmf_update_time"
- self._qmf_update_time = RosemaryHeader(self, name, sql_timestamp)
- self._qmf_update_time.title = "Update Time"
-
- name = "_qmf_delete_time"
- self._qmf_delete_time = RosemaryHeader(self, name, sql_timestamp)
- self._qmf_delete_time.title = "Delete Time"
-
- self._qmf_delete_time.optional = True
-
def load(self, elem, groups_by_name):
log.debug("Loading %s", self)
@@ -253,7 +262,6 @@
for child in elem.findall("group"):
name = child.get("name")
-
try:
self.load_class_components(groups_by_name[name])
except KeyError:
@@ -266,12 +274,10 @@
def load_class_components(self, elem):
for child in elem.findall("property"):
name = child.get("name")
-
if child.get("references"):
attr = RosemaryReference(self, name)
else:
attr = RosemaryProperty(self, name)
-
attr.load(child)
for child in elem.findall("statistic"):
@@ -330,6 +336,10 @@
hdr.init()
for prop in self._properties:
+ # Properties will add themselves to
+ # self._unique_col_groups.
+ # The unique constraints will
+ # be generated in add_sql_contraints.
prop.init()
for stat in self._statistics:
@@ -343,6 +353,7 @@
self.add_sql_constraints()
self.add_sql_operations()
+ self.add_sql_sample_indices()
def init_references(self):
for ref in self._references:
@@ -361,18 +372,16 @@
SqlPrimaryKeyConstraint(self.sql_table, name, (id_col,))
def add_sql_constraints(self):
- if self.qmf_headers:
- name = "%s_qmf_id_uq" % self._name
- cols = (self._qmf_agent_id.sql_column, self._qmf_object_id.sql_column)
- SqlUniqueConstraint(self.sql_table, name, cols)
+ for k,v in self._unique_col_groups.iteritems():
+ SqlUniqueConstraint(self.sql_table, k, v)
+ def add_sql_sample_indices(self):
+ for k,v in self._sample_index_col_groups.iteritems():
+ SqlIndex(self._package.sql_schema, k, v)
+
def add_sql_operations(self):
self.sql_get_new_id = SqlGetNewId(self.sql_table, self.sql_sequence)
self.sql_select_by_id = SqlSelectObject(self.sql_table)
-
- if self.qmf_headers:
- self.sql_select_by_qmf_id = SqlSelectObjectByQmfId(self.sql_table)
-
self.sql_insert_object = SqlInsertObject(self.sql_table)
self.sql_update_object = SqlUpdateObject(self.sql_table)
self.sql_delete_object = SqlDeleteObject(self.sql_table)
@@ -382,20 +391,7 @@
table = SqlTable(self._package.sql_schema, name)
self.sql_samples_table = table
-
- if self.qmf_headers:
- agent_col = SqlColumn(table, "_qmf_agent_id", sql_text)
- object_col = SqlColumn(table, "_qmf_object_id", sql_text)
-
- name = "%s_qmf_id_idx" % table._name
- SqlIndex(self._package.sql_schema, name, (agent_col, object_col))
-
- SqlColumn(table, "_qmf_update_time", sql_timestamp)
-
self.sql_samples_insert = SqlInsertObjectSamples(table, self.qmf_headers)
- if self.qmf_headers:
- # We will need another way to delete non-qmf samples
- self.sql_samples_delete = SqlDeleteObjectSamples(table)
def get_object(self, cursor, **criteria):
sql = SqlSelect(self.sql_table)
@@ -413,7 +409,7 @@
if not record:
return
- obj = RosemaryObject(self, None)
+ obj = self._get_rosemary_object(None)
self._set_object_attributes(obj, self.sql_table._columns, record)
@@ -432,7 +428,7 @@
sql.execute(cursor, criteria)
for record in cursor.fetchall():
- obj = RosemaryObject(self, None)
+ obj = self._get_rosemary_object(None)
self._set_object_attributes(obj, self.sql_table._columns, record)
@@ -451,7 +447,7 @@
sql.execute(cursor, criteria)
for record in cursor.fetchall():
- obj = RosemaryObject(self, None)
+ obj = self._get_rosemary_object(None)
self.set_object_attributes(obj, self.sql_table._columns, record)
@@ -462,20 +458,12 @@
def get_object_by_id(self, cursor, id):
assert id
- obj = RosemaryObject(self, id)
+ obj = self._get_rosemary_object(id)
self.load_object_by_id(cursor, obj)
return obj
- def get_object_by_qmf_id(self, cursor, agent_id, object_id):
- assert self.qmf_headers
- obj = RosemaryObject(self, None)
- obj._qmf_agent_id = agent_id
- obj._qmf_object_id = object_id
- self.load_object_by_qmf_id(cursor, obj)
- return obj
-
def get_new_id(self, cursor):
self.sql_get_new_id.execute(cursor)
@@ -485,11 +473,17 @@
return values[0]
+ # Allow this to be overloaded so that
+ # descendents of RosemaryClass can produce
+ # descendents of RosemaryObject in create_object
+ # and other places
+ def _get_rosemary_object(self, id):
+ return RosemaryObject(self, id)
+
def create_object(self, cursor):
id = self.get_new_id(cursor)
+ return self._get_rosemary_object(id)
- return RosemaryObject(self, id)
-
def load_object_by_id(self, cursor, obj):
assert isinstance(obj, RosemaryObject)
assert obj._id, obj
@@ -503,21 +497,6 @@
self._set_object_attributes(obj, self.sql_table._columns, values)
- def load_object_by_qmf_id(self, cursor, obj):
- assert isinstance(obj, RosemaryObject)
- assert self.qmf_headers
- assert obj._qmf_agent_id, obj
- assert obj._qmf_object_id, obj
-
- self.sql_select_by_qmf_id.execute(cursor, obj.__dict__)
-
- values = cursor.fetchone()
-
- if not values:
- raise RosemaryNotFound()
-
- self._set_object_attributes(obj, self.sql_table._columns, values)
-
def _set_object_attributes(self, obj, columns, values):
assert isinstance(obj, RosemaryObject)
@@ -572,6 +551,182 @@
args = (self.__class__.__name__, self._package._name, self._name)
return "%s(%s,%s)" % args
+class Qmf_class(RosemaryClass):
+ def __init__(self, package, name):
+ super(Qmf_class, self).__init__(package, name)
+
+ self.qmf_headers = True
+ name = "_qmf_agent_id"
+ self._qmf_agent_id = RosemaryHeader(self, name, sql_text)
+ self._qmf_agent_id.title = "Agent ID"
+
+ name = "_qmf_object_id"
+ self._qmf_object_id = RosemaryHeader(self, name, sql_text)
+ self._qmf_object_id.title = "Object ID"
+
+ name = "_qmf_create_time"
+ self._qmf_create_time = RosemaryHeader(self, name, sql_timestamp)
+ self._qmf_create_time.title = "Create Time"
+
+ name = "_qmf_update_time"
+ self._qmf_update_time = RosemaryHeader(self, name, sql_timestamp)
+ self._qmf_update_time.title = "Update Time"
+
+ name = "_qmf_delete_time"
+ self._qmf_delete_time = RosemaryHeader(self, name, sql_timestamp)
+ self._qmf_delete_time.title = "Delete Time"
+ self._qmf_delete_time.optional = True
+
+ def add_sql_operations(self):
+ super(Qmf_class, self).add_sql_operations()
+ self.sql_select_by_qmf_id = SqlSelectObjectByQmfId(self.sql_table)
+
+ def add_sql_constraints(self):
+ # Add agent and object columns as a unique column set
+ name = "%s_qmf_id_uq" % self._name
+ self._unique_col_groups[name] = (self._qmf_agent_id.sql_column,
+ self._qmf_object_id.sql_column)
+ super(Qmf_class, self).add_sql_constraints()
+
+ def add_samples_sql(self):
+ super(Qmf_class, self).add_samples_sql()
+ agent_col = SqlColumn(self.sql_samples_table,
+ "_qmf_agent_id", sql_text)
+
+ object_col = SqlColumn(self.sql_samples_table,
+ "_qmf_object_id", sql_text)
+
+ name = "%s_qmf_id_idx" % self.sql_samples_table._name
+ SqlIndex(self._package.sql_schema, name, (agent_col, object_col))
+
+ SqlColumn(self.sql_samples_table, "_qmf_update_time", sql_timestamp)
+
+ # We will need another way to delete non-qmf samples
+ self.sql_samples_delete = SqlDeleteObjectSamples(self.sql_samples_table)
+
+ def get_object_by_qmf_id(self, cursor, agent_id, object_id):
+ obj = self._get_rosemary_object(None)
+ obj._qmf_agent_id = agent_id
+ obj._qmf_object_id = object_id
+ self.load_object_by_qmf_id(cursor, obj)
+ return obj
+
+ def load_object_by_qmf_id(self, cursor, obj):
+ assert isinstance(obj, RosemaryObject)
+ assert self.qmf_headers
+ assert obj._qmf_agent_id, obj
+ assert obj._qmf_object_id, obj
+
+ self.sql_select_by_qmf_id.execute(cursor, obj.__dict__)
+
+ values = cursor.fetchone()
+
+ if not values:
+ raise RosemaryNotFound()
+
+ self._set_object_attributes(obj, self.sql_table._columns, values)
+
+ def get_signature(self, obj):
+ # Return an ordered set of values that can uniquely
+ # identify the object
+ return [obj._qmf_agent_id.replace(":", "|"),
+ obj._qmf_object_id]
+
+ def get_object_by_signature(self, cursor, signature):
+ # Use the signature to find the object
+ agent_id, obj_id = signature
+ # open-flash-chart can't handle the : in the agent id. Net even if
+ # it's converted to %3A, so we have to change it
+ agent_id = agent_id.replace("|", ":")
+ return self.get_object_by_qmf_id(cursor, agent_id, obj_id)
+
+ def get_sample_filters_by_signature(self, signature):
+ agent_id, obj_id = signature
+ agent_id = agent_id.replace("|", ":")
+ filters = []
+
+ col = self.sql_samples_table._qmf_agent_id
+ filters.append(SqlComparisonFilter(col, "'%s'" % agent_id))
+
+ col = self.sql_samples_table._qmf_object_id
+ filters.append(SqlComparisonFilter(col, "'%s'" % obj_id))
+
+ return filters
+
+ # Allow this to be overloaded so that
+ # descendents of RosemaryClass can produce
+ # descendents of RosemaryObject in create_object
+ # and other places
+ def _get_rosemary_object(self, id):
+ return RosemaryQmfObject(self, id)
+
+class Plumage_class(RosemaryClass):
+ def __init__(self, package, name):
+ super(Plumage_class, self).__init__(package, name)
+
+ # Columns marked unique will be added to the
+ # sample table
+ self.add_unique_to_samples = True
+
+ def init(self):
+ super(Plumage_class, self).init()
+
+ # Store a sorted list of the names of unique
+ # column groups
+ self.col_group_list = self._unique_col_groups.keys()
+ self.col_group_list.sort()
+
+ # Todo: this could be optimized if we use the shortest
+ # set of unique column groups but probably there is only one
+ self.signature_cols = self._unique_col_groups[self.col_group_list[0]]
+
+ # Set up a query that we can use on a unique
+ # column set
+ self.sql_select_signature = SqlSelect(self.sql_table)
+ for c in self.signature_cols:
+ self.sql_select_signature.add_filter(SqlValueFilter(c))
+
+ def load_class_components(self, elem):
+ for child in elem.findall("property"):
+ name = child.get("name")
+ attr = RosemaryProperty(self, name)
+ attr.load(child)
+
+ for child in elem.findall("statistic"):
+ stat = RosemaryStatistic(self, child.get("name"))
+ stat.load(child)
+
+ def get_signature(self, obj):
+ # Return an ordered set of values that can uniquely
+ # identify the object.
+ # For plumage objects, this is going to be the list
+ # of values in the columns marked unique.
+ sig = []
+ for c in self.signature_cols:
+ # For each col, record the value from the object
+ sig.append(getattr(obj, c.name))
+ return sig
+
+ def get_object_by_signature(self, cursor, signature):
+ # signature is an ordered list of values corresponding
+ # to self.signature_cols
+ obj = self._get_rosemary_object(None)
+ sig_idx = 0
+ for c in self.signature_cols:
+ setattr(obj, c.name, signature[sig_idx])
+ sig_idx += 1
+ self.load_object_by_signature(cursor, obj)
+ return obj
+
+ def load_object_by_signature(self, cursor, obj):
+ assert isinstance(obj, RosemaryObject)
+
+ self.sql_select_signature.execute(cursor, obj.__dict__)
+ values = cursor.fetchone()
+ if not values:
+ raise RosemaryNotFound()
+ self._set_object_attributes(obj, self.sql_table._columns, values)
+
class RosemaryAttribute(object):
def __init__(self, cls, name):
self.cls = cls
@@ -598,6 +753,19 @@
self.sql_column = None
+ def process_unique(self, value):
+ # If unique is None, "y" or "n" then it
+ # is a boolean value and the column is not part
+ # of a group, it is simply unique by itself
+ if value is None:
+ return False
+ elif value in ("n", "y"):
+ return value == "y"
+
+ # Otherwise, the column is part of a multi-field
+ # unique set where value is the name of the set
+ return value
+
def load(self, elem):
self.type = elem.get("type")
self.references = elem.get("references")
@@ -605,12 +773,12 @@
self.unit = elem.get("unit")
self.index = elem.get("index", "n") == "y" and
True
self.optional = elem.get("optional", "n") == "y"
and True
+ self.unique = self.process_unique(elem.get("unique"))
self.description = clean_description(elem.get("desc"))
def extend(self, elem):
self.title = elem.findtext("title")
self.short_title = elem.findtext("short")
- self.unique = elem.get("unique", "n") == "y" and
True
formatter = elem.findtext("formatter")
# Allow overloading of optional in the extensions
@@ -620,6 +788,11 @@
if optional is not None:
self.optional = optional == "y"
+ # Same here
+ unique = elem.get("unique")
+ if unique is not None:
+ self.unique = self.process_unique(unique)
+
if formatter:
meth = getattr(util, formatter, None)
@@ -712,10 +885,32 @@
self.sql_column = SqlColumn(self.cls.sql_table, self.name, type)
self.sql_column.nullable = self.optional
- if self.unique:
- name = "%s_%s_uq" % (self.cls._name, self.name)
- SqlUniqueConstraint(self.cls.sql_table, name, (self.sql_column,))
+ # Save away refernces for generation of unique constraints and indices
+ # If it's a single column marked unique, save it under its own name.
+ if self.unique == True:
+ name = "%s_%s_uq" % (self.cls.sql_table._name, self.name)
+ self.cls._unique_col_groups[name] = (self.sql_column,)
+
+ # If it's part of a group, append it to a group of columns named
+ # for the group.
+ elif self.unique:
+ name = "%s_%s_uq" % (self.cls.sql_table._name, self.unique)
+ self.cls._unique_col_groups[name].append(self.sql_column)
+ # For some classes, add unique columns to the sample table, too.
+ # Produce an index for the column(s) based on the 'unique' value.
+ # These columns take the place of qmf_agent_id and qmf_object_id
+ # for non-qmf records
+ if self.cls.add_unique_to_samples and self.unique:
+ col = SqlColumn(self.cls.sql_samples_table, self.name, type)
+ col.nullable = self.optional
+ if self.unique == True:
+ name = "%s_%s_idx" % (self.cls.sql_samples_table._name,
self.name)
+ self.cls._sample_index_col_groups[name] = (col,)
+ else:
+ name = "%s_%s_idx" % (self.cls.sql_samples_table._name,
self.unique)
+ self.cls._sample_index_col_groups[name].append(col)
+
class RosemaryStatistic(RosemaryAttribute):
def __init__(self, cls, name):
super(RosemaryStatistic, self).__init__(cls, name)
@@ -842,12 +1037,6 @@
def add_sample(self, cursor, columns=None):
self._class.add_object_sample(cursor, self, columns)
- def fake_qmf_values(self):
- self._qmf_agent_id = "__rosemary__"
- self._qmf_object_id = str(datetime.now())
- self._qmf_create_time = datetime.now()
- self._qmf_update_time = datetime.now()
-
def get_title(self):
if self._class._object_title:
return self._class._object_title % self.__dict__
@@ -877,11 +1066,30 @@
return value
+ def get_timestamp(self):
+ raise Exception("Not implemented")
+
def __repr__(self):
name = self.__class__.__name__
args = (name, self._class, self._id, self._save_time)
return "%s(%s,%i,%s)" % args
+class RosemaryQmfObject(RosemaryObject):
+ def fake_qmf_values(self):
+ self._qmf_agent_id = "__rosemary__"
+ self._qmf_object_id = str(datetime.now())
+ self._qmf_create_time = datetime.now()
+ self._qmf_update_time = datetime.now()
+
+ def get_timestamp(self):
+ return self._qmf_update_time
+
+ def get_signature(self):
+ # return an ordered list of values that can uniquely
+ # identify this object. Let the class determine
+ # the list.
+ return self._class.get_signature(self)
+
class RosemaryNotFound(Exception):
pass
Modified: branches/play_plumage/rosemary/python/rosemary/sqltype.py
===================================================================
--- branches/play_plumage/rosemary/python/rosemary/sqltype.py 2012-04-09 16:56:04 UTC (rev
5286)
+++ branches/play_plumage/rosemary/python/rosemary/sqltype.py 2012-04-09 19:48:49 UTC (rev
5287)
@@ -76,3 +76,4 @@
sql_types_by_qmf_type_code = dict([(x[1], x[0]) for x in __mappings])
sql_types_by_qmf_type_string = dict([(x[2], x[0]) for x in __mappings])
+qmf_type_code_by_string = dict([(x[2], x[1]) for x in __mappings])