Author: tmckay Date: 2012-04-09 19:48:49 +0000 (Mon, 09 Apr 2012) New Revision: 5287
Modified: branches/play_plumage/cumin/model/plumage/plumage.xml branches/play_plumage/cumin/python/cumin/model.py branches/play_plumage/cumin/python/cumin/stat.py branches/play_plumage/mint/python/mint/plumage/session.py branches/play_plumage/mint/python/mint/plumage/update.py branches/play_plumage/rosemary/python/rosemary/model.py branches/play_plumage/rosemary/python/rosemary/sqltype.py Log: cumin-report which generates false data, complete with objects/samples in the db Add obj.get_signature() and cls.get_object_by_signature()
Modified: branches/play_plumage/cumin/model/plumage/plumage.xml =================================================================== --- branches/play_plumage/cumin/model/plumage/plumage.xml 2012-04-09 16:56:04 UTC (rev 5286) +++ branches/play_plumage/cumin/model/plumage/plumage.xml 2012-04-09 19:48:49 UTC (rev 5287) @@ -1,11 +1,13 @@ <?xml version="1.0" encoding="utf-8"?> -<schema package="com.redhat.grid.plumage"> - <class name="OSUtil" qmf_headers="False"> - <property name="uuid" index="y" type="uuid" access="RC" desc="UUID of System Image"/> +<schema package="com.redhat.grid.plumage" package_type="plumage"> + <class name="OSUtil"> + <property name="uuid" unique="y" + type="uuid" access="RC" desc="UUID of System Image"/>
- <property name="osName" type="sstr" access="RO" desc="Operating System Name" /> + <property name="osName" + type="sstr" access="RO" desc="Operating System Name" />
- <statistic name="avail" type="uint32" unit="uint32" /> - <statistic name="used" type="uint32" unit="uint32" /> + <statistic name="avail" type="uint32" unit="uint32" /> + <statistic name="used" type="uint32" unit="uint32" /> </class> </schema>
Modified: branches/play_plumage/cumin/python/cumin/model.py =================================================================== --- branches/play_plumage/cumin/python/cumin/model.py 2012-04-09 16:56:04 UTC (rev 5286) +++ branches/play_plumage/cumin/python/cumin/model.py 2012-04-09 19:48:49 UTC (rev 5287) @@ -320,20 +320,13 @@ class SamplesSqlAdapter(SqlAdapter): qmf_update_col = '_qmf_update_time'
- def __init__(self, app, table, agent_id, object_id): - super(SamplesSqlAdapter, self).__init__(app, table) + def __init__(self, app, cls, sig): + super(SamplesSqlAdapter, self).__init__(app, cls.sql_samples_table)
- assert ";" not in agent_id - assert ";" not in object_id + filters = cls.get_sample_filters_by_signature(sig) + for f in filters: + self.query.add_filter(f)
- col = self.table._qmf_agent_id - filter = SqlComparisonFilter(col, "'%s'" % agent_id) - self.query.add_filter(filter) - - col = self.table._qmf_object_id - filter = SqlComparisonFilter(col, "'%s'" % object_id) - self.query.add_filter(filter) - def get_sql_options(self, options): return options
Modified: branches/play_plumage/cumin/python/cumin/stat.py =================================================================== --- branches/play_plumage/cumin/python/cumin/stat.py 2012-04-09 16:56:04 UTC (rev 5286) +++ branches/play_plumage/cumin/python/cumin/stat.py 2012-04-09 19:48:49 UTC (rev 5287) @@ -154,10 +154,14 @@ object = self.object.get(session) params = list()
- # open-flash-chart can't handle the : in the agent id. Net even if - # it's converted to %3A, so we have to change it - params.append("agent=%s" % object._qmf_agent_id.replace(":", "|")) - params.append("object_id=%s" % object._qmf_object_id) + # Ask the object for a list of values that uniquely identifies it. + # Signature is the name of a ListParameter that will hold the values + # as an ordered list. The values can be passed back to the class to + # retrieve the object (or its samples) + signature = object.get_signature() + for s in signature: + params.append("signature=%s" % s) + params.append("chart_id=%s" % self.render_id(session, None)) params.append("duration=%s" % self.duration.get(session)) params.append("rpkg=%s" % object._class._package._name) @@ -340,12 +344,10 @@ self.rosemary_package = Parameter(app, "rpkg") self.add_parameter(self.rosemary_package)
- self.agent_id = ModifiedAgentIdParameter(app, "agent") - self.add_parameter(self.agent_id) + param = StringParameter(app, "sigparam") + self.object_signature = ListParameter(app, "signature", param) + self.add_parameter(self.object_signature)
- self.object_id = StringParameter(app, "object_id") - self.add_parameter(self.object_id) - param = Parameter(app, "param") self.stats = ListParameter(app, "stat", param) self.add_parameter(self.stats) @@ -389,15 +391,15 @@ return self.samples.get(session) and "text/plain" or "image/png"
def get_object_property(self, session, property): - agent_id = self.agent_id.get(session) - rpackage = self.rosemary_package.get(session) rclass = self.rosemary_class.get(session) + signature = self.object_signature.get(session) + rosemary_package = self.app.model._packages_by_name[rpackage] cls = rosemary_package._classes_by_name[rclass] - cursor = self.app.database.get_read_cursor() - object = cls.get_object(cursor, _qmf_agent_id=agent_id) + object = cls.get_object_by_signature(cursor, signature) + return object.get_value(property)
def get_adapter_stats(self, session): @@ -406,15 +408,9 @@ rosemary_package = self.app.model._packages_by_name[rpackage] rosemary_class = rosemary_package._classes_by_name[rclass]
- table = rosemary_class.sql_samples_table - agent_id = self.agent_id.get(session) - object_id = self.object_id.get(session) + signature = self.object_signature.get(session) + adapter = SamplesSqlAdapter(self.app, rosemary_class, signature)
- assert agent_id - assert object_id - - adapter = SamplesSqlAdapter(self.app, table, agent_id, object_id) - stats = [getattr(rosemary_class, x) for x in self.stats.get(session)]
return (adapter, stats) @@ -480,12 +476,10 @@ self.updater.update_enabled = True self.add_child(self.updater)
- self.object_id = StringParameter(app, "object_id") - self.add_parameter(self.object_id) + param = StringParameter(app, "sigparam") + self.object_signature = ListParameter(app, "signature", param) + self.add_parameter(self.object_signature)
- self.agent_id = ModifiedAgentIdParameter(app, "agent") - self.add_parameter(self.agent_id) - self.rosemary_class = Parameter(app, "rcls") self.add_parameter(self.rosemary_class)
@@ -531,14 +525,13 @@ if not obj: rpackage = self.widget.rosemary_package.get(session) rclass = self.widget.rosemary_class.get(session) + signature = self.widget.object_signature.get(session) rosemary_package = self.app.model._packages_by_name[rpackage] rosemary_class = rosemary_package._classes_by_name[rclass] - agent_id = self.widget.agent_id.get(session)
cursor = self.app.database.get_read_cursor() + obj = rosemary_class.get_object_by_signature(cursor, signature)
- obj = rosemary_class.get_object(cursor, _qmf_agent_id=agent_id) - self.set(session, obj)
return obj
Modified: branches/play_plumage/mint/python/mint/plumage/session.py =================================================================== --- branches/play_plumage/mint/python/mint/plumage/session.py 2012-04-09 16:56:04 UTC (rev 5286) +++ branches/play_plumage/mint/python/mint/plumage/session.py 2012-04-09 19:48:49 UTC (rev 5287) @@ -2,6 +2,8 @@
from mint.util import MintDaemonThread from time import sleep +from update import ObjectUpdate +from random import randrange
log = logging.getLogger("mint.plumage.session")
@@ -9,7 +11,8 @@ def __init__(self, app, server_url): self.app = app self.server_url = server_url - + self.thread = PlumageSessionThread(app, server_url) + def check(self): log.info("Checking %s", self)
@@ -18,25 +21,57 @@
def start(self): log.info("Starting %s", self) + self.thread.start()
def stop(self): log.info("Stopping %s", self) + self.thread.stop()
def __repr__(self): return "%s(%s)" % (self.__class__.__name__, self.server_url)
class PlumageSessionThread(MintDaemonThread): - def __init__(self, app): - super(PumageSessionThread, self).__init__(app) + def __init__(self, app, server_url): + super(PlumageSessionThread, self).__init__(app) + self.server_url = server_url
def run(self): + uuid = 1 while True: if self.stop_requested: break
- # We create objects here - sleep(5) + # We create objects here. Tag them with the right class, + # probably specified to us from a config option (with a corresponding + # query specification in the xml) + cls = self.app.model.com_redhat_grid_plumage.OSUtil
+ class OSUtil(object): + pass + + # We need to get some data, and we need to turn it into some kind + # of update object. + os = OSUtil() + + os.uuid = str(uuid) + os.osName = "Linux" + + # random, 16 slots + os.avail = randrange(0, 17) + + # some number of available used + os.used = randrange(0, os.avail+1) + + obj = ObjectUpdate(self.app.model, os, cls) + self.app.update_thread.enqueue(obj) + + sleep(1) + + # five machines + uuid = (uuid + 1) % 6 + if uuid == 0: + uuid = 1 + ''' class MintConsole(Console): def __init__(self, model):
Modified: branches/play_plumage/mint/python/mint/plumage/update.py =================================================================== --- branches/play_plumage/mint/python/mint/plumage/update.py 2012-04-09 16:56:04 UTC (rev 5286) +++ branches/play_plumage/mint/python/mint/plumage/update.py 2012-04-09 19:48:49 UTC (rev 5287) @@ -1,6 +1,7 @@ import copy import resource import pickle +from rosemary.sqltype import qmf_type_code_by_string
from psycopg2 import IntegrityError, TimestampFromTicks from psycopg2.extensions import cursor as Cursor @@ -14,6 +15,12 @@ sample_window_min = 60 sample_window_max = 60 * 5
+class UpdateDropped(Exception): + pass + +class MappingException(Exception): + pass + class UpdateThread(MintDaemonThread): def __init__(self, app): super(UpdateThread, self).__init__(app) @@ -94,13 +101,14 @@ self.stats.sql_ops += 1
class Update(object): - def __init__(self, model): + def __init__(self, model, cls): self.model = model + self.cls = cls
def process(self, thread): log.debug("Processing %s", self) try: - self.do_process(thread.cursor, thread.stats, thread.app.packages) + self.do_process(thread.cursor, thread.stats) thread.conn.commit() except UpdateDropped: log.debug("Update dropped") @@ -118,55 +126,34 @@ if thread.halt_on_error: raise
- def do_process(self, cursor, stats, packages): + def do_process(self, cursor, stats): raise Exception("Not implemented")
def __repr__(self): return self.__class__.__name__
class ObjectUpdate(Update): - def __init__(self, model, plumage_object): - super(ObjectUpdate, self).__init__(model) + def __init__(self, model, plumage_object, cls): + super(ObjectUpdate, self).__init__(model, cls)
self.plumage_object = plumage_object
- def do_process(self, cursor, stats, bound_classes, bound_packages): - cls = self.get_class() - object_id = self.get_object_id() - + def do_process(self, cursor, stats): # Okay, we want to do a lookup by object id here. - # If we can't find it, we create it. Agent is - # a mint Agent class. - # We need a lookup by field on machine. - + # Let the class find the unique identifiers in the + # object based on the xml table description and do + # the lookup for us. Names in the record must match + # the names in the xml, or we need a mapping construct + # in the rosemary.xml to override. + sig = self.cls.get_signature(self.plumage_object) try: - obj = agent.get_object( cursor, cls, object_id) - except RosemaryNotFound: - obj = self.create_object(cursor, stats, cls) + obj = self.cls.get_object_by_signature(cursor, sig) + except: + self.create_object(cursor, stats, self.cls) return
self.update_object(cursor, stats, obj)
- def maybe_drop_sample(self, obj): - properties = self.qmf_object.getProperties() - statistics = self.qmf_object.getStatistics() - - if not properties and statistics: - # Just stats; do we want it? - # if stats.enqueued - stats.dequeued > 500: - - now = time.time() - update = self.qmf_object.getTimestamps()[0] / 1000000000 - sample = obj._sample_time - - if update < now - sample_window_max: - # The sample is too old - raise UpdateDropped() - - if sample and sample > now - sample_window_min: - # The samples are too fidelitous - raise UpdateDropped() - def create_object(self, cursor, stats, cls): obj = cls.create_object(cursor)
@@ -177,7 +164,7 @@
object_columns.append(table._id)
- self.process_properties(obj, object_columns, cursor) + self.process_properties(obj, object_columns) self.process_statistics(obj, object_columns, sample_columns)
statements = list() @@ -204,13 +191,12 @@ object_columns = list() sample_columns = list()
- self.process_properties(obj, object_columns, cursor) + self.process_properties(obj, object_columns) self.process_statistics(obj, object_columns, sample_columns)
statements = list() cls = obj._class
- # force a write if it's been too long, even if the values match if object_columns: sql = cls.sql_update_object.emit(object_columns) statements.append(sql) @@ -232,73 +218,56 @@ stats.objects_updated += 1 #stats.objects_updated_by_class[cls] += 1
- def process_properties(self, obj, columns, cursor): + def process_properties(self, obj, columns): cls = obj._class
- # getProperties comes from console.py - # list of tuples, first is an instance - # which can be spoofed, second is a value for the property - - for prop, value in self.qmf_object.getProperties(): - try: - col, nvalue = self.process_value(cls, prop, value) - except MappingException, e: - log.debug(e) + # get a list of property names from the cls + # loop through the list of properties, look for + # the value in the object + for name, prop in cls._properties_by_name.iteritems(): + value = getattr(self.plumage_object, name) + if value is not None: + value = transform_value(prop, value) + if value == getattr(obj, prop.sql_column.name): continue
- # XXX This optimization will be obsolete when QMF does it - # instead - if nvalue == getattr(obj, col.name): - continue + setattr(obj, prop.sql_column.name, value) + columns.append(prop.sql_column)
- setattr(obj, col.name, nvalue) - columns.append(col) - - def process_value(self, cls, prop, value): - try: - col = cls._properties_by_name[prop.name].sql_column - except KeyError: - raise MappingException("Property %s is unknown" % prop) - - if value is not None: - value = transform_value(prop, value) - - return col, value - def process_statistics(self, obj, update_columns, insert_columns): build_columns = list() saw_change = False + cls = obj._class
- - # getStatistics comes from console.py - # list of tuples, first is an instance - # which can be spoofed, second is a value for the property - - for stat, value in self.qmf_object.getStatistics(): - try: - col = obj._class._statistics_by_name[stat.name].sql_column - except KeyError: - log.debug("Statistic %s is unknown", stat) - - continue - + # get a list of statistic names from the cls + # loop through the list of properties, look for + # the value in the object + for name, stat in cls._statistics_by_name.iteritems(): + value = getattr(self.plumage_object, name) if value is not None: value = transform_value(stat, value)
# Don't write unchanged values - # - # XXX This optimization will be obsolete when QMF does it - # instead - if value != getattr(obj, col.name): - setattr(obj, col.name, value) - update_columns.append(col) + if value != getattr(obj, stat.sql_column.name): + setattr(obj, stat.sql_column.name, value) + update_columns.append(stat.sql_column) saw_change = True
# If we do end up seeing a value change, we will # need to insert an entire row so build it up - build_columns.append(col) + build_columns.append(stat.sql_column)
- if saw_change: + if saw_change: + # Okay, we saw some changed statistics. We also + # need to include here all the property columns marked + # unique since they are reflected in the sample tables + # in addition to the statistics. + key_cols = [] + for p in cls._properties: + if p.unique: + key_cols.append(p.sql_column) + + insert_columns.extend(key_cols) insert_columns.extend(build_columns)
def execute_sql(self, cursor, text, args): @@ -318,14 +287,14 @@ log.error("object properties:")
# see process_properties - for item in sorted(self.qmf_object.getProperties()): - log.error(" %-34s %r", *item) + #for item in sorted(self.qmf_object.getProperties()): + # log.error(" %-34s %r", *item)
- log.error("object statistics:") + #log.error("object statistics:")
# see process statistics - for item in sorted(self.qmf_object.getStatistics()): - log.error(" %-34s %r", *item) + #for item in sorted(self.qmf_object.getStatistics()): + # log.error(" %-34s %r", *item)
raise
@@ -360,4 +329,5 @@ transformers[15] = transform_pickle
def transform_value(attr, value): - return transformers[attr.type](value) + t = qmf_type_code_by_string[attr.type] + return transformers[t](value)
Modified: branches/play_plumage/rosemary/python/rosemary/model.py =================================================================== --- branches/play_plumage/rosemary/python/rosemary/model.py 2012-04-09 16:56:04 UTC (rev 5286) +++ branches/play_plumage/rosemary/python/rosemary/model.py 2012-04-09 19:48:49 UTC (rev 5287) @@ -4,6 +4,7 @@ from sqltype import * import util from cumin.util import xml_escape +from parsley.collectionsex import defaultdict
log = logging.getLogger("rosemary.model")
@@ -66,7 +67,8 @@ attr = elem.get("package")
if attr: - pkg = RosemaryPackage(self, attr) + pkg_type = elem.get("package_type", None) + pkg = RosemaryPackage(self, attr, pkg_type) pkg.load(elem)
# This on the other hand is a tag itself. It should @@ -76,7 +78,8 @@ # stuff # </package> for child in elem.findall("package"): - pkg = RosemaryPackage(self, child.get("name")) + pkg_type = child.get("package_type", None) + pkg = RosemaryPackage(self, child.get("name"), pkg_type) pkg.load(child)
def extend(self, elem): @@ -100,16 +103,20 @@ if len(objects) > 0: youngest = objects[0] for obj in objects: - if obj._qmf_update_time > youngest._qmf_update_time: + if obj.get_timestamp() > youngest.get_timestamp(): youngest = obj except: pass return youngest
class RosemaryPackage(object): - def __init__(self, model, name): + def __init__(self, model, name, pkg_type): self._model = model self._name = name + self.pkg_type = pkg_type + if self.pkg_type in (None, ""): + self.pkg_type = "qmf" + self.xml_class = self.get_xml_class()
# Set in the extension xml file. This marks a package as # containing persistent data which should not be deleted @@ -147,9 +154,26 @@ groups_by_name[name] = child
for child in elem.findall("class"): - cls = RosemaryClass(self, child.get("name"), child.get("qmf_headers")) + cls = self.xml_class(self, child.get("name")) cls.load(child, groups_by_name)
+ def get_xml_class(self): + # Split the pkg_type specifier on last . + # Last item is the class type to use for this package. + # The convention is that type, capitalized and appended with "_class", + # names a class corresponding to the xml definition. + # Previous items name a module under rosemary. + try: + mdl, cls = self.pkg_type.rsplit(".", 1) + except ValueError: + # no dot, just the class is given + # default module to 'model' + mdl = "model" + cls = self.pkg_type + m = sys.modules["rosemary." + mdl.lower()] + cls = m.__dict__.get(cls.capitalize() + "_class") + return cls + def extend(self, elem): persistent = elem.get("persistent") if persistent is not None: @@ -172,7 +196,7 @@ return "%s(%s)" % args
class RosemaryClass(object): - def __init__(self, package, name, qmf_headers): + def __init__(self, package, name): self._package = package self._name = name
@@ -215,35 +239,20 @@ self._indexes = list() self._indexes_by_name = dict()
- self._id = RosemaryAttribute(self, "_id") + self.add_unique_to_samples = False
- self.qmf_headers = qmf_headers is None or qmf_headers.lower() == "true" - if self.qmf_headers: - self.add_headers() + # Ditionary of tuples, indicates which + # columns are unique (either single columns + # or groups of columns) + self._unique_col_groups = defaultdict(list)
- def add_headers(self): - name = "_qmf_agent_id" - self._qmf_agent_id = RosemaryHeader(self, name, sql_text) - self._qmf_agent_id.title = "Agent ID" + # Similar for samples, indicate which indices + # should be generated for the sample tables + self._sample_index_col_groups = defaultdict(list)
- name = "_qmf_object_id" - self._qmf_object_id = RosemaryHeader(self, name, sql_text) - self._qmf_object_id.title = "Object ID" + self._id = RosemaryAttribute(self, "_id") + self.qmf_headers = False
- name = "_qmf_create_time" - self._qmf_create_time = RosemaryHeader(self, name, sql_timestamp) - self._qmf_create_time.title = "Create Time" - - name = "_qmf_update_time" - self._qmf_update_time = RosemaryHeader(self, name, sql_timestamp) - self._qmf_update_time.title = "Update Time" - - name = "_qmf_delete_time" - self._qmf_delete_time = RosemaryHeader(self, name, sql_timestamp) - self._qmf_delete_time.title = "Delete Time" - - self._qmf_delete_time.optional = True - def load(self, elem, groups_by_name): log.debug("Loading %s", self)
@@ -253,7 +262,6 @@
for child in elem.findall("group"): name = child.get("name") - try: self.load_class_components(groups_by_name[name]) except KeyError: @@ -266,12 +274,10 @@ def load_class_components(self, elem): for child in elem.findall("property"): name = child.get("name") - if child.get("references"): attr = RosemaryReference(self, name) else: attr = RosemaryProperty(self, name) - attr.load(child)
for child in elem.findall("statistic"): @@ -330,6 +336,10 @@ hdr.init()
for prop in self._properties: + # Properties will add themselves to + # self._unique_col_groups. + # The unique constraints will + # be generated in add_sql_contraints. prop.init()
for stat in self._statistics: @@ -343,6 +353,7 @@
self.add_sql_constraints() self.add_sql_operations() + self.add_sql_sample_indices()
def init_references(self): for ref in self._references: @@ -361,18 +372,16 @@ SqlPrimaryKeyConstraint(self.sql_table, name, (id_col,))
def add_sql_constraints(self): - if self.qmf_headers: - name = "%s_qmf_id_uq" % self._name - cols = (self._qmf_agent_id.sql_column, self._qmf_object_id.sql_column) - SqlUniqueConstraint(self.sql_table, name, cols) + for k,v in self._unique_col_groups.iteritems(): + SqlUniqueConstraint(self.sql_table, k, v)
+ def add_sql_sample_indices(self): + for k,v in self._sample_index_col_groups.iteritems(): + SqlIndex(self._package.sql_schema, k, v) + def add_sql_operations(self): self.sql_get_new_id = SqlGetNewId(self.sql_table, self.sql_sequence) self.sql_select_by_id = SqlSelectObject(self.sql_table) - - if self.qmf_headers: - self.sql_select_by_qmf_id = SqlSelectObjectByQmfId(self.sql_table) - self.sql_insert_object = SqlInsertObject(self.sql_table) self.sql_update_object = SqlUpdateObject(self.sql_table) self.sql_delete_object = SqlDeleteObject(self.sql_table) @@ -382,20 +391,7 @@ table = SqlTable(self._package.sql_schema, name)
self.sql_samples_table = table - - if self.qmf_headers: - agent_col = SqlColumn(table, "_qmf_agent_id", sql_text) - object_col = SqlColumn(table, "_qmf_object_id", sql_text) - - name = "%s_qmf_id_idx" % table._name - SqlIndex(self._package.sql_schema, name, (agent_col, object_col)) - - SqlColumn(table, "_qmf_update_time", sql_timestamp) - self.sql_samples_insert = SqlInsertObjectSamples(table, self.qmf_headers) - if self.qmf_headers: - # We will need another way to delete non-qmf samples - self.sql_samples_delete = SqlDeleteObjectSamples(table)
def get_object(self, cursor, **criteria): sql = SqlSelect(self.sql_table) @@ -413,7 +409,7 @@ if not record: return
- obj = RosemaryObject(self, None) + obj = self._get_rosemary_object(None)
self._set_object_attributes(obj, self.sql_table._columns, record)
@@ -432,7 +428,7 @@ sql.execute(cursor, criteria)
for record in cursor.fetchall(): - obj = RosemaryObject(self, None) + obj = self._get_rosemary_object(None)
self._set_object_attributes(obj, self.sql_table._columns, record)
@@ -451,7 +447,7 @@ sql.execute(cursor, criteria)
for record in cursor.fetchall(): - obj = RosemaryObject(self, None) + obj = self._get_rosemary_object(None)
self.set_object_attributes(obj, self.sql_table._columns, record)
@@ -462,20 +458,12 @@ def get_object_by_id(self, cursor, id): assert id
- obj = RosemaryObject(self, id) + obj = self._get_rosemary_object(id)
self.load_object_by_id(cursor, obj)
return obj
- def get_object_by_qmf_id(self, cursor, agent_id, object_id): - assert self.qmf_headers - obj = RosemaryObject(self, None) - obj._qmf_agent_id = agent_id - obj._qmf_object_id = object_id - self.load_object_by_qmf_id(cursor, obj) - return obj - def get_new_id(self, cursor): self.sql_get_new_id.execute(cursor)
@@ -485,11 +473,17 @@
return values[0]
+ # Allow this to be overloaded so that + # descendents of RosemaryClass can produce + # descendents of RosemaryObject in create_object + # and other places + def _get_rosemary_object(self, id): + return RosemaryObject(self, id) + def create_object(self, cursor): id = self.get_new_id(cursor) + return self._get_rosemary_object(id)
- return RosemaryObject(self, id) - def load_object_by_id(self, cursor, obj): assert isinstance(obj, RosemaryObject) assert obj._id, obj @@ -503,21 +497,6 @@
self._set_object_attributes(obj, self.sql_table._columns, values)
- def load_object_by_qmf_id(self, cursor, obj): - assert isinstance(obj, RosemaryObject) - assert self.qmf_headers - assert obj._qmf_agent_id, obj - assert obj._qmf_object_id, obj - - self.sql_select_by_qmf_id.execute(cursor, obj.__dict__) - - values = cursor.fetchone() - - if not values: - raise RosemaryNotFound() - - self._set_object_attributes(obj, self.sql_table._columns, values) - def _set_object_attributes(self, obj, columns, values): assert isinstance(obj, RosemaryObject)
@@ -572,6 +551,182 @@ args = (self.__class__.__name__, self._package._name, self._name) return "%s(%s,%s)" % args
+class Qmf_class(RosemaryClass): + def __init__(self, package, name): + super(Qmf_class, self).__init__(package, name) + + self.qmf_headers = True + name = "_qmf_agent_id" + self._qmf_agent_id = RosemaryHeader(self, name, sql_text) + self._qmf_agent_id.title = "Agent ID" + + name = "_qmf_object_id" + self._qmf_object_id = RosemaryHeader(self, name, sql_text) + self._qmf_object_id.title = "Object ID" + + name = "_qmf_create_time" + self._qmf_create_time = RosemaryHeader(self, name, sql_timestamp) + self._qmf_create_time.title = "Create Time" + + name = "_qmf_update_time" + self._qmf_update_time = RosemaryHeader(self, name, sql_timestamp) + self._qmf_update_time.title = "Update Time" + + name = "_qmf_delete_time" + self._qmf_delete_time = RosemaryHeader(self, name, sql_timestamp) + self._qmf_delete_time.title = "Delete Time" + self._qmf_delete_time.optional = True + + def add_sql_operations(self): + super(Qmf_class, self).add_sql_operations() + self.sql_select_by_qmf_id = SqlSelectObjectByQmfId(self.sql_table) + + def add_sql_constraints(self): + # Add agent and object columns as a unique column set + name = "%s_qmf_id_uq" % self._name + self._unique_col_groups[name] = (self._qmf_agent_id.sql_column, + self._qmf_object_id.sql_column) + super(Qmf_class, self).add_sql_constraints() + + def add_samples_sql(self): + super(Qmf_class, self).add_samples_sql() + agent_col = SqlColumn(self.sql_samples_table, + "_qmf_agent_id", sql_text) + + object_col = SqlColumn(self.sql_samples_table, + "_qmf_object_id", sql_text) + + name = "%s_qmf_id_idx" % self.sql_samples_table._name + SqlIndex(self._package.sql_schema, name, (agent_col, object_col)) + + SqlColumn(self.sql_samples_table, "_qmf_update_time", sql_timestamp) + + # We will need another way to delete non-qmf samples + self.sql_samples_delete = SqlDeleteObjectSamples(self.sql_samples_table) + + def get_object_by_qmf_id(self, cursor, agent_id, object_id): + obj = self._get_rosemary_object(None) + obj._qmf_agent_id = agent_id + obj._qmf_object_id = object_id + self.load_object_by_qmf_id(cursor, obj) + return obj + + def load_object_by_qmf_id(self, cursor, obj): + assert isinstance(obj, RosemaryObject) + assert self.qmf_headers + assert obj._qmf_agent_id, obj + assert obj._qmf_object_id, obj + + self.sql_select_by_qmf_id.execute(cursor, obj.__dict__) + + values = cursor.fetchone() + + if not values: + raise RosemaryNotFound() + + self._set_object_attributes(obj, self.sql_table._columns, values) + + def get_signature(self, obj): + # Return an ordered set of values that can uniquely + # identify the object + return [obj._qmf_agent_id.replace(":", "|"), + obj._qmf_object_id] + + def get_object_by_signature(self, cursor, signature): + # Use the signature to find the object + agent_id, obj_id = signature + # open-flash-chart can't handle the : in the agent id. Net even if + # it's converted to %3A, so we have to change it + agent_id = agent_id.replace("|", ":") + return self.get_object_by_qmf_id(cursor, agent_id, obj_id) + + def get_sample_filters_by_signature(self, signature): + agent_id, obj_id = signature + agent_id = agent_id.replace("|", ":") + filters = [] + + col = self.sql_samples_table._qmf_agent_id + filters.append(SqlComparisonFilter(col, "'%s'" % agent_id)) + + col = self.sql_samples_table._qmf_object_id + filters.append(SqlComparisonFilter(col, "'%s'" % obj_id)) + + return filters + + # Allow this to be overloaded so that + # descendents of RosemaryClass can produce + # descendents of RosemaryObject in create_object + # and other places + def _get_rosemary_object(self, id): + return RosemaryQmfObject(self, id) + +class Plumage_class(RosemaryClass): + def __init__(self, package, name): + super(Plumage_class, self).__init__(package, name) + + # Columns marked unique will be added to the + # sample table + self.add_unique_to_samples = True + + def init(self): + super(Plumage_class, self).init() + + # Store a sorted list of the names of unique + # column groups + self.col_group_list = self._unique_col_groups.keys() + self.col_group_list.sort() + + # Todo: this could be optimized if we use the shortest + # set of unique column groups but probably there is only one + self.signature_cols = self._unique_col_groups[self.col_group_list[0]] + + # Set up a query that we can use on a unique + # column set + self.sql_select_signature = SqlSelect(self.sql_table) + for c in self.signature_cols: + self.sql_select_signature.add_filter(SqlValueFilter(c)) + + def load_class_components(self, elem): + for child in elem.findall("property"): + name = child.get("name") + attr = RosemaryProperty(self, name) + attr.load(child) + + for child in elem.findall("statistic"): + stat = RosemaryStatistic(self, child.get("name")) + stat.load(child) + + def get_signature(self, obj): + # Return an ordered set of values that can uniquely + # identify the object. + # For plumage objects, this is going to be the list + # of values in the columns marked unique. + sig = [] + for c in self.signature_cols: + # For each col, record the value from the object + sig.append(getattr(obj, c.name)) + return sig + + def get_object_by_signature(self, cursor, signature): + # signature is an ordered list of values corresponding + # to self.signature_cols + obj = self._get_rosemary_object(None) + sig_idx = 0 + for c in self.signature_cols: + setattr(obj, c.name, signature[sig_idx]) + sig_idx += 1 + self.load_object_by_signature(cursor, obj) + return obj + + def load_object_by_signature(self, cursor, obj): + assert isinstance(obj, RosemaryObject) + + self.sql_select_signature.execute(cursor, obj.__dict__) + values = cursor.fetchone() + if not values: + raise RosemaryNotFound() + self._set_object_attributes(obj, self.sql_table._columns, values) + class RosemaryAttribute(object): def __init__(self, cls, name): self.cls = cls @@ -598,6 +753,19 @@
self.sql_column = None
+ def process_unique(self, value): + # If unique is None, "y" or "n" then it + # is a boolean value and the column is not part + # of a group, it is simply unique by itself + if value is None: + return False + elif value in ("n", "y"): + return value == "y" + + # Otherwise, the column is part of a multi-field + # unique set where value is the name of the set + return value + def load(self, elem): self.type = elem.get("type") self.references = elem.get("references") @@ -605,12 +773,12 @@ self.unit = elem.get("unit") self.index = elem.get("index", "n") == "y" and True self.optional = elem.get("optional", "n") == "y" and True + self.unique = self.process_unique(elem.get("unique")) self.description = clean_description(elem.get("desc"))
def extend(self, elem): self.title = elem.findtext("title") self.short_title = elem.findtext("short") - self.unique = elem.get("unique", "n") == "y" and True formatter = elem.findtext("formatter")
# Allow overloading of optional in the extensions @@ -620,6 +788,11 @@ if optional is not None: self.optional = optional == "y"
+ # Same here + unique = elem.get("unique") + if unique is not None: + self.unique = self.process_unique(unique) + if formatter: meth = getattr(util, formatter, None)
@@ -712,10 +885,32 @@ self.sql_column = SqlColumn(self.cls.sql_table, self.name, type) self.sql_column.nullable = self.optional
- if self.unique: - name = "%s_%s_uq" % (self.cls._name, self.name) - SqlUniqueConstraint(self.cls.sql_table, name, (self.sql_column,)) + # Save away refernces for generation of unique constraints and indices + # If it's a single column marked unique, save it under its own name. + if self.unique == True: + name = "%s_%s_uq" % (self.cls.sql_table._name, self.name) + self.cls._unique_col_groups[name] = (self.sql_column,) + + # If it's part of a group, append it to a group of columns named + # for the group. + elif self.unique: + name = "%s_%s_uq" % (self.cls.sql_table._name, self.unique) + self.cls._unique_col_groups[name].append(self.sql_column)
+ # For some classes, add unique columns to the sample table, too. + # Produce an index for the column(s) based on the 'unique' value. + # These columns take the place of qmf_agent_id and qmf_object_id + # for non-qmf records + if self.cls.add_unique_to_samples and self.unique: + col = SqlColumn(self.cls.sql_samples_table, self.name, type) + col.nullable = self.optional + if self.unique == True: + name = "%s_%s_idx" % (self.cls.sql_samples_table._name, self.name) + self.cls._sample_index_col_groups[name] = (col,) + else: + name = "%s_%s_idx" % (self.cls.sql_samples_table._name, self.unique) + self.cls._sample_index_col_groups[name].append(col) + class RosemaryStatistic(RosemaryAttribute): def __init__(self, cls, name): super(RosemaryStatistic, self).__init__(cls, name) @@ -842,12 +1037,6 @@ def add_sample(self, cursor, columns=None): self._class.add_object_sample(cursor, self, columns)
- def fake_qmf_values(self): - self._qmf_agent_id = "__rosemary__" - self._qmf_object_id = str(datetime.now()) - self._qmf_create_time = datetime.now() - self._qmf_update_time = datetime.now() - def get_title(self): if self._class._object_title: return self._class._object_title % self.__dict__ @@ -877,11 +1066,30 @@
return value
+ def get_timestamp(self): + raise Exception("Not implemented") + def __repr__(self): name = self.__class__.__name__ args = (name, self._class, self._id, self._save_time) return "%s(%s,%i,%s)" % args
+class RosemaryQmfObject(RosemaryObject): + def fake_qmf_values(self): + self._qmf_agent_id = "__rosemary__" + self._qmf_object_id = str(datetime.now()) + self._qmf_create_time = datetime.now() + self._qmf_update_time = datetime.now() + + def get_timestamp(self): + return self._qmf_update_time + + def get_signature(self): + # return an ordered list of values that can uniquely + # identify this object. Let the class determine + # the list. + return self._class.get_signature(self) + class RosemaryNotFound(Exception): pass
Modified: branches/play_plumage/rosemary/python/rosemary/sqltype.py =================================================================== --- branches/play_plumage/rosemary/python/rosemary/sqltype.py 2012-04-09 16:56:04 UTC (rev 5286) +++ branches/play_plumage/rosemary/python/rosemary/sqltype.py 2012-04-09 19:48:49 UTC (rev 5287) @@ -76,3 +76,4 @@
sql_types_by_qmf_type_code = dict([(x[1], x[0]) for x in __mappings]) sql_types_by_qmf_type_string = dict([(x[2], x[0]) for x in __mappings]) +qmf_type_code_by_string = dict([(x[2], x[1]) for x in __mappings])